From 6510e0674c7e384902d9a20515b89c172aaa360b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 18 Jul 2023 18:03:45 +0300 Subject: [PATCH 001/211] Initial commit --- .gitignore | 1 + Cargo.toml | 21 ++ lib/vfs/Cargo.toml | 10 + lib/vfs/src/block.rs | 6 + lib/vfs/src/char.rs | 49 ++++ lib/vfs/src/file.rs | 88 ++++++ lib/vfs/src/fs.rs | 15 + lib/vfs/src/ioctx.rs | 216 ++++++++++++++ lib/vfs/src/lib.rs | 29 ++ lib/vfs/src/node.rs | 225 +++++++++++++++ src/arch/aarch64/boot/mod.rs | 128 +++++++++ src/arch/aarch64/context.S | 103 +++++++ src/arch/aarch64/context.rs | 172 ++++++++++++ src/arch/aarch64/cpu.rs | 111 ++++++++ src/arch/aarch64/devtree.rs | 161 +++++++++++ src/arch/aarch64/exception.rs | 173 ++++++++++++ src/arch/aarch64/gic/gicc.rs | 60 ++++ src/arch/aarch64/gic/gicd.rs | 161 +++++++++++ src/arch/aarch64/gic/mod.rs | 187 +++++++++++++ src/arch/aarch64/intrinsics.rs | 13 + src/arch/aarch64/mod.rs | 204 ++++++++++++++ src/arch/aarch64/plat_qemu/mod.rs | 83 ++++++ src/arch/aarch64/smp.rs | 131 +++++++++ src/arch/aarch64/table.rs | 451 ++++++++++++++++++++++++++++++ src/arch/aarch64/timer.rs | 78 ++++++ src/arch/aarch64/vectors.S | 128 +++++++++ src/arch/mod.rs | 48 ++++ src/debug.rs | 124 ++++++++ src/device/interrupt.rs | 78 ++++++ src/device/mod.rs | 21 ++ src/device/platform.rs | 51 ++++ src/device/serial/mod.rs | 15 + src/device/serial/pl011.rs | 187 +++++++++++++ src/device/timer.rs | 13 + src/device/tty.rs | 155 ++++++++++ src/fs/devfs.rs | 57 ++++ src/fs/mod.rs | 3 + src/main.rs | 86 ++++++ src/mem/device.rs | 76 +++++ src/mem/heap.rs | 51 ++++ src/mem/mod.rs | 90 ++++++ src/mem/phys/manager.rs | 95 +++++++ src/mem/phys/mod.rs | 199 +++++++++++++ src/mem/phys/reserved.rs | 33 +++ src/mem/table.rs | 40 +++ src/panic.rs | 77 +++++ src/proc/exec.rs | 101 +++++++ src/proc/io.rs | 71 +++++ src/proc/mod.rs | 91 ++++++ src/proc/wait.rs | 174 ++++++++++++ src/sync.rs | 177 ++++++++++++ src/syscall.rs | 145 ++++++++++ src/task/mod.rs | 110 ++++++++ src/task/process.rs | 246 ++++++++++++++++ src/task/sched.rs | 273 ++++++++++++++++++ src/util.rs | 117 ++++++++ 56 files changed, 6008 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 lib/vfs/Cargo.toml create mode 100644 lib/vfs/src/block.rs create mode 100644 lib/vfs/src/char.rs create mode 100644 lib/vfs/src/file.rs create mode 100644 lib/vfs/src/fs.rs create mode 100644 lib/vfs/src/ioctx.rs create mode 100644 lib/vfs/src/lib.rs create mode 100644 lib/vfs/src/node.rs create mode 100644 src/arch/aarch64/boot/mod.rs create mode 100644 src/arch/aarch64/context.S create mode 100644 src/arch/aarch64/context.rs create mode 100644 src/arch/aarch64/cpu.rs create mode 100644 src/arch/aarch64/devtree.rs create mode 100644 src/arch/aarch64/exception.rs create mode 100644 src/arch/aarch64/gic/gicc.rs create mode 100644 src/arch/aarch64/gic/gicd.rs create mode 100644 src/arch/aarch64/gic/mod.rs create mode 100644 src/arch/aarch64/intrinsics.rs create mode 100644 src/arch/aarch64/mod.rs create mode 100644 src/arch/aarch64/plat_qemu/mod.rs create mode 100644 src/arch/aarch64/smp.rs create mode 100644 src/arch/aarch64/table.rs create mode 100644 src/arch/aarch64/timer.rs create mode 100644 src/arch/aarch64/vectors.S create mode 100644 src/arch/mod.rs create mode 100644 src/debug.rs create mode 100644 src/device/interrupt.rs create mode 100644 src/device/mod.rs create mode 100644 src/device/platform.rs create mode 100644 src/device/serial/mod.rs create mode 100644 src/device/serial/pl011.rs create mode 100644 src/device/timer.rs create mode 100644 src/device/tty.rs create mode 100644 src/fs/devfs.rs create mode 100644 src/fs/mod.rs create mode 100644 src/main.rs create mode 100644 src/mem/device.rs create mode 100644 src/mem/heap.rs create mode 100644 src/mem/mod.rs create mode 100644 src/mem/phys/manager.rs create mode 100644 src/mem/phys/mod.rs create mode 100644 src/mem/phys/reserved.rs create mode 100644 src/mem/table.rs create mode 100644 src/panic.rs create mode 100644 src/proc/exec.rs create mode 100644 src/proc/io.rs create mode 100644 src/proc/mod.rs create mode 100644 src/proc/wait.rs create mode 100644 src/sync.rs create mode 100644 src/syscall.rs create mode 100644 src/task/mod.rs create mode 100644 src/task/process.rs create mode 100644 src/task/sched.rs create mode 100644 src/util.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..a45603cc --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "yggdrasil-kernel" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +vfs = { path = "lib/vfs" } + +aarch64-cpu = "9.3.1" +atomic_enum = "0.2.0" +bitflags = "2.3.3" +fdt-rs = { version = "0.4.3", default-features = false } +linked_list_allocator = "0.10.5" +spinning_top = "0.2.5" +static_assertions = "1.1.0" +tock-registers = "0.8.1" + +elf = { version = "0.7.2", default-features = false } diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml new file mode 100644 index 00000000..42c6b09a --- /dev/null +++ b/lib/vfs/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "vfs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +bitflags = "2.3.3" diff --git a/lib/vfs/src/block.rs b/lib/vfs/src/block.rs new file mode 100644 index 00000000..1c6faa90 --- /dev/null +++ b/lib/vfs/src/block.rs @@ -0,0 +1,6 @@ +use yggdrasil_abi::error::Error; + +pub trait BlockDevice { + fn read(&self, pos: usize, buf: &mut [u8]) -> Result<(), Error>; + fn write(&self, pos: usize, buf: &[u8]) -> Result<(), Error>; +} diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs new file mode 100644 index 00000000..848869f0 --- /dev/null +++ b/lib/vfs/src/char.rs @@ -0,0 +1,49 @@ +use yggdrasil_abi::error::Error; + +use crate::node::{VnodeImpl, VnodeRef}; + +pub trait CharDevice { + fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result; + fn write(&self, blocking: bool, data: &[u8]) -> Result; +} + +pub struct CharDeviceWrapper { + device: &'static dyn CharDevice, +} + +impl CharDeviceWrapper { + pub const fn new(device: &'static dyn CharDevice) -> Self { + Self { device } + } +} + +impl VnodeImpl for CharDeviceWrapper { + fn open( + &mut self, + _node: &VnodeRef, + _opts: yggdrasil_abi::io::OpenFlags, + ) -> Result { + Ok(0) + } + + fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + Ok(()) + } + + fn read(&mut self, _node: &VnodeRef, _pos: usize, data: &mut [u8]) -> Result { + self.device.read(true, data) + } + + fn write(&mut self, _node: &VnodeRef, _pos: usize, data: &[u8]) -> Result { + self.device.write(true, data) + } + + fn create( + &mut self, + _at: &VnodeRef, + _name: &str, + _kind: crate::node::VnodeKind, + ) -> Result { + todo!() + } +} diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs new file mode 100644 index 00000000..e3116880 --- /dev/null +++ b/lib/vfs/src/file.rs @@ -0,0 +1,88 @@ +use core::cell::RefCell; + +use alloc::rc::Rc; +use bitflags::bitflags; +use yggdrasil_abi::error::Error; + +use crate::{ + node::{VnodeKind, VnodeRef}, + Read, Write, +}; + +bitflags! { + pub struct FileFlags: u32 { + const READ = 1 << 0; + const WRITE = 1 << 1; + } +} + +pub type FileRef = Rc>; + +pub struct NormalFile { + vnode: VnodeRef, + pos: usize, +} + +pub enum FileInner { + Normal(NormalFile), +} + +pub struct File { + inner: FileInner, + flags: FileFlags, +} + +impl File { + pub fn normal(vnode: VnodeRef, pos: usize, flags: FileFlags) -> FileRef { + Rc::new(RefCell::new(Self { + inner: FileInner::Normal(NormalFile { vnode, pos }), + flags, + })) + } +} + +impl Write for File { + fn write(&mut self, data: &[u8]) -> Result { + if !self.flags.contains(FileFlags::WRITE) { + panic!(); + } + + match &mut self.inner { + FileInner::Normal(inner) => { + let count = inner.vnode.write(inner.pos, data)?; + if inner.vnode.kind() != VnodeKind::Char { + inner.pos += count; + } + Ok(count) + } + } + } +} + +impl Read for File { + fn read(&mut self, data: &mut [u8]) -> Result { + if !self.flags.contains(FileFlags::READ) { + panic!(); + } + + match &mut self.inner { + FileInner::Normal(inner) => { + let count = inner.vnode.read(inner.pos, data)?; + if inner.vnode.kind() != VnodeKind::Char { + inner.pos += count; + } + Ok(count) + } + } + } +} + +impl Drop for File { + fn drop(&mut self) { + match &mut self.inner { + FileInner::Normal(inner) => { + inner.vnode.close().ok(); + } + } + } +} diff --git a/lib/vfs/src/fs.rs b/lib/vfs/src/fs.rs new file mode 100644 index 00000000..27d51d91 --- /dev/null +++ b/lib/vfs/src/fs.rs @@ -0,0 +1,15 @@ +use core::{any::Any, cell::Ref}; + +use alloc::rc::Rc; + +use yggdrasil_abi::error::Error; + +use crate::{block::BlockDevice, node::VnodeRef}; + +pub trait Filesystem { + fn root(self: Rc) -> Result; + + fn dev(self: Rc) -> Option<&'static dyn BlockDevice>; + + fn data(&self) -> Option>; +} diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs new file mode 100644 index 00000000..007e302d --- /dev/null +++ b/lib/vfs/src/ioctx.rs @@ -0,0 +1,216 @@ +use yggdrasil_abi::{error::Error, io::OpenFlags, path}; + +use crate::{file::FileRef, node::VnodeRef}; + +pub struct IoContext { + root: VnodeRef, + cwd: VnodeRef, +} + +impl IoContext { + pub fn new(root: VnodeRef) -> Self { + Self { + cwd: root.clone(), + root, + } + } + + fn _find(&self, mut at: VnodeRef, path: &str, follow: bool) -> Result { + let mut element; + let mut rest = path; + + loop { + (element, rest) = path::split_left(rest); + + if !at.is_directory() { + todo!(); + } + + match element { + path::PARENT_NAME => { + at = at.parent(); + } + path::SELF_NAME => {} + _ => break, + } + } + + // TODO resolve link target + + if element.is_empty() && rest.is_empty() { + return Ok(at); + } + + let node = at.lookup_or_load(element)?; + + if rest.is_empty() { + Ok(node) + } else { + self._find(node, rest, follow) + } + } + + pub fn find( + &self, + at: Option, + mut path: &str, + follow: bool, + ) -> Result { + let at = if path.starts_with('/') { + path = path.trim_start_matches('/'); + self.root.clone() + } else if let Some(at) = at { + at + } else { + self.cwd.clone() + }; + + self._find(at, path, follow) + } + + pub fn open( + &self, + at: Option, + path: &str, + opts: OpenFlags, + ) -> Result { + let node = match self.find(at.clone(), path, true) { + Err(Error::DoesNotExist) => { + // TODO check for create option + return Err(Error::DoesNotExist); + } + o => o, + }?; + + node.open(opts) + } +} + +#[cfg(test)] +mod tests { + use abi::error::Error; + + use crate::{node::VnodeRef, IoContext}; + use std::fmt; + + macro_rules! node { + ($name:literal) => {{ + $crate::node::Vnode::new($name, $crate::node::VnodeKind::Regular) + }}; + + ($name:literal [ $($child:expr),* ]) => {{ + let _node = $crate::node::Vnode::new($name, $crate::node::VnodeKind::Directory); + + $( + _node.add_child($child); + )* + + _node + }}; + } + + struct DumpNode<'a> { + node: &'a VnodeRef, + } + + impl fmt::Debug for DumpNode<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.node.dump(f, 0) + } + } + + #[test] + fn test_vnode_find() { + let t = node! { + "" [ + node!("file1.txt"), + node!("file2.txt"), + node! { + "dir1" [ + node!("file3.txt") + ] + } + ] + }; + + let ctx = IoContext::new(t); + + // Absolute lookups + assert_eq!( + ctx.find(None, "/file1.txt", false).unwrap().name(), + "file1.txt" + ); + assert_eq!( + ctx.find(None, "/file3.txt", false).unwrap_err(), + Error::DoesNotExist + ); + assert_eq!( + ctx.find(None, "/dir1/file3.txt", false).unwrap().name(), + "file3.txt" + ); + + // Non-absolute lookups from root + assert_eq!( + ctx.find(None, "file1.txt", false).unwrap().name(), + "file1.txt" + ); + assert_eq!( + ctx.find(None, "dir1/file3.txt", false).unwrap().name(), + "file3.txt" + ); + + // Absolute lookups from non-root + let cwd = ctx.find(None, "/dir1", false).unwrap(); + + assert_eq!( + ctx.find(Some(cwd.clone()), "/file1.txt", false) + .unwrap() + .name(), + "file1.txt" + ); + assert_eq!( + ctx.find(Some(cwd.clone()), "/dir1/file3.txt", false) + .unwrap() + .name(), + "file3.txt" + ); + assert_eq!( + ctx.find(Some(cwd.clone()), "/file3.txt", false) + .unwrap_err(), + Error::DoesNotExist + ); + assert_eq!( + ctx.find(Some(cwd.clone()), "/dir2", false).unwrap_err(), + Error::DoesNotExist + ); + + // Non-absolute lookups in non-root + assert_eq!( + ctx.find(Some(cwd.clone()), "file3.txt", false) + .unwrap() + .name(), + "file3.txt" + ); + assert_eq!( + ctx.find(Some(cwd.clone()), "././file3.txt", false) + .unwrap() + .name(), + "file3.txt" + ); + assert_eq!( + ctx.find(Some(cwd.clone()), "../dir1/file3.txt", false) + .unwrap() + .name(), + "file3.txt" + ); + assert_eq!( + ctx.find(Some(cwd.clone()), ".", false).unwrap().name(), + "dir1" + ); + assert_eq!(ctx.find(Some(cwd.clone()), "..", false).unwrap().name(), ""); + assert_eq!( + ctx.find(Some(cwd.clone()), "../..", false).unwrap().name(), + "" + ); + } +} diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs new file mode 100644 index 00000000..4e005159 --- /dev/null +++ b/lib/vfs/src/lib.rs @@ -0,0 +1,29 @@ +#![no_std] + +use yggdrasil_abi::error::Error; + +extern crate alloc; + +#[cfg(test)] +extern crate std; + +pub(crate) mod block; +pub(crate) mod char; +pub(crate) mod file; +pub(crate) mod fs; +pub(crate) mod ioctx; +pub(crate) mod node; + +pub use self::block::BlockDevice; +pub use self::char::{CharDevice, CharDeviceWrapper}; +pub use file::{File, FileFlags, FileRef}; +pub use ioctx::IoContext; +pub use node::{Vnode, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; + +pub trait Write { + fn write(&mut self, data: &[u8]) -> Result; +} + +pub trait Read { + fn read(&mut self, data: &mut [u8]) -> Result; +} diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs new file mode 100644 index 00000000..bb439309 --- /dev/null +++ b/lib/vfs/src/node.rs @@ -0,0 +1,225 @@ +use core::{ + cell::{RefCell, RefMut}, + fmt, +}; + +use alloc::{ + boxed::Box, + rc::{Rc, Weak}, + string::String, + vec::Vec, +}; +use yggdrasil_abi::{error::Error, io::OpenFlags}; + +use crate::file::{File, FileFlags, FileRef}; + +pub type VnodeRef = Rc; +pub type VnodeWeak = Weak; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum VnodeKind { + Directory, + Regular, + Char, + Block, +} + +pub(crate) struct TreeNode { + parent: Option, + children: Vec, +} + +pub struct Vnode { + name: String, + tree: RefCell, + kind: VnodeKind, + data: RefCell>>, +} + +pub trait VnodeImpl { + fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result; + + fn open(&mut self, node: &VnodeRef, opts: OpenFlags) -> Result; + fn close(&mut self, node: &VnodeRef) -> Result<(), Error>; + + fn read(&mut self, node: &VnodeRef, pos: usize, data: &mut [u8]) -> Result; + fn write(&mut self, node: &VnodeRef, pos: usize, data: &[u8]) -> Result; +} + +impl Vnode { + pub fn new>(name: S, kind: VnodeKind) -> VnodeRef { + Rc::new(Self { + name: name.into(), + tree: RefCell::new(TreeNode { + parent: None, + children: Vec::new(), + }), + kind, + data: RefCell::new(None), + }) + } + + #[inline] + pub fn name(&self) -> &str { + &self.name + } + + #[inline] + pub fn kind(&self) -> VnodeKind { + self.kind + } + + #[inline] + pub fn data(&self) -> RefMut>> { + self.data.borrow_mut() + } + + pub fn parent(self: &VnodeRef) -> VnodeRef { + match &self.tree.borrow().parent { + Some(parent) => parent.upgrade().unwrap(), + None => self.clone(), + } + } + + pub fn set_data(&self, data: Box) { + self.data.borrow_mut().replace(data); + } + + #[inline] + pub fn is_directory(&self) -> bool { + self.kind == VnodeKind::Directory + } + + // Cache tree operations + pub fn add_child(self: &VnodeRef, child: VnodeRef) { + let parent_weak = Rc::downgrade(self); + let mut parent_borrow = self.tree.borrow_mut(); + + assert!(child + .tree + .borrow_mut() + .parent + .replace(parent_weak) + .is_none()); + parent_borrow.children.push(child); + } + + pub fn dump(&self, f: &mut fmt::Formatter<'_>, depth: usize) -> fmt::Result { + for _ in 0..depth { + f.write_str(" ")?; + } + + write!(f, "{:?}", self.name)?; + + match self.kind { + VnodeKind::Directory => { + let tree = self.tree.borrow(); + if tree.children.is_empty() { + f.write_str(" []")?; + } else { + f.write_str(" [\n")?; + for child in tree.children.iter() { + child.dump(f, depth + 1)?; + f.write_str("\n")?; + } + for _ in 0..depth { + f.write_str(" ")?; + } + f.write_str("]")?; + } + } + _ => (), + } + + Ok(()) + } + + pub fn lookup(self: &VnodeRef, name: &str) -> Option { + assert!(self.is_directory()); + self.tree + .borrow() + .children + .iter() + .find(|e| e.name == name) + .cloned() + } + + // + pub fn lookup_or_load(self: &VnodeRef, name: &str) -> Result { + // Lookup in cache + if let Some(node) = self.lookup(name) { + return Ok(node); + } + + // TODO load from FS + Err(Error::DoesNotExist) + } + + // Node operations + pub fn open(self: &VnodeRef, flags: OpenFlags) -> Result { + let mut open_flags = FileFlags::empty(); + + if flags.is_read() { + open_flags |= FileFlags::READ; + } + if flags.is_write() { + open_flags |= FileFlags::WRITE; + } + + if self.kind == VnodeKind::Directory { + return Err(Error::IsADirectory); + } + + if let Some(ref mut data) = *self.data() { + let pos = data.open(self, flags)?; + Ok(File::normal(self.clone(), pos, open_flags)) + } else { + todo!() + } + } + + pub fn close(self: &VnodeRef) -> Result<(), Error> { + if let Some(ref mut data) = *self.data() { + data.close(self) + } else { + todo!() + } + } + + pub fn write(self: &VnodeRef, pos: usize, buf: &[u8]) -> Result { + if self.kind == VnodeKind::Directory { + todo!(); + } + + if let Some(ref mut data) = *self.data() { + data.write(self, pos, buf) + } else { + todo!() + } + } + + pub fn read(self: &VnodeRef, pos: usize, buf: &mut [u8]) -> Result { + if self.kind == VnodeKind::Directory { + todo!(); + } + + if let Some(ref mut data) = *self.data() { + data.read(self, pos, buf) + } else { + todo!() + } + } +} + +impl fmt::Debug for Vnode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let prefix = match self.kind { + VnodeKind::Directory => "DIR ", + VnodeKind::Regular => "REG ", + VnodeKind::Char => "CHR ", + VnodeKind::Block => "BLK ", + }; + + write!(f, "[{} {}]", prefix, self.name) + } +} diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs new file mode 100644 index 00000000..87130ca7 --- /dev/null +++ b/src/arch/aarch64/boot/mod.rs @@ -0,0 +1,128 @@ +//! Main entry point for the AArch64 platforms +use core::{arch::asm, sync::atomic::Ordering}; + +use aarch64_cpu::registers::{CurrentEL, CPACR_EL1}; +use tock_registers::interfaces::{ReadWriteable, Readable}; + +use super::{ + cpu::Cpu, exception, kernel_main, smp::CPU_COUNT, AArch64, KernelStack, ARCHITECTURE, + BOOT_STACK_SIZE, +}; +use crate::{ + absolute_address, + arch::{Architecture, PLATFORM}, + device::platform::Platform, + mem::{ConvertAddress, KERNEL_VIRT_OFFSET}, + sync::SpinFence, + task, +}; + +pub(super) static CPU_INIT_FENCE: SpinFence = SpinFence::new(); + +fn __aarch64_common_lower_entry() { + // Unmask FP operations + CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); + + if CurrentEL.read(CurrentEL::EL) != 1 { + panic!("Only EL1 is supported for now"); + } +} + +fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { + unsafe { + asm!(r#" + mov sp, {sp} + mov x0, {arg} + br {entry} + "#, entry = in(reg) elr, arg = in(reg) arg, sp = in(reg) sp, options(noreturn)); + } +} + +pub(super) extern "C" fn __aarch64_ap_lower_entry(sp: usize) -> ! { + __aarch64_common_lower_entry(); + + unsafe { + ARCHITECTURE.init_mmu(false); + } + + let sp = unsafe { sp.virtualize() }; + let elr = absolute_address!(__aarch64_ap_upper_entry); + enter_higher_half(sp, elr, 0); +} + +extern "C" fn __aarch64_bsp_lower_entry(dtb_phys: usize) -> ! { + __aarch64_common_lower_entry(); + + unsafe { + ARCHITECTURE.init_mmu(true); + } + + let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE).virtualize() }; + let elr = absolute_address!(__aarch64_bsp_upper_entry); + enter_higher_half(sp as usize, elr, dtb_phys); +} + +extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { + kernel_main(dtb_phys); +} + +extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { + unsafe { + AArch64::set_interrupt_mask(true); + } + + // Signal to BSP that we're up + CPU_COUNT.fetch_add(1, Ordering::SeqCst); + aarch64_cpu::asm::sev(); + + exception::init_exceptions(); + + // Initialize CPU-local GIC and timer + unsafe { + PLATFORM.init(false).expect("AP platform init failed"); + + Cpu::init_local(); + + // Synchronize the CPUs to this point + CPU_INIT_FENCE.signal(); + CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); + + task::enter(); + } +} + +#[link_section = ".text.entry"] +#[no_mangle] +#[naked] +unsafe extern "C" fn __aarch64_entry() -> ! { + // Setup the stack and pass on to a proper function + asm!( + r#" + // Multiple processor cores may or may not be running at this point + mrs x1, mpidr_el1 + ands x1, x1, #0xF + bne 1f + + // BSP in SMP or uniprocessor + ldr x1, ={stack_bottom} + {stack_size} - {kernel_virt_offset} + mov sp, x1 + + bl {kernel_lower_entry} - {kernel_virt_offset} + + // AP in a SMP system + // TODO spin loop for this method of init +1: + b . +"#, + kernel_lower_entry = sym __aarch64_bsp_lower_entry, + stack_bottom = sym BSP_STACK, + stack_size = const BOOT_STACK_SIZE, + kernel_virt_offset = const KERNEL_VIRT_OFFSET, + options(noreturn) + ); +} + +#[link_section = ".bss"] +static BSP_STACK: KernelStack = KernelStack { + data: [0; BOOT_STACK_SIZE], +}; diff --git a/src/arch/aarch64/context.S b/src/arch/aarch64/context.S new file mode 100644 index 00000000..2b918c12 --- /dev/null +++ b/src/arch/aarch64/context.S @@ -0,0 +1,103 @@ +.global __aarch64_enter_task +.global __aarch64_switch_task + +.section .text + +.macro SAVE_TASK_STATE + sub sp, sp, #{context_size} + + stp x19, x20, [sp, #16 * 0] + stp x21, x22, [sp, #16 * 1] + stp x23, x24, [sp, #16 * 2] + stp x25, x26, [sp, #16 * 3] + stp x27, x28, [sp, #16 * 4] + stp x29, x30, [sp, #16 * 5] + + mrs x19, tpidr_el0 + mrs x20, ttbr0_el1 + stp x29, x20, [sp, #16 * 6] +.endm + +.macro LOAD_TASK_STATE + // x19 == tpidr_el0, x20 = ttbr0_el1 + ldp x19, x20, [sp, #16 * 6] + msr tpidr_el0, x19 + msr ttbr0_el1, x20 + + ldp x19, x20, [sp, #16 * 0] + ldp x21, x22, [sp, #16 * 1] + ldp x23, x24, [sp, #16 * 2] + ldp x25, x26, [sp, #16 * 3] + ldp x27, x28, [sp, #16 * 4] + ldp x29, x30, [sp, #16 * 5] + + add sp, sp, #{context_size} +.endm + +__aarch64_task_enter_kernel: + # EL1h, IRQs unmasked + mov x0, #5 + msr spsr_el1, x0 + + # x0 == argument, x1 == entry point + ldp x0, x1, [sp, #0] + msr elr_el1, x1 + + add sp, sp, #16 + + eret + +__aarch64_task_enter_user: + // x0 == sp, x1 == ignored + ldp x0, x1, [sp, #16 * 0] + msr sp_el0, x0 + + # EL0t, IRQs unmasked + msr spsr_el1, xzr + + // x0 == arg, x1 == entry + ldp x0, x1, [sp, #16 * 1] + msr elr_el1, x1 + add sp, sp, #32 + + // Zero the registers + mov x1, xzr + mov x2, xzr + mov x3, xzr + mov x4, xzr + mov x5, xzr + mov x6, xzr + mov x7, xzr + mov x8, xzr + mov x9, xzr + mov x10, xzr + mov x11, xzr + mov x12, xzr + mov x13, xzr + mov x14, xzr + mov x15, xzr + mov x16, xzr + mov x17, xzr + mov x18, xzr + + mov lr, xzr + + eret + +__aarch64_switch_task: + SAVE_TASK_STATE + mov x19, sp + str x19, [x1] + + ldr x0, [x0] + mov sp, x0 + LOAD_TASK_STATE + + ret + +__aarch64_enter_task: + ldr x0, [x0] + mov sp, x0 + LOAD_TASK_STATE + + ret diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs new file mode 100644 index 00000000..8120a97f --- /dev/null +++ b/src/arch/aarch64/context.rs @@ -0,0 +1,172 @@ +//! AArch64-specific task context implementation +use core::{arch::global_asm, cell::UnsafeCell}; + +use abi::error::Error; +use alloc::boxed::Box; + +use crate::mem::{ + phys::{self, PageUsage}, + ConvertAddress, +}; + +struct StackBuilder { + base: usize, + sp: usize, +} + +#[repr(C, align(0x10))] +struct TaskContextInner { + // 0x00 + sp: usize, +} + +/// AArch64 implementation of a task context +pub struct TaskContext { + inner: UnsafeCell, +} + +const COMMON_CONTEXT_SIZE: usize = 8 * 14; + +unsafe impl Sync for TaskContext {} + +impl StackBuilder { + fn new(base: usize, size: usize) -> Self { + Self { + base, + sp: base + size, + } + } + + fn push(&mut self, value: usize) { + if self.sp == self.base { + panic!(); + } + self.sp -= 8; + unsafe { + (self.sp as *mut usize).write_volatile(value); + } + } + + fn _skip(&mut self, count: usize) { + self.sp -= count * 8; + if self.sp < self.base { + panic!(); + } + } + + fn build(self) -> usize { + self.sp + } + + fn init_common(&mut self, entry: usize, ttbr0: usize) { + self.push(ttbr0); // ttbr0_el1 + self.push(0); // tpidr_el0 + + self.push(entry); // x30/lr + self.push(0); // x29 + self.push(0); // x28 + self.push(0); // x27 + self.push(0); // x26 + self.push(0); // x25 + self.push(0); // x24 + self.push(0); // x23 + self.push(0); // x22 + self.push(0); // x21 + self.push(0); // x20 + self.push(0); // x19 + } +} + +impl TaskContext { + /// Constructs a kernel thread context. For a more convenient way of constructing kernel + /// processes, see [TaskContext::kernel_closure()]. + pub fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { + const KERNEL_TASK_PAGES: usize = 4; + let stack_base = unsafe { + phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() + }; + + let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); + + // Entry and argument + stack.push(entry as _); + stack.push(arg); + + stack.init_common(__aarch64_task_enter_kernel as _, 0); + + let sp = stack.build(); + + // TODO stack is leaked + + Ok(Self { + inner: UnsafeCell::new(TaskContextInner { sp }), + }) + } + + /// Constructs a safe wrapper process to execute a kernel-space closure + pub fn kernel_closure(f: F) -> Result { + extern "C" fn closure_wrapper(closure_addr: usize) -> ! { + let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; + closure(); + todo!("Process termination"); + } + + let closure = Box::new(f); + Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) + } + + /// Constructs a user thread context. The caller is responsible for allocating the userspace + /// stack and setting up a valid address space for the context. + pub fn user( + entry: usize, + arg: usize, + ttbr0: usize, + user_stack_sp: usize, + ) -> Result { + const USER_TASK_PAGES: usize = 8; + let stack_base = + unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + stack.push(entry as _); + stack.push(arg); + stack.push(0); + stack.push(user_stack_sp); + + stack.init_common(__aarch64_task_enter_user as _, ttbr0); + + let sp = stack.build(); + + Ok(Self { + inner: UnsafeCell::new(TaskContextInner { sp }), + }) + } + + /// Starts execution of `self` task on local CPU. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + pub unsafe fn enter(&self) -> ! { + __aarch64_enter_task(self.inner.get()) + } + + /// Switches from `from` task to `self` task. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + pub unsafe fn switch(&self, from: &Self) { + __aarch64_switch_task(self.inner.get(), from.inner.get()) + } +} + +extern "C" { + fn __aarch64_enter_task(to: *mut TaskContextInner) -> !; + fn __aarch64_switch_task(to: *mut TaskContextInner, from: *mut TaskContextInner); + fn __aarch64_task_enter_kernel(); + fn __aarch64_task_enter_user(); +} + +global_asm!(include_str!("context.S"), context_size = const COMMON_CONTEXT_SIZE); diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs new file mode 100644 index 00000000..dab7db23 --- /dev/null +++ b/src/arch/aarch64/cpu.rs @@ -0,0 +1,111 @@ +//! Per-CPU data structures +use core::sync::atomic::Ordering; + +use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; +use alloc::{boxed::Box, vec::Vec}; +use tock_registers::interfaces::{Readable, Writeable}; + +use crate::{arch::CpuMessage, sync::IrqSafeSpinlock, task::sched::CpuQueue, util::OneTimeInit}; + +use super::smp::CPU_COUNT; + +/// Per-CPU private data structure +#[repr(C, align(0x10))] +pub struct Cpu { + id: u32, + + queue: OneTimeInit<&'static CpuQueue>, +} + +struct IpiQueue { + data: IrqSafeSpinlock>, +} + +static IPI_QUEUES: OneTimeInit> = OneTimeInit::new(); + +impl IpiQueue { + pub const fn new() -> Self { + Self { + data: IrqSafeSpinlock::new(None), + } + } + + pub fn push(&self, msg: CpuMessage) { + let mut lock = self.data.lock(); + + assert!(lock.is_none()); + lock.replace(msg); + } + + pub fn pop(&self) -> Option { + let mut lock = self.data.lock(); + lock.take() + } +} + +impl Cpu { + /// Returns a safe reference to the local CPU's private data structure + #[inline(always)] + pub fn local<'a>() -> &'a Self { + Self::get_local().unwrap() + } + + /// Returns the local CPU data structure reference, if it was set up + #[inline(always)] + pub fn get_local<'a>() -> Option<&'a Self> { + let tpidr = TPIDR_EL1.get() as *mut Cpu; + unsafe { tpidr.as_ref() } + } + + /// Sets up the local CPU's private data structure. + /// + /// # Safety + /// + /// The function is only meant to be called once during the early init process. + pub unsafe fn init_local() { + let this = Box::new(Cpu { + id: Self::local_id(), + queue: OneTimeInit::new(), + }); + TPIDR_EL1.set(Box::into_raw(this) as _); + } + + /// Sets up the local CPU's execution queue. + pub fn init_queue(&self, queue: &'static CpuQueue) { + self.queue.init(queue); + } + + /// Returns the local CPU's execution queue. + pub fn queue(&self) -> &'static CpuQueue { + self.queue.get() + } + + /// Returns the index of the local CPU + #[inline(always)] + pub fn local_id() -> u32 { + (MPIDR_EL1.get() & 0xFF) as _ + } + + /// Inserts an IPI message to the back of the target CPU's message queue + pub fn push_ipi_queue(cpu_id: u32, msg: CpuMessage) { + let ipi_queue = &IPI_QUEUES.get()[cpu_id as usize]; + ipi_queue.push(msg); + } + + /// Pops the first IPI message received for this CPU. + /// + /// # Note + /// + /// Currently the queue consists of only one entry, so the CPU will only receive the last one. + pub fn get_ipi(&self) -> Option { + let ipi_queue = &IPI_QUEUES.get()[self.id as usize]; + ipi_queue.pop() + } + + /// Sets up global list of interprocessor message queues + pub fn init_ipi_queues() { + IPI_QUEUES.init(Vec::from_iter( + (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), + )); + } +} diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs new file mode 100644 index 00000000..f18ad736 --- /dev/null +++ b/src/arch/aarch64/devtree.rs @@ -0,0 +1,161 @@ +//! ARM device tree utlities +use fdt_rs::{ + base::DevTree, + index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, + prelude::PropReader, +}; + +use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; + +#[repr(C, align(0x10))] +struct FdtIndexBuffer([u8; 32768]); + +static mut FDT_INDEX_BUFFER: FdtIndexBuffer = FdtIndexBuffer::zeroed(); + +impl FdtIndexBuffer { + const fn zeroed() -> Self { + Self([0; 32768]) + } +} + +/// Device tree node +pub type TNode<'a> = DevTreeIndexNode<'a, 'a, 'a>; +/// Device tree property +pub type TProp<'a> = DevTreeIndexProp<'a, 'a, 'a>; + +/// Iterator for physical memory regions present in the device tree +#[derive(Clone)] +pub struct FdtMemoryRegionIter<'a> { + inner: DevTreeIndexNodeSiblingIter<'a, 'a, 'a>, +} + +/// Device tree wrapper struct +pub struct DeviceTree<'a> { + tree: DevTree<'a>, + index: DevTreeIndex<'a, 'a>, +} + +impl<'a> DeviceTree<'a> { + /// Constructs a device tree wrapper from the DTB virtual address. + /// + /// # Safety + /// + /// The caller must ensure the validity of the address. + pub unsafe fn from_addr(virt: usize) -> Self { + let tree = DevTree::from_raw_pointer(virt as _).unwrap(); + let index = DevTreeIndex::new(tree, &mut FDT_INDEX_BUFFER.0).unwrap(); + Self { tree, index } + } + + /// Looks up a node for a given path + pub fn node_by_path(&self, path: &str) -> Option { + find_node(self.index.root(), path.trim_start_matches('/')) + } + + /// Prints the device tree to log output + pub fn dump(&self, level: LogLevel) { + dump_node(&self.index.root(), 0, level) + } + + /// Returns the total size of the device tree in memory + pub fn size(&self) -> usize { + self.tree.totalsize() + } +} + +impl<'a> FdtMemoryRegionIter<'a> { + /// Constructs a memory region iterator for given device tree + pub fn new(dt: &'a DeviceTree) -> Self { + let inner = dt.index.root().children(); + Self { inner } + } +} + +impl Iterator for FdtMemoryRegionIter<'_> { + type Item = PhysicalMemoryRegion; + + fn next(&mut self) -> Option { + loop { + let Some(item) = self.inner.next() else { + break None; + }; + + if item.name().unwrap_or("").starts_with("memory@") { + let reg = item + .props() + .find(|p| p.name().unwrap_or("") == "reg") + .unwrap(); + + break Some(PhysicalMemoryRegion { + base: reg.u64(0).unwrap() as usize, + size: reg.u64(1).unwrap() as usize, + }); + } + } + } +} + +/// Looks up a property with given name in the node +pub fn find_prop<'a>(node: &TNode<'a>, name: &str) -> Option> { + node.props().find(|p| p.name().unwrap_or("") == name) +} + +fn path_component_left(path: &str) -> (&str, &str) { + if let Some((left, right)) = path.split_once('/') { + (left, right.trim_start_matches('/')) + } else { + (path, "") + } +} + +fn find_node<'a>(at: TNode<'a>, path: &str) -> Option> { + let (item, path) = path_component_left(path); + if item.is_empty() { + assert_eq!(path, ""); + Some(at) + } else { + let child = at.children().find(|c| c.name().unwrap() == item)?; + if path.is_empty() { + Some(child) + } else { + find_node(child, path) + } + } +} + +fn dump_node(node: &TNode, depth: usize, level: LogLevel) { + fn indent(level: LogLevel, depth: usize) { + for _ in 0..depth { + log_print!(level, " "); + } + } + + let node_name = node.name().unwrap(); + + // Don't dump these + if node_name.starts_with("virtio_mmio@") { + return; + } + + indent(level, depth); + log_print!(level, "{:?} {{\n", node_name); + for prop in node.props() { + indent(level, depth + 1); + let name = prop.name().unwrap(); + log_print!(level, "{name:?} = "); + + match name { + "compatible" | "stdout-path" => log_print!(level, "{:?}", prop.str().unwrap()), + _ => log_print!(level, "{:x?}", prop.raw()), + } + + log_print!(level, "\n"); + } + + for child in node.children() { + dump_node(&child, depth + 1, level); + } + + indent(level, depth); + log_print!(level, "}}\n"); +} diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs new file mode 100644 index 00000000..e9fd41a1 --- /dev/null +++ b/src/arch/aarch64/exception.rs @@ -0,0 +1,173 @@ +//! Exception and interrupt management functions +use core::{arch::global_asm, fmt}; + +use aarch64_cpu::registers::{ELR_EL1, ESR_EL1, FAR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; +use tock_registers::interfaces::{Readable, Writeable}; + +use crate::{ + arch::{aarch64::cpu::Cpu, CpuMessage, PLATFORM}, + debug::LogLevel, + device::{interrupt::IrqContext, platform::Platform}, + panic::panic_secondary, + syscall::raw_syscall_handler, + task::process::Process, +}; + +/// Struct for register values saved when taking an exception +#[repr(C)] +pub struct ExceptionFrame { + r: [u64; 32], + c: [u64; 4], + // ... +} + +impl fmt::Debug for ExceptionFrame { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for i in (0..32).step_by(2) { + write!( + f, + "x{:<2} = {:#020x}\tx{:<2} = {:#020x}", + i, + self.r[i], + i + 1, + self.r[i + 1] + )?; + if i != 30 { + f.write_str("\n")?; + } + } + + Ok(()) + } +} + +/// Initializes the exception/interrupt vectors. May be called repeatedly (though that makes no +/// sense). +pub fn init_exceptions() { + extern "C" { + static __aarch64_el1_vectors: u8; + } + let vbar = unsafe { &__aarch64_el1_vectors as *const _ }; + VBAR_EL1.set(vbar as u64); +} + +fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { + let cpu = Cpu::get_local(); + + log_print_raw!(LogLevel::Fatal, "SYNC exception:\n"); + log_print_raw!(LogLevel::Fatal, "FAR: {:#x}\n", FAR_EL1.get()); + log_print_raw!(LogLevel::Fatal, "ELR: {:#x}\n", ELR_EL1.get()); + log_print_raw!(LogLevel::Fatal, "ESR: {:#x}\n", ESR_EL1.get()); + log_print_raw!(LogLevel::Fatal, "TTBR0_EL1: {:#x}\n", TTBR0_EL1.get()); + log_print_raw!(LogLevel::Fatal, "TTBR1_EL1: {:#x}\n", TTBR1_EL1.get()); + log_print_raw!(LogLevel::Fatal, "Register dump:\n"); + log_print_raw!(LogLevel::Fatal, "{:?}\n", frame); + + if let Some(cpu) = cpu { + let current = cpu.queue().current_process(); + + if let Some(current) = current { + log_print_raw!(LogLevel::Fatal, "In process {}\n", current.id()); + } + } + + match ec { + // Data abort from lower level + 0b100100 => { + log_print_raw!(LogLevel::Fatal, "Exception kind: Data Abort from EL0\n"); + let dfsc = iss & 0x3F; + + if iss & (1 << 24) != 0 { + let access_size_str = match (iss >> 22) & 0x3 { + 0 => "i8", + 1 => "i16", + 2 => "i32", + 3 => "i64", + _ => unreachable!(), + }; + let access_type_str = if iss & (1 << 6) != 0 { "write" } else { "read" }; + + log_print_raw!( + LogLevel::Fatal, + "Invalid {} of a {} to/from {:#x}\n", + access_type_str, + access_size_str, + FAR_EL1.get() + ); + } + + log_print_raw!(LogLevel::Fatal, "DFSC = {:#x}\n", dfsc); + } + // Instruction abort from lower level + 0b100000 => { + log_print_raw!( + LogLevel::Fatal, + "Exception kind: Instruction Abort from EL0\n" + ); + let ifsc = iss & 0x3F; + log_print_raw!(LogLevel::Fatal, "IFSC = {:#x}\n", ifsc); + } + + _ => (), + } +} + +#[no_mangle] +extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { + let frame = unsafe { &mut *frame }; + + let esr_el1 = ESR_EL1.get(); + let ec = (esr_el1 >> 26) & 0x3F; + + match ec { + // SVC in AArch64 + 0b010101 => { + let func = frame.r[8]; + let args = &frame.r[0..6]; + let result = raw_syscall_handler(func, args); + frame.r[0] = result; + } + // BRK in AArch64 + 0b111100 => { + Process::current().exit(1); + panic!("Cannot return here"); + } + _ => { + let iss = esr_el1 & 0x1FFFFFF; + dump_irrecoverable_exception(frame, ec, iss); + + panic!("Irrecoverable exception"); + } + } +} + +#[no_mangle] +extern "C" fn __aa64_exc_irq_handler(_frame: *mut ExceptionFrame) { + unsafe { + let ic = IrqContext::new(); + PLATFORM.interrupt_controller().handle_pending_irqs(&ic); + } +} + +#[no_mangle] +extern "C" fn __aa64_exc_fiq_handler() { + todo!(); +} + +#[no_mangle] +extern "C" fn __aa64_exc_serror_handler() { + todo!(); +} + +pub(super) fn ipi_handler(msg: Option) { + if let Some(msg) = msg { + match msg { + CpuMessage::Panic => panic_secondary(), + } + } else { + warnln!("Spurious IPI received by cpu{}", Cpu::local_id()); + todo!(); + } +} + +global_asm!(include_str!("vectors.S")); diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs new file mode 100644 index 00000000..4a591f67 --- /dev/null +++ b/src/arch/aarch64/gic/gicc.rs @@ -0,0 +1,60 @@ +//! ARM GICv2 CPU registers +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::ReadWrite, +}; + +use crate::{device::interrupt::IrqContext, mem::device::DeviceMemoryIo}; + +register_bitfields! { + u32, + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + PMR [ + Priority OFFSET(0) NUMBITS(8) [] + ], + IAR [ + InterruptID OFFSET(0) NUMBITS(10) [] + ], + EOIR [ + EOINTID OFFSET(0) NUMBITS(10) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + pub(super) GiccRegs { + (0x00 => CTLR: ReadWrite), + (0x04 => PMR: ReadWrite), + (0x08 => _0), + (0x0C => IAR: ReadWrite), + (0x10 => EOIR: ReadWrite), + (0x14 => @END), + } +} + +pub(super) struct Gicc { + regs: DeviceMemoryIo, +} + +impl Gicc { + pub const fn new(regs: DeviceMemoryIo) -> Self { + Self { regs } + } + + pub unsafe fn init(&self) { + debugln!("Enabling GICv2 GICC"); + self.regs.CTLR.write(CTLR::Enable::SET); + self.regs.PMR.write(PMR::Priority.val(0xFF)); + } + + pub fn pending_irq_number<'irq>(&'irq self, _ic: &IrqContext<'irq>) -> usize { + self.regs.IAR.read(IAR::InterruptID) as usize + } + + pub fn clear_irq<'irq>(&'irq self, irq: usize, _ic: &IrqContext<'irq>) { + self.regs.EOIR.write(EOIR::EOINTID.val(irq as u32)); + } +} diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs new file mode 100644 index 00000000..e428f7ae --- /dev/null +++ b/src/arch/aarch64/gic/gicd.rs @@ -0,0 +1,161 @@ +//! ARM GICv2 Distributor registers +use spinning_top::Spinlock; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite, WriteOnly}, +}; + +use crate::{device::interrupt::IpiDeliveryTarget, mem::device::DeviceMemoryIo}; + +use super::IrqNumber; + +register_bitfields! { + u32, + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + TYPER [ + ITLinesNumber OFFSET(0) NUMBITS(5) [] + ], + ITARGETSR [ + Offset3 OFFSET(24) NUMBITS(8) [], + Offset2 OFFSET(16) NUMBITS(8) [], + Offset1 OFFSET(8) NUMBITS(8) [], + Offset0 OFFSET(0) NUMBITS(8) [] + ], + SGIR [ + TargetListFilter OFFSET(24) NUMBITS(2) [ + SpecifiedOnly = 0, + AllExceptLocal = 1, + LocalOnly = 2, + ], + CPUTargetList OFFSET(16) NUMBITS(8) [], + INTID OFFSET(0) NUMBITS(4) [] + ], +} + +register_structs! { + #[allow(non_snake_case)] + pub(super) GicdSharedRegs { + (0x000 => CTLR: ReadWrite), + (0x004 => TYPER: ReadWrite), + (0x008 => _0), + (0x104 => ISENABLER: [ReadWrite; 31]), + (0x180 => _1), + (0x820 => ITARGETSR: [ReadWrite; 248]), + (0xC00 => _2), + (0xC08 => ICFGR: [ReadWrite; 62]), + (0xD00 => _3), + (0xF00 => SGIR: WriteOnly), + (0xF04 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + pub(super) GicdBankedRegs { + (0x000 => _0), + (0x100 => ISENABLER: ReadWrite), + (0x104 => _1), + (0x800 => ITARGETSR: [ReadOnly; 8]), + (0x820 => _2), + (0xC00 => ICFGR: [ReadWrite; 2]), + (0xC08 => @END), + } +} + +pub(super) struct Gicd { + shared_regs: Spinlock>, + banked_regs: DeviceMemoryIo, +} + +impl GicdSharedRegs { + #[inline(always)] + fn num_irqs(&self) -> usize { + ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 + } + + #[inline(always)] + fn itargets_slice(&self) -> &[ReadWrite] { + assert!(self.num_irqs() >= 36); + let itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1; + &self.ITARGETSR[0..itargetsr_max_index] + } +} + +impl Gicd { + pub const fn new( + shared_regs: DeviceMemoryIo, + banked_regs: DeviceMemoryIo, + ) -> Self { + let shared_regs = Spinlock::new(shared_regs); + Self { + shared_regs, + banked_regs, + } + } + + pub unsafe fn set_sgir(&self, target: IpiDeliveryTarget, interrupt_id: u64) { + assert_eq!(interrupt_id & !0xF, 0); + let value = match target { + IpiDeliveryTarget::AllExceptLocal => SGIR::TargetListFilter::AllExceptLocal, + IpiDeliveryTarget::Specified(_mask) => { + // TODO: need to handle self-ipi case, releasing the lock somehow + todo!(); + } + } + SGIR::INTID.val(interrupt_id as u32); + + self.shared_regs.lock().SGIR.write(value); + } + + fn local_gic_target_mask(&self) -> u32 { + self.banked_regs.ITARGETSR[0].read(ITARGETSR::Offset0) + } + + fn enable_irq_inner(&self, irq: usize) { + let reg = irq >> 5; + let bit = 1u32 << (irq & 0x1F); + + match reg { + // Private IRQs + 0 => { + let reg = &self.banked_regs.ISENABLER; + + reg.set(reg.get() | bit); + } + // Shared IRQs + _ => { + let regs = self.shared_regs.lock(); + let reg = ®s.ISENABLER[reg - 1]; + + reg.set(reg.get() | bit); + } + } + } + + pub fn enable_irq(&self, irq: IrqNumber) { + let irq = irq.get(); + + self.enable_irq_inner(irq); + } + + pub unsafe fn init(&self) { + let mask = self.local_gic_target_mask(); + let regs = self.shared_regs.lock(); + + debugln!("Enabling GICv2 GICD, max IRQ number: {}", regs.num_irqs()); + + regs.CTLR.modify(CTLR::Enable::SET); + + for reg in regs.itargets_slice().iter() { + // Redirect all IRQs to cpu0 (this CPU) + reg.write( + ITARGETSR::Offset0.val(mask) + + ITARGETSR::Offset1.val(mask) + + ITARGETSR::Offset2.val(mask) + + ITARGETSR::Offset3.val(mask), + ); + } + } +} diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs new file mode 100644 index 00000000..d6461f96 --- /dev/null +++ b/src/arch/aarch64/gic/mod.rs @@ -0,0 +1,187 @@ +//! ARM Generic Interrupt Controller v2 driver +use core::sync::atomic::Ordering; + +use aarch64_cpu::asm::barrier; +use abi::error::Error; +use spinning_top::Spinlock; + +use crate::{ + arch::CpuMessage, + device::{ + interrupt::{InterruptController, InterruptSource, IpiDeliveryTarget}, + Device, + }, + mem::device::{DeviceMemory, DeviceMemoryIo}, + util::OneTimeInit, +}; + +use self::{gicc::Gicc, gicd::Gicd}; + +use super::{cpu::Cpu, exception::ipi_handler, smp::CPU_COUNT}; + +const MAX_IRQ: usize = 300; +const IPI_VECTOR: u64 = 1; + +pub mod gicc; +pub mod gicd; + +/// Wrapper type for ARM interrupt vector +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct IrqNumber(usize); + +/// ARM Generic Interrupt Controller v2 +pub struct Gic { + gicc: OneTimeInit, + gicd: OneTimeInit, + gicd_base: usize, + gicc_base: usize, + irq_table: Spinlock<[Option<&'static (dyn InterruptSource + Sync)>; MAX_IRQ]>, +} + +impl IrqNumber { + /// Returns the underlying vector number + #[inline(always)] + pub const fn get(self) -> usize { + self.0 + } + + /// Wraps the interrupt vector value in the [IrqNumber] type. + /// + /// # Panics + /// + /// Will panic if `v` is not a valid interrupt number. + #[inline(always)] + pub const fn new(v: usize) -> Self { + assert!(v < MAX_IRQ); + Self(v) + } +} + +impl Device for Gic { + fn name(&self) -> &'static str { + "ARM Generic Interrupt Controller v2" + } + + unsafe fn init(&self) -> Result<(), Error> { + let gicd_mmio = DeviceMemory::map("GICv2 Distributor registers", self.gicd_base, 0x1000)?; + let gicd_mmio_shared = DeviceMemoryIo::new(gicd_mmio.clone()); + let gicd_mmio_banked = DeviceMemoryIo::new(gicd_mmio); + let gicc_mmio = DeviceMemoryIo::map("GICv2 CPU registers", self.gicc_base)?; + + let gicd = Gicd::new(gicd_mmio_shared, gicd_mmio_banked); + let gicc = Gicc::new(gicc_mmio); + + gicd.init(); + gicc.init(); + + self.gicd.init(gicd); + self.gicc.init(gicc); + + Ok(()) + } +} + +impl InterruptController for Gic { + type IrqNumber = IrqNumber; + + fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { + self.gicd.get().enable_irq(irq); + Ok(()) + } + + fn handle_pending_irqs<'irq>(&'irq self, ic: &crate::device::interrupt::IrqContext<'irq>) { + let gicc = self.gicc.get(); + let irq_number = gicc.pending_irq_number(ic); + if irq_number >= MAX_IRQ { + return; + } + + gicc.clear_irq(irq_number, ic); + + if irq_number as u64 == IPI_VECTOR { + // IPI received + let msg = Cpu::local().get_ipi(); + ipi_handler(msg); + return; + } + + { + let table = self.irq_table.lock(); + match table[irq_number] { + None => panic!("No IRQ handler registered for irq{}", irq_number), + Some(handler) => { + drop(table); + handler.handle_irq().expect("IRQ handler failed"); + } + } + } + } + + fn register_handler( + &self, + irq: Self::IrqNumber, + handler: &'static (dyn InterruptSource + Sync), + ) -> Result<(), Error> { + let mut table = self.irq_table.lock(); + let irq = irq.get(); + if table[irq].is_some() { + return Err(Error::AlreadyExists); + } + + debugln!("Bound irq{} to {:?}", irq, Device::name(handler)); + table[irq] = Some(handler); + + Ok(()) + } + + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { + // TODO message queue insertion should be moved + match target { + IpiDeliveryTarget::AllExceptLocal => { + let local = Cpu::local_id(); + for i in 0..CPU_COUNT.load(Ordering::Acquire) { + if i != local as usize { + Cpu::push_ipi_queue(i as u32, msg); + } + } + } + IpiDeliveryTarget::Specified(_) => todo!(), + } + + // Issue a memory barrier + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); + + self.gicd.get().set_sgir(target, IPI_VECTOR); + + Ok(()) + } +} + +impl Gic { + /// Constructs an instance of GICv2. + /// + /// # Safety + /// + /// The caller must ensure the addresses actually point to the GIC components. + pub const unsafe fn new(gicd_base: usize, gicc_base: usize) -> Self { + Self { + gicc: OneTimeInit::new(), + gicd: OneTimeInit::new(), + gicd_base, + gicc_base, + irq_table: Spinlock::new([None; MAX_IRQ]), + } + } + + /// Initializes GICv2 for an application processor. + /// + /// # Safety + /// + /// Must not be called more than once per each AP. Must not be called from BSP. + pub unsafe fn init_smp_ap(&self) -> Result<(), Error> { + self.gicc.get().init(); + Ok(()) + } +} diff --git a/src/arch/aarch64/intrinsics.rs b/src/arch/aarch64/intrinsics.rs new file mode 100644 index 00000000..66d1b8cd --- /dev/null +++ b/src/arch/aarch64/intrinsics.rs @@ -0,0 +1,13 @@ +//! Intrinsic helper functions for AArch64 platforms + +/// Returns an absolute address to the given symbol +#[macro_export] +macro_rules! absolute_address { + ($sym:expr) => {{ + let mut _x: usize; + unsafe { + core::arch::asm!("ldr {0}, ={1}", out(reg) _x, sym $sym); + } + _x + }}; +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs new file mode 100644 index 00000000..9bfe55b7 --- /dev/null +++ b/src/arch/aarch64/mod.rs @@ -0,0 +1,204 @@ +//! AArch64 architecture and platforms implementation + +use core::sync::atomic::Ordering; + +use aarch64_cpu::registers::{DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1}; +use abi::error::Error; +use plat_qemu::PLATFORM; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +use crate::{ + absolute_address, + arch::{ + aarch64::{boot::CPU_INIT_FENCE, cpu::Cpu, devtree::FdtMemoryRegionIter, smp::CPU_COUNT}, + Architecture, + }, + debug, + device::platform::Platform, + fs::devfs, + mem::{ + heap, + phys::{self, reserved::reserve_region, PageUsage, PhysicalMemoryRegion}, + ConvertAddress, + }, + task, + util::OneTimeInit, +}; + +use self::{ + devtree::DeviceTree, + table::{init_fixed_tables, KERNEL_TABLES}, +}; + +pub mod intrinsics; + +pub mod plat_qemu; + +pub mod boot; +pub mod context; +pub mod cpu; +pub mod devtree; +pub mod exception; +pub mod gic; +pub mod smp; +pub mod table; +pub mod timer; + +pub(self) const BOOT_STACK_SIZE: usize = 65536; + +#[derive(Clone, Copy)] +#[repr(C, align(0x20))] +pub(self) struct KernelStack { + data: [u8; BOOT_STACK_SIZE], +} + +/// AArch64 platform interface +pub struct AArch64 { + dt: OneTimeInit>, +} + +/// Global platform handle +pub static ARCHITECTURE: AArch64 = AArch64 { + dt: OneTimeInit::new(), +}; + +impl Architecture for AArch64 { + const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; + + unsafe fn init_mmu(&self, bsp: bool) { + if bsp { + init_fixed_tables(); + } + + let tables_phys = absolute_address!(KERNEL_TABLES).physicalize() as u64; + + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { + todo!(); + } + + TCR_EL1.modify( + // General + TCR_EL1::IPS::Bits_48 + + // TTBR0 + TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner + + // TTBR1 + TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, + ); + + TTBR0_EL1.set_baddr(tables_phys); + TTBR1_EL1.set_baddr(tables_phys); + + SCTLR_EL1.modify(SCTLR_EL1::M::Enable); + } + + fn map_device_pages(&self, phys: usize, count: usize) -> Result { + unsafe { KERNEL_TABLES.map_device_pages(phys, count) } + } + + fn wait_for_interrupt() { + aarch64_cpu::asm::wfi(); + } + + unsafe fn set_interrupt_mask(mask: bool) { + if mask { + DAIF.modify(DAIF::I::SET); + } else { + DAIF.modify(DAIF::I::CLEAR); + } + } + + fn interrupt_mask() -> bool { + DAIF.read(DAIF::I) != 0 + } +} + +impl AArch64 { + /// Initializes the architecture's device tree + /// + /// # Safety + /// + /// Only makes sense to call during the early initialization, once. + pub unsafe fn init_device_tree(&self, dtb_phys: usize) { + let dt = DeviceTree::from_addr(dtb_phys.virtualize()); + self.dt.init(dt); + } + + /// Returns the device tree + /// + /// # Panics + /// + /// Will panic if the device tree has not yet been initialized + pub fn device_tree(&self) -> &DeviceTree { + self.dt.get() + } + + unsafe fn init_physical_memory(&self, dtb_phys: usize) -> Result<(), Error> { + let dt = self.device_tree(); + + reserve_region( + "dtb", + PhysicalMemoryRegion { + base: dtb_phys, + size: dt.size(), + }, + ); + + let regions = FdtMemoryRegionIter::new(dt); + phys::init_from_iter(regions) + } +} + +/// AArch64 kernel main entry point +pub fn kernel_main(dtb_phys: usize) -> ! { + // NOTE it is critical that the code does not panic until the debug is set up, otherwise no + // message will be displayed + + // Unmap TTBR0 + TTBR0_EL1.set(0); + + unsafe { + AArch64::set_interrupt_mask(true); + + ARCHITECTURE.init_device_tree(dtb_phys); + PLATFORM.init_primary_serial(); + } + // Setup debugging functions + debug::init(); + + exception::init_exceptions(); + + debugln!("Initializing {} platform", PLATFORM.name()); + unsafe { + ARCHITECTURE + .init_physical_memory(dtb_phys) + .expect("Failed to initialize the physical memory manager"); + + // Setup heap + let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) + .expect("Could not allocate a block for heap"); + heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + + Cpu::init_local(); + + devfs::init(); + PLATFORM.init(true).unwrap(); + + let dt = ARCHITECTURE.dt.get(); + if let Err(e) = smp::start_ap_cores(dt) { + errorln!( + "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", + e + ); + } + + Cpu::init_ipi_queues(); + + CPU_INIT_FENCE.signal(); + CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); + + task::init().expect("Failed to initialize the scheduler"); + + // Initialize and enter the scheduler + task::enter(); + } +} diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs new file mode 100644 index 00000000..a6420b97 --- /dev/null +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -0,0 +1,83 @@ +//! Qemu's "virt" platform implementation for AArch64 +use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; +use abi::error::Error; +use tock_registers::interfaces::Writeable; + +use crate::{ + device::{ + interrupt::{InterruptController, InterruptSource}, + platform::Platform, + serial::{pl011::Pl011, SerialDevice}, + timer::TimestampSource, + Device, + }, + fs::devfs::{self, CharDeviceType}, +}; + +use super::{ + gic::{Gic, IrqNumber}, + timer::ArmTimer, +}; + +/// AArch64 "virt" platform implementation +pub struct QemuPlatform { + gic: Gic, + pl011: Pl011, + local_timer: ArmTimer, +} + +impl Platform for QemuPlatform { + type IrqNumber = IrqNumber; + + const KERNEL_PHYS_BASE: usize = 0x40080000; + + unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { + if is_bsp { + self.gic.init()?; + + self.pl011.init_irq()?; + devfs::add_char_device(&self.pl011, CharDeviceType::TtySerial)?; + + self.local_timer.init()?; + self.local_timer.init_irq()?; + } else { + self.gic.init_smp_ap()?; + + // TODO somehow merge this with the rest of the code + CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); + CNTP_TVAL_EL0.set(10000000); + self.gic.enable_irq(IrqNumber::new(30))?; + } + + Ok(()) + } + + unsafe fn init_primary_serial(&self) { + self.pl011.init().ok(); + } + + fn name(&self) -> &'static str { + "qemu" + } + + fn primary_serial(&self) -> Option<&dyn SerialDevice> { + Some(&self.pl011) + } + + fn interrupt_controller(&self) -> &dyn InterruptController { + &self.gic + } + + fn timestamp_source(&self) -> &dyn TimestampSource { + &self.local_timer + } +} + +/// AArch64 "virt" platform +pub static PLATFORM: QemuPlatform = unsafe { + QemuPlatform { + pl011: Pl011::new(0x09000000, IrqNumber::new(33)), + gic: Gic::new(0x08000000, 0x08010000), + local_timer: ArmTimer::new(IrqNumber::new(30)), + } +}; diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs new file mode 100644 index 00000000..4e63d68e --- /dev/null +++ b/src/arch/aarch64/smp.rs @@ -0,0 +1,131 @@ +//! Simultaneous multiprocessing support for aarch64 +use core::{ + arch::asm, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use abi::error::Error; +use fdt_rs::prelude::PropReader; + +use crate::{ + absolute_address, + arch::aarch64::boot::__aarch64_ap_lower_entry, + mem::{ + phys::{self, PageUsage}, + ConvertAddress, KERNEL_VIRT_OFFSET, + }, +}; + +use super::devtree::{self, DeviceTree}; + +/// ARM Power State Coordination Interface +pub struct Psci {} + +/// Number of online CPUs, initially set to 1 (BSP processor is up) +pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); + +impl Psci { + /// Function ID for CPU startup request + const CPU_ON: u32 = 0xC4000003; + + /// Constructs an interface instance for PSCI + pub const fn new() -> Self { + Self {} + } + + #[inline] + unsafe fn call(&self, mut x0: u64, x1: u64, x2: u64, x3: u64) -> u64 { + asm!("hvc #0", inout("x0") x0, in("x1") x1, in("x2") x2, in("x3") x3); + x0 + } + + /// Enables a single processor through a hvc/svc call. + /// + /// # Safety + /// + /// Calling this outside of initialization sequence or more than once may lead to unexpected + /// behavior. + pub unsafe fn cpu_on(&self, target_cpu: usize, entry_point_address: usize, context_id: usize) { + self.call( + Self::CPU_ON as _, + target_cpu as _, + entry_point_address as _, + context_id as _, + ); + } +} + +/// Starts application processors using the method specified in the device tree. +/// +/// TODO: currently does not handle systems where APs are already started before entry. +/// +/// # Safety +/// +/// The caller must ensure the physical memory manager was initialized, virtual memory tables are +/// set up and the function has not been called before. +pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> { + let cpus = dt.node_by_path("/cpus").unwrap(); + let psci = Psci::new(); + + for cpu in cpus.children() { + let Some(compatible) = devtree::find_prop(&cpu, "compatible") else { + continue; + }; + let Ok(compatible) = compatible.str() else { + continue; + }; + if !compatible.starts_with("arm,cortex-a") { + continue; + } + + let reg = devtree::find_prop(&cpu, "reg").unwrap().u32(0).unwrap(); + if reg == 0 { + continue; + } + + debugln!( + "Will start {}, compatible={:?}, reg={}", + cpu.name().unwrap(), + compatible, + reg + ); + + const AP_STACK_PAGES: usize = 4; + let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used)?; + debugln!( + "{} stack: {:#x}..{:#x}", + cpu.name().unwrap(), + stack_pages, + stack_pages + AP_STACK_PAGES * 0x1000 + ); + // Wait for the CPU to come up + let old_count = CPU_COUNT.load(Ordering::Acquire); + + psci.cpu_on( + reg as usize, + absolute_address!(__aarch64_ap_entry).physicalize(), + stack_pages + AP_STACK_PAGES * 0x1000, + ); + + while CPU_COUNT.load(Ordering::Acquire) == old_count { + aarch64_cpu::asm::wfe(); + } + + debugln!("{} is up", cpu.name().unwrap()); + } + + Ok(()) +} + +#[naked] +unsafe extern "C" fn __aarch64_ap_entry() -> ! { + asm!( + r#" + mov sp, x0 + bl {entry} - {kernel_virt_offset} + "#, + entry = sym __aarch64_ap_lower_entry, + kernel_virt_offset = const KERNEL_VIRT_OFFSET, + options(noreturn) + ); +} diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs new file mode 100644 index 00000000..0a6fd586 --- /dev/null +++ b/src/arch/aarch64/table.rs @@ -0,0 +1,451 @@ +//! AArch64 virtual memory management facilities +use core::{ + marker::PhantomData, + ops::{Index, IndexMut}, + sync::atomic::{AtomicU8, Ordering}, +}; + +use abi::error::Error; +use bitflags::bitflags; + +use crate::mem::{ + phys::{self, PageUsage}, + table::{EntryLevel, NextPageTable, VirtualMemoryManager}, + ConvertAddress, KERNEL_VIRT_OFFSET, +}; + +/// TODO +#[derive(Clone)] +#[repr(C)] +pub struct AddressSpace { + l1: *mut PageTable, + asid: u8, +} + +/// Page table representing a single level of address translation +#[derive(Clone)] +#[repr(C, align(0x1000))] +pub struct PageTable { + data: [PageEntry; 512], +} + +/// Translation level 1: Entry is 1GiB page/table +#[derive(Clone, Copy)] +pub struct L1; +/// Translation level 2: Entry is 2MiB page/table +#[derive(Clone, Copy)] +pub struct L2; +/// Translation level 3: Entry is 4KiB page +#[derive(Clone, Copy)] +pub struct L3; + +/// Tag trait to mark that the page table level may point to a next-level table +pub trait NonTerminalEntryLevel: EntryLevel { + /// Tag type of the level this entry level may point to + type NextLevel: EntryLevel; +} + +impl NonTerminalEntryLevel for L1 { + type NextLevel = L2; +} +impl NonTerminalEntryLevel for L2 { + type NextLevel = L3; +} + +bitflags! { + /// TODO split attrs for different translation levels + /// + /// Describes how each page is mapped: access, presence, type of the mapping. + #[derive(Clone, Copy)] + pub struct PageAttributes: u64 { + /// When set, the mapping is considered valid and assumed to point to a page/table + const PRESENT = 1 << 0; + + /// For L1/L2 mappings, indicates that the mapping points to the next-level translation + /// table + const TABLE = 1 << 1; + /// (Must be set) For L3 mappings, indicates that the mapping points to a page + const PAGE = 1 << 1; + /// For L1/L2 mappings, indicates that the mapping points to a page of given level's size + const BLOCK = 0 << 1; + + /// (Must be set) For page/block mappings, indicates to the hardware that the page is + /// accessed + const ACCESS = 1 << 10; + + /// For page/block mappings, allows both user and kernel code to read/write to the page + const AP_BOTH_READWRITE = 1 << 6; + /// For page/block mappings, only allows read access for EL0/EL1 + const AP_BOTH_READONLY = 3 << 6; + } +} + +impl const EntryLevel for L1 { + fn index(addr: usize) -> usize { + (addr >> 30) & 0x1FF + } + + fn page_offset(addr: usize) -> usize { + addr & 0x3FFFFFFF + } +} +impl const EntryLevel for L2 { + fn index(addr: usize) -> usize { + (addr >> 21) & 0x1FF + } + + fn page_offset(addr: usize) -> usize { + addr & 0x1FFFFF + } +} +impl const EntryLevel for L3 { + fn index(addr: usize) -> usize { + (addr >> 12) & 0x1FF + } + + fn page_offset(addr: usize) -> usize { + addr & 0xFFF + } +} + +/// Represents a single entry in a translation table +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct PageEntry(u64, PhantomData); + +/// Fixed-layout kernel-space address mapping tables +pub struct FixedTables { + l1: PageTable, + device_l2: PageTable, + device_l3: PageTable, + + device_l3i: usize, +} + +impl PageEntry { + /// Creates a 4KiB page mapping + pub fn page(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) + | (PageAttributes::PAGE | PageAttributes::PRESENT | PageAttributes::ACCESS | attrs) + .bits(), + PhantomData, + ) + } + + /// Returns the physical address of the page this entry refers to, returning None if it does + /// not + pub fn as_page(self) -> Option { + let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); + + if self.0 & mask == mask { + Some((self.0 & !0xFFF) as usize) + } else { + None + } + } +} + +impl PageEntry { + /// Creates a 2MiB page mapping + pub fn block(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | attrs) + .bits(), + PhantomData, + ) + } + + /// Creates a mapping pointing to the next-level translation table + pub fn table(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), + PhantomData, + ) + } + + /// Returns the physical address of the table this entry refers to, returning None if it + /// does not + pub fn as_table(self) -> Option { + if self.0 & (PageAttributes::TABLE | PageAttributes::PRESENT).bits() + == (PageAttributes::TABLE | PageAttributes::PRESENT).bits() + { + Some((self.0 & !0xFFF) as usize) + } else { + None + } + } +} + +impl PageEntry { + /// Represents an absent/invalid mapping in the table + pub const INVALID: Self = Self(0, PhantomData); + + /// Converts a raw mapping value into this wrapper type + /// + /// # Safety + /// + /// The caller is responsible for making sure that `raw` is a valid mapping value for the + /// current translation level. + pub unsafe fn from_raw(raw: u64) -> Self { + Self(raw, PhantomData) + } + + /// Returns `true` if the entry refers to some table/block/page + pub fn is_present(&self) -> bool { + self.0 & PageAttributes::PRESENT.bits() != 0 + } +} + +impl NextPageTable for PageTable { + type NextLevel = PageTable; + + fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> { + let entry = self[index]; + + entry + .as_table() + .map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) }) + } + + fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) }) + } else { + let table = PageTable::new_zeroed()?; + self[index] = PageEntry::::table(table.physical_address(), PageAttributes::empty()); + Ok(table) + } + } +} + +impl PageTable { + /// Constructs a page table with all entries marked as invalid + pub const fn zeroed() -> Self { + Self { + data: [PageEntry::INVALID; 512], + } + } + + /// Allocates a new page table, filling it with non-preset entries + pub fn new_zeroed() -> Result<&'static mut Self, Error> { + let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() }; + let table = unsafe { &mut *(page as *mut Self) }; + for i in 0..512 { + table[i] = PageEntry::INVALID; + } + Ok(table) + } + + /// Returns a physical address pointing to this page table + pub fn physical_address(&self) -> usize { + // &self may already by a physical address + let addr = self.data.as_ptr() as usize; + if addr < KERNEL_VIRT_OFFSET { + addr + } else { + unsafe { addr.physicalize() } + } + } +} + +impl Index for PageTable { + type Output = PageEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.data[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.data[index] + } +} + +impl FixedTables { + /// Constructs an empty table group + pub const fn zeroed() -> Self { + Self { + l1: PageTable::zeroed(), + device_l2: PageTable::zeroed(), + device_l3: PageTable::zeroed(), + + device_l3i: 0, + } + } + + /// Maps a physical memory region as device memory and returns its allocated base address + pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result { + if count > 512 * 512 { + panic!("Unsupported device memory mapping size"); + } else if count > 512 { + // 2MiB mappings + todo!(); + } else { + // 4KiB mappings + if self.device_l3i + count > 512 { + return Err(Error::OutOfMemory); + } + + let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12); + for i in 0..count { + self.device_l3[self.device_l3i + i] = + PageEntry::page(phys + i * 0x1000, PageAttributes::empty()); + } + self.device_l3i += count; + + tlb_flush_vaae1(virt); + + Ok(virt) + } + } +} + +impl VirtualMemoryManager for AddressSpace { + fn allocate( + &self, + hint: Option, + len: usize, + attrs: PageAttributes, + ) -> Result { + if hint.is_some() { + todo!(); + } + + const TRY_ALLOC_START: usize = 0x100000000; + const TRY_ALLOC_END: usize = 0xF00000000; + + 'l0: for base in (TRY_ALLOC_START..TRY_ALLOC_END - len * 0x1000).step_by(0x1000) { + for i in 0..len { + if self.translate(base + i * 0x1000).is_some() { + continue 'l0; + } + } + + for i in 0..len { + let page = phys::alloc_page(PageUsage::Used)?; + self.map_page(base + i * 0x1000, page, attrs)?; + } + + return Ok(base); + } + + Err(Error::OutOfMemory) + } + + fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> { + for page in (addr..addr + len).step_by(0x1000) { + let Some(_phys) = self.translate(page) else { + todo!(); + }; + + self.write_entry(page, PageEntry::INVALID, true)?; + } + + Ok(()) + } +} + +impl AddressSpace { + /// Allocates an empty address space with all entries marked as non-present + pub fn new_empty() -> Result { + static LAST_ASID: AtomicU8 = AtomicU8::new(1); + + let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); + + let l1 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable }; + + for i in 0..512 { + unsafe { + (*l1)[i] = PageEntry::INVALID; + } + } + + Ok(Self { l1, asid }) + } + + unsafe fn as_mut(&self) -> &'static mut PageTable { + self.l1.as_mut().unwrap() + } + + // TODO return page size and attributes + /// Returns the physical address to which the `virt` address is mapped + pub fn translate(&self, virt: usize) -> Option { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l2 = unsafe { self.as_mut().get_mut(l1i) }?; + let l3 = l2.get_mut(l2i)?; + + l3[l3i].as_page() + } + + // Write a single 4KiB entry + fn write_entry(&self, virt: usize, entry: PageEntry, overwrite: bool) -> Result<(), Error> { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l2 = unsafe { self.as_mut().get_mut_or_alloc(l1i) }?; + let l3 = l2.get_mut_or_alloc(l2i)?; + + if l3[l3i].is_present() && !overwrite { + todo!() + } + l3[l3i] = entry; + + Ok(()) + } + + /// Inserts a single 4KiB virt -> phys mapping into the address apce + pub fn map_page(&self, virt: usize, phys: usize, attrs: PageAttributes) -> Result<(), Error> { + self.write_entry(virt, PageEntry::page(phys, attrs), true) + } + + /// Returns the physical address of the address space (to be used in a TTBRn_ELx) + pub fn physical_address(&self) -> usize { + unsafe { (self.l1 as usize).physicalize() | ((self.asid as usize) << 48) } + } +} + +/// Flushes the virtual address from TLB +pub fn tlb_flush_vaae1(page: usize) { + assert_eq!(page & 0xFFF, 0); + unsafe { + core::arch::asm!("tlbi vaae1, {addr}", addr = in(reg) page); + } +} + +/// Initializes mappings for the kernel and device memory tables. +/// +/// # Safety +/// +/// Only allowed to be called once during lower-half part of the initialization process. +pub unsafe fn init_fixed_tables() { + // Map first 256GiB + for i in 0..256 { + KERNEL_TABLES.l1[i] = PageEntry::::block(i << 30, PageAttributes::empty()); + } + + KERNEL_TABLES.l1[256] = PageEntry::::table( + KERNEL_TABLES.device_l2.physical_address(), + PageAttributes::empty(), + ); + KERNEL_TABLES.device_l2[0] = PageEntry::::table( + KERNEL_TABLES.device_l3.physical_address(), + PageAttributes::empty(), + ); +} + +/// Offset applied to device virtual memory mappings +pub const DEVICE_VIRT_OFFSET: usize = KERNEL_VIRT_OFFSET + (256 << 30); +/// Global kernel address space translation tables +pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed(); diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs new file mode 100644 index 00000000..7f8e37a1 --- /dev/null +++ b/src/arch/aarch64/timer.rs @@ -0,0 +1,78 @@ +//! AArch64 Generic Timer + +use core::time::Duration; + +use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0}; +use abi::error::Error; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +use crate::{ + arch::PLATFORM, + device::{interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device}, + proc::wait, +}; + +use super::{cpu::Cpu, gic::IrqNumber}; + +/// ARM Generic Timer driver +pub struct ArmTimer { + irq: IrqNumber, +} + +/// ARM timer tick interval (in some time units?) +pub const TICK_INTERVAL: u64 = 1000000; + +impl Device for ArmTimer { + fn name(&self) -> &'static str { + "ARM Generic Timer" + } + + unsafe fn init(&self) -> Result<(), Error> { + CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); + Ok(()) + } +} + +impl TimestampSource for ArmTimer { + fn timestamp(&self) -> Result { + let count = CNTPCT_EL0.get() * 1_000_000; + let freq = CNTFRQ_EL0.get(); + + Ok(Duration::from_nanos((count / freq) * 1_000)) + } +} + +impl InterruptSource for ArmTimer { + fn handle_irq(&self) -> Result<(), Error> { + CNTP_TVAL_EL0.set(TICK_INTERVAL); + wait::tick(); + + unsafe { + Cpu::local().queue().yield_cpu(); + } + + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = PLATFORM.interrupt_controller(); + + intc.register_handler(self.irq, self)?; + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::IMASK::CLEAR); + CNTP_TVAL_EL0.set(TICK_INTERVAL); + intc.enable_irq(self.irq)?; + + Ok(()) + } +} + +impl ArmTimer { + /// Constructs an instance of ARM generic timer. + /// + /// # Safety + /// + /// The caller must ensure the function has not been called before. + pub const unsafe fn new(irq: IrqNumber) -> Self { + Self { irq } + } +} diff --git a/src/arch/aarch64/vectors.S b/src/arch/aarch64/vectors.S new file mode 100644 index 00000000..d16c12fa --- /dev/null +++ b/src/arch/aarch64/vectors.S @@ -0,0 +1,128 @@ +// vi:ft=a64asm: + +.macro EXC_VECTOR el, ht, bits, kind +.p2align 7 + b __aa\bits\()_el\el\ht\()_\kind +.endm + +.macro EXC_HANDLER el, ht, bits, kind +__aa\bits\()_el\el\ht\()_\kind: +.if \bits == 32 + // TODO + b . +.endif + + EXC_SAVE_STATE + mov x0, sp + mov lr, xzr + bl __aa64_exc_\kind\()_handler + EXC_RESTORE_STATE + eret +.endm + +// 32 gp regs + 3 special regs +.set PT_REGS_SIZE, (16 * 16 + 16 * 2) + +.macro EXC_SAVE_STATE + sub sp, sp, #PT_REGS_SIZE + + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + stp x30, x31, [sp, #16 * 15] + + mrs x0, spsr_el1 + mrs x1, elr_el1 + mrs x2, sp_el0 + + // TODO + stp x0, x1, [sp, #16 * 16] + stp x2, xzr, [sp, #16 * 17] +.endm + +.macro EXC_RESTORE_STATE + ldp x0, x1, [sp, #16 * 16] + ldp x2, x3, [sp, #16 * 17] + + msr spsr_el1, x0 + msr elr_el1, x1 + msr sp_el0, x2 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + ldp x30, x31, [sp, #16 * 15] + + add sp, sp, #PT_REGS_SIZE +.endm + +.section .text +.p2align 12 +__aarch64_el1_vectors: + EXC_VECTOR 1, t, 64, sync + EXC_VECTOR 1, t, 64, irq + EXC_VECTOR 1, t, 64, fiq + EXC_VECTOR 1, t, 64, serror + + EXC_VECTOR 1, h, 64, sync + EXC_VECTOR 1, h, 64, irq + EXC_VECTOR 1, h, 64, fiq + EXC_VECTOR 1, h, 64, serror + + EXC_VECTOR 0, t, 64, sync + EXC_VECTOR 0, t, 64, irq + EXC_VECTOR 0, t, 64, fiq + EXC_VECTOR 0, t, 64, serror + + EXC_VECTOR 0, t, 32, sync + EXC_VECTOR 0, t, 32, irq + EXC_VECTOR 0, t, 32, fiq + EXC_VECTOR 0, t, 32, serror + + +.p2align 7 +EXC_HANDLER 1, t, 64, sync +EXC_HANDLER 1, t, 64, irq +EXC_HANDLER 1, t, 64, fiq +EXC_HANDLER 1, t, 64, serror + +EXC_HANDLER 1, h, 64, sync +EXC_HANDLER 1, h, 64, irq +EXC_HANDLER 1, h, 64, fiq +EXC_HANDLER 1, h, 64, serror + +EXC_HANDLER 0, t, 64, sync +EXC_HANDLER 0, t, 64, irq +EXC_HANDLER 0, t, 64, fiq +EXC_HANDLER 0, t, 64, serror + +EXC_HANDLER 0, t, 32, sync +EXC_HANDLER 0, t, 32, irq +EXC_HANDLER 0, t, 32, fiq +EXC_HANDLER 0, t, 32, serror diff --git a/src/arch/mod.rs b/src/arch/mod.rs new file mode 100644 index 00000000..06a59214 --- /dev/null +++ b/src/arch/mod.rs @@ -0,0 +1,48 @@ +//! Provides architecture/platform-specific implementation details +pub mod aarch64; + +pub use aarch64::plat_qemu::{QemuPlatform as PlatformImpl, PLATFORM}; +pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; +use abi::error::Error; + +/// Describes messages sent from some CPU to others +#[derive(Clone, Copy, PartialEq, Debug)] +#[repr(u64)] +pub enum CpuMessage { + /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow + Panic, +} + +/// Interface for an architecture-specific facilities +pub trait Architecture { + /// Address, to which "zero" address is mapped in the virtual address space + const KERNEL_VIRT_OFFSET: usize; + + /// Initializes the memory management unit and sets up virtual memory management. + /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP + /// system. + /// + /// # Safety + /// + /// Unsafe to call if the MMU has already been initialized. + unsafe fn init_mmu(&self, bsp: bool); + + /// Allocates a virtual mapping for the specified physical memory region + fn map_device_pages(&self, phys: usize, count: usize) -> Result; + + // Architecture intrinsics + + /// Suspends CPU until an interrupt is received + fn wait_for_interrupt(); + + /// Sets the local CPU's interrupt mask. + /// + /// # Safety + /// + /// Enabling interrupts may lead to unexpected behavior unless the context explicitly expects + /// them. + unsafe fn set_interrupt_mask(mask: bool); + + /// Returns the local CPU's interrupt mask + fn interrupt_mask() -> bool; +} diff --git a/src/debug.rs b/src/debug.rs new file mode 100644 index 00000000..456c315a --- /dev/null +++ b/src/debug.rs @@ -0,0 +1,124 @@ +//! Utilities for debug information logging +use core::fmt::{self, Arguments}; + +use crate::{ + arch::PLATFORM, + device::{platform::Platform, serial::SerialDevice}, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +/// Defines the severity of the message +#[derive(Clone, Copy)] +pub enum LogLevel { + /// Debugging and verbose information + Debug, + /// General information about transitions in the system state + Info, + /// Non-critical abnormalities or notices + Warning, + /// Failures of non-essential components + Error, + /// Irrecoverable errors which result in kernel panic + Fatal, +} + +struct DebugPrinter { + sink: &'static dyn SerialDevice, +} + +macro_rules! log_print_raw { + ($level:expr, $($args:tt)+) => { + $crate::debug::debug_internal(format_args!($($args)+), $level) + }; +} + +macro_rules! log_print { + ($level:expr, $($args:tt)+) => { + log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::arch::aarch64::cpu::Cpu::local_id(), file!(), line!(), format_args!($($args)+)) + }; +} + +macro_rules! debug_tpl { + ($d:tt $name:ident, $nameln:ident, $level:ident) => { + #[allow(unused_macros)] + /// Prints the message to the log + macro_rules! $name { + ($d($d args:tt)+) => (log_print!($crate::debug::LogLevel::$level, $d($d args)+)); + } + + /// Prints the message to the log, terminated by a newline character + #[allow(unused_macros)] + macro_rules! $nameln { + () => { + $name!("\n") + }; + ($d($d args:tt)+) => ($name!("{}\n", format_args!($d($d args)+))); + } + }; +} + +debug_tpl!($ debug, debugln, Debug); +debug_tpl!($ info, infoln, Info); +debug_tpl!($ warn, warnln, Warning); +debug_tpl!($ error, errorln, Error); +debug_tpl!($ fatal, fatalln, Fatal); + +#[no_mangle] +static DEBUG_PRINTER: OneTimeInit> = OneTimeInit::new(); + +impl LogLevel { + fn log_prefix(self) -> &'static str { + match self { + LogLevel::Debug => "", + LogLevel::Info => "\x1b[36m\x1b[1m", + LogLevel::Warning => "\x1b[33m\x1b[1m", + LogLevel::Error => "\x1b[31m\x1b[1m", + LogLevel::Fatal => "\x1b[38;2;255;0;0m\x1b[1m", + } + } + + fn log_suffix(self) -> &'static str { + match self { + LogLevel::Debug => "", + LogLevel::Info => "\x1b[0m", + LogLevel::Warning => "\x1b[0m", + LogLevel::Error => "\x1b[0m", + LogLevel::Fatal => "\x1b[0m", + } + } +} + +impl fmt::Write for DebugPrinter { + fn write_str(&mut self, s: &str) -> fmt::Result { + for c in s.bytes() { + self.sink.send(c).ok(); + } + + Ok(()) + } +} + +/// Initializes the debug logging faclities. +/// +/// # Panics +/// +/// Will panic if called more than once. +pub fn init() { + DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { + sink: PLATFORM.primary_serial().unwrap(), + })); +} + +#[doc = "hide"] +pub fn debug_internal(args: Arguments, level: LogLevel) { + use fmt::Write; + + if DEBUG_PRINTER.is_initialized() { + let mut printer = DEBUG_PRINTER.get().lock(); + + printer.write_str(level.log_prefix()).ok(); + printer.write_fmt(args).ok(); + printer.write_str(level.log_suffix()).ok(); + } +} diff --git a/src/device/interrupt.rs b/src/device/interrupt.rs new file mode 100644 index 00000000..b2a531f7 --- /dev/null +++ b/src/device/interrupt.rs @@ -0,0 +1,78 @@ +//! Interrupt-related interfaces +use core::marker::PhantomData; + +use abi::error::Error; + +use crate::arch::CpuMessage; + +use super::Device; + +/// Specifies the target(s) of interprocessor interrupt delivery +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum IpiDeliveryTarget { + /// IPI will be delivered to every CPU except the local one + AllExceptLocal, + /// IPI will only be sent to CPUs specified in the mask + Specified(u64), +} + +/// Interface for a device capable of emitting interrupts +pub trait InterruptSource: Device { + /// Initializes and enables IRQs for the device. + /// + /// # Safety + /// + /// The caller must ensure the function hasn't been called before. + unsafe fn init_irq(&'static self) -> Result<(), Error>; + + /// Handles the interrupt raised by the device + fn handle_irq(&self) -> Result<(), Error>; +} + +/// Interface for a device responsible for routing and handling IRQs +pub trait InterruptController: Device { + /// Interrupt number wrapper type + type IrqNumber; + + /// Binds an interrupt number to its handler implementation + fn register_handler( + &self, + irq: Self::IrqNumber, + handler: &'static (dyn InterruptSource + Sync), + ) -> Result<(), Error>; + + /// Enables given interrupt number/vector + fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error>; + + /// Handles all pending interrupts on this controller + fn handle_pending_irqs<'irq>(&'irq self, ic: &IrqContext<'irq>); + + /// Sends a message to the requested set of CPUs through an interprocessor interrupt. + /// + /// # Note + /// + /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations + /// may impose narrower restrictions. + /// + /// # Safety + /// + /// As the call may alter the flow of execution on CPUs, this function is unsafe. + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; +} + +/// Token type to indicate that the code is being run from an interrupt handler +pub struct IrqContext<'irq> { + _0: PhantomData<&'irq ()>, +} + +impl<'irq> IrqContext<'irq> { + /// Constructs an IRQ context token + /// + /// # Safety + /// + /// Only allowed to be constructed in top-level IRQ handlers + #[inline(always)] + pub const unsafe fn new() -> Self { + Self { _0: PhantomData } + } +} diff --git a/src/device/mod.rs b/src/device/mod.rs new file mode 100644 index 00000000..456339b3 --- /dev/null +++ b/src/device/mod.rs @@ -0,0 +1,21 @@ +//! Device management and interfaces +use abi::error::Error; + +pub mod interrupt; +pub mod platform; +pub mod serial; +pub mod timer; +pub mod tty; + +/// General device interface +pub trait Device { + /// Initializes the device to a state where it can be used. + /// + /// # Safety + /// + /// Unsafe to call if the device has already been initialized. + unsafe fn init(&self) -> Result<(), Error>; + + /// Returns a display name for the device + fn name(&self) -> &'static str; +} diff --git a/src/device/platform.rs b/src/device/platform.rs new file mode 100644 index 00000000..674ef590 --- /dev/null +++ b/src/device/platform.rs @@ -0,0 +1,51 @@ +//! Hardware platform interface + +use abi::error::Error; + +use super::{interrupt::InterruptController, serial::SerialDevice, timer::TimestampSource}; + +/// Platform interface for interacting with a general hardware set +pub trait Platform { + /// Interrupt number type for the platform + type IrqNumber; + + /// Address, to which the kernel is expected to be loaded for this platform + const KERNEL_PHYS_BASE: usize; + + /// Initializes the platform devices to their usable state. + /// + /// # Safety + /// + /// Unsafe to call if the platform has already been initialized. + unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error>; + /// Initializes the primary serial device to provide the debugging output as early as possible. + /// + /// # Safety + /// + /// Unsafe to call if the device has already been initialized. + unsafe fn init_primary_serial(&self); + + /// Returns a display name for the platform + fn name(&self) -> &'static str; + + /// Returns a reference to the primary serial device. + /// + /// # Note + /// + /// May not be initialized at the moment of calling. + fn primary_serial(&self) -> Option<&dyn SerialDevice>; + + /// Returns a reference to the platform's interrupt controller. + /// + /// # Note + /// + /// May not be initialized at the moment of calling. + fn interrupt_controller(&self) -> &dyn InterruptController; + + /// Returns the platform's primary timestamp source. + /// + /// # Note + /// + /// May not be initialized at the moment of calling. + fn timestamp_source(&self) -> &dyn TimestampSource; +} diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs new file mode 100644 index 00000000..ab79c5c7 --- /dev/null +++ b/src/device/serial/mod.rs @@ -0,0 +1,15 @@ +//! Serial device interfaces +use abi::error::Error; + +use super::Device; + +pub mod pl011; + +/// Generic serial device interface +pub trait SerialDevice: Device { + /// Sends (blocking) a single byte into the serial port + fn send(&self, byte: u8) -> Result<(), Error>; + + /// Receive a single byte from the serial port, blocking if necessary + fn receive(&self, blocking: bool) -> Result; +} diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs new file mode 100644 index 00000000..be767314 --- /dev/null +++ b/src/device/serial/pl011.rs @@ -0,0 +1,187 @@ +//! ARM PL011 driver +use abi::error::Error; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite, WriteOnly}, +}; +use vfs::CharDevice; + +use super::SerialDevice; +use crate::{ + arch::{aarch64::gic::IrqNumber, PLATFORM}, + device::{ + interrupt::InterruptSource, + platform::Platform, + tty::{CharRing, TtyDevice}, + Device, + }, + mem::device::DeviceMemoryIo, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +register_bitfields! { + u32, + FR [ + TXFF OFFSET(5) NUMBITS(1) [], + RXFE OFFSET(4) NUMBITS(1) [], + BUSY OFFSET(3) NUMBITS(1) [], + ], + CR [ + RXE OFFSET(9) NUMBITS(1) [], + TXE OFFSET(8) NUMBITS(1) [], + UARTEN OFFSET(0) NUMBITS(1) [], + ], + ICR [ + ALL OFFSET(0) NUMBITS(11) [], + ], + IMSC [ + RXIM OFFSET(4) NUMBITS(1) [], + ] +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + /// Transmit/receive data register + (0x00 => DR: ReadWrite), + (0x04 => _0), + (0x18 => FR: ReadOnly), + (0x1C => _1), + (0x2C => LCR_H: ReadWrite), + (0x30 => CR: ReadWrite), + (0x34 => IFLS: ReadWrite), + (0x38 => IMSC: ReadWrite), + (0x3C => _2), + (0x44 => ICR: WriteOnly), + (0x48 => @END), + } +} + +struct Pl011Inner { + regs: DeviceMemoryIo, +} + +/// PL011 device instance +pub struct Pl011 { + inner: OneTimeInit>, + base: usize, + irq: IrqNumber, + ring: CharRing<16>, +} + +impl Pl011Inner { + fn send_byte(&mut self, b: u8) -> Result<(), Error> { + while self.regs.FR.matches_all(FR::TXFF::SET) { + core::hint::spin_loop(); + } + self.regs.DR.set(b as u32); + Ok(()) + } + + fn recv_byte(&mut self, blocking: bool) -> Result { + if self.regs.FR.matches_all(FR::RXFE::SET) { + if !blocking { + todo!(); + } + while self.regs.FR.matches_all(FR::RXFE::SET) { + core::hint::spin_loop(); + } + } + + Ok(self.regs.DR.get() as u8) + } + + unsafe fn init(&mut self) { + self.regs.CR.set(0); + self.regs.ICR.write(ICR::ALL::CLEAR); + self.regs + .CR + .write(CR::UARTEN::SET + CR::TXE::SET + CR::RXE::SET); + } +} + +impl TtyDevice<16> for Pl011 { + fn ring(&self) -> &CharRing<16> { + &self.ring + } +} + +impl CharDevice for Pl011 { + fn write(&self, blocking: bool, data: &[u8]) -> Result { + assert!(blocking); + self.line_write(data) + } + + fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { + assert!(blocking); + self.line_read(data) + } +} + +impl SerialDevice for Pl011 { + fn send(&self, byte: u8) -> Result<(), Error> { + self.inner.get().lock().send_byte(byte) + } + + fn receive(&self, blocking: bool) -> Result { + self.inner.get().lock().recv_byte(blocking) + } +} + +impl InterruptSource for Pl011 { + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = PLATFORM.interrupt_controller(); + + intc.register_handler(self.irq, self)?; + self.inner.get().lock().regs.IMSC.modify(IMSC::RXIM::SET); + intc.enable_irq(self.irq)?; + + Ok(()) + } + + fn handle_irq(&self) -> Result<(), Error> { + let inner = self.inner.get().lock(); + inner.regs.ICR.write(ICR::ALL::CLEAR); + + let byte = inner.regs.DR.get(); + drop(inner); + + self.recv_byte(byte as u8); + + Ok(()) + } +} + +impl Device for Pl011 { + unsafe fn init(&self) -> Result<(), Error> { + let mut inner = Pl011Inner { + regs: DeviceMemoryIo::map("pl011 UART", self.base)?, + }; + inner.init(); + + self.inner.init(IrqSafeSpinlock::new(inner)); + Ok(()) + } + + fn name(&self) -> &'static str { + "pl011" + } +} + +impl Pl011 { + /// Constructs an instance of the device at `base`. + /// + /// # Safety + /// + /// The caller must ensure the address is valid. + pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self { + Self { + inner: OneTimeInit::new(), + ring: CharRing::new(), + base, + irq, + } + } +} diff --git a/src/device/timer.rs b/src/device/timer.rs new file mode 100644 index 00000000..bba34bd8 --- /dev/null +++ b/src/device/timer.rs @@ -0,0 +1,13 @@ +//! Time-providing device interfaces +use core::time::Duration; + +use abi::error::Error; + +use super::Device; + +/// Interface for devices capable of providing some notion of time +pub trait TimestampSource: Device { + /// Returns current time signalled by the device. The time may not be a "real" time and instead + /// is assumed to be monotonically increasing. + fn timestamp(&self) -> Result; +} diff --git a/src/device/tty.rs b/src/device/tty.rs new file mode 100644 index 00000000..df3ca931 --- /dev/null +++ b/src/device/tty.rs @@ -0,0 +1,155 @@ +//! Terminal driver implementation +use abi::error::Error; + +use crate::{proc::wait::Wait, sync::IrqSafeSpinlock}; + +use super::serial::SerialDevice; + +struct CharRingInner { + rd: usize, + wr: usize, + data: [u8; N], + flags: u8, +} + +/// Ring buffer for a character device. Handles reads, writes and channel notifications for a +/// terminal device. +pub struct CharRing { + wait_read: Wait, + wait_write: Wait, + inner: IrqSafeSpinlock>, +} + +/// Terminal device interface +pub trait TtyDevice: SerialDevice { + /// Returns the ring buffer associated with the device + fn ring(&self) -> &CharRing; + + /// Returns `true` if data is ready to be read from or written to the terminal + fn is_ready(&self, write: bool) -> Result { + let ring = self.ring(); + if write { + todo!(); + } else { + Ok(ring.is_readable()) + } + } + + /// Sends a single byte to the terminal + fn line_send(&self, byte: u8) -> Result<(), Error> { + self.send(byte) + } + + /// Receives a single byte from the terminal + fn recv_byte(&self, byte: u8) { + let ring = self.ring(); + ring.putc(byte, false).ok(); + } + + /// Reads and processes data from the terminal + fn line_read(&'static self, data: &mut [u8]) -> Result { + let ring = self.ring(); + + if data.is_empty() { + return Ok(0); + } + + let byte = ring.getc()?; + data[0] = byte; + Ok(1) + } + + /// Processes and writes the data to the terminal + fn line_write(&self, data: &[u8]) -> Result { + for &byte in data { + self.line_send(byte)?; + } + Ok(data.len()) + } + + /// Writes raw data to the terminal bypassing the processing functions + fn raw_write(&self, _data: &[u8]) -> Result { + todo!(); + } +} + +impl CharRingInner { + #[inline] + const fn is_readable(&self) -> bool { + if self.rd <= self.wr { + (self.wr - self.rd) > 0 + } else { + (self.wr + (N - self.rd)) > 0 + } + } + + #[inline] + unsafe fn read_unchecked(&mut self) -> u8 { + let res = self.data[self.rd]; + self.rd = (self.rd + 1) % N; + res + } + + #[inline] + unsafe fn write_unchecked(&mut self, ch: u8) { + self.data[self.wr] = ch; + self.wr = (self.wr + 1) % N; + } +} + +impl CharRing { + /// Constructs an empty ring buffer + pub const fn new() -> Self { + Self { + inner: IrqSafeSpinlock::new(CharRingInner { + rd: 0, + wr: 0, + data: [0; N], + flags: 0, + }), + wait_read: Wait::new("char_ring_read"), + wait_write: Wait::new("char_ring_write"), + } + } + + /// Returns `true` if the buffer has data to read + pub fn is_readable(&self) -> bool { + let inner = self.inner.lock(); + inner.is_readable() || inner.flags != 0 + } + + /// Reads a single character from the buffer, blocking until available + pub fn getc(&'static self) -> Result { + let mut lock = self.inner.lock(); + loop { + if !lock.is_readable() && lock.flags == 0 { + drop(lock); + self.wait_read.wait(None)?; + lock = self.inner.lock(); + } else { + break; + } + } + + let byte = unsafe { lock.read_unchecked() }; + drop(lock); + self.wait_write.wakeup_one(); + // TODO WAIT_SELECT + Ok(byte) + } + + /// Sends a single character to the buffer + pub fn putc(&self, ch: u8, blocking: bool) -> Result<(), Error> { + let mut lock = self.inner.lock(); + if blocking { + todo!(); + } + unsafe { + lock.write_unchecked(ch); + } + drop(lock); + self.wait_read.wakeup_one(); + // TODO WAIT_SELECT + Ok(()) + } +} diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs new file mode 100644 index 00000000..e6631fa8 --- /dev/null +++ b/src/fs/devfs.rs @@ -0,0 +1,57 @@ +//! Device virtual file system +use core::sync::atomic::{AtomicUsize, Ordering}; + +use abi::error::Error; +use alloc::{boxed::Box, format, string::String}; +use vfs::{CharDevice, CharDeviceWrapper, Vnode, VnodeKind, VnodeRef}; + +use crate::util::OneTimeInit; + +/// Describes the kind of a character device +#[derive(Debug)] +pub enum CharDeviceType { + /// Serial terminal + TtySerial, +} + +static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); + +/// Sets up the device filesystem +pub fn init() { + let node = Vnode::new("", VnodeKind::Directory); + DEVFS_ROOT.init(node); +} + +/// Returns the root of the devfs. +/// +/// # Panics +/// +/// Will panic if the devfs hasn't yet been initialized. +pub fn root() -> &'static VnodeRef { + DEVFS_ROOT.get() +} + +fn _add_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Error> { + infoln!("Add char device: {}", name); + + let node = Vnode::new(name, VnodeKind::Char); + node.set_data(Box::new(CharDeviceWrapper::new(dev))); + + DEVFS_ROOT.get().add_child(node); + + Ok(()) +} + +/// Adds a character device to the devfs +pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Result<(), Error> { + static TTYS_COUNT: AtomicUsize = AtomicUsize::new(0); + + let (count, prefix) = match kind { + CharDeviceType::TtySerial => (&TTYS_COUNT, "ttyS"), + }; + + let value = count.fetch_add(1, Ordering::AcqRel); + let name = format!("{}{}", prefix, value); + + _add_char_device(dev, name) +} diff --git a/src/fs/mod.rs b/src/fs/mod.rs new file mode 100644 index 00000000..65c874be --- /dev/null +++ b/src/fs/mod.rs @@ -0,0 +1,3 @@ +//! Filesystem implementations + +pub mod devfs; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 00000000..a5afd20c --- /dev/null +++ b/src/main.rs @@ -0,0 +1,86 @@ +//! osdev-x kernel crate +#![feature( + naked_functions, + asm_const, + panic_info_message, + optimize_attribute, + const_trait_impl, + maybe_uninit_slice, + linked_list_cursors +)] +#![allow(clippy::new_without_default)] +#![warn(missing_docs)] +#![no_std] +#![no_main] + +extern crate yggdrasil_abi as abi; + +use abi::io::{OpenFlags, RawFd}; +use task::process::Process; +use vfs::IoContext; + +use crate::fs::devfs; + +extern crate alloc; + +#[macro_use] +pub mod debug; +#[macro_use] +pub mod arch; + +pub mod device; +pub mod fs; +pub mod mem; +pub mod panic; +pub mod proc; +pub mod sync; +pub mod syscall; +pub mod task; +pub mod util; + +/// Entry point for common kernel code. +/// +/// # Note +/// +/// This function is meant to be used as a kernel-space process after all the platform-specific +/// initialization has finished. +pub fn kernel_main() { + // static USER_PROGRAM: &[u8] = include_bytes!(concat!( + // "../../target/aarch64-unknown-yggdrasil/", + // env!("PROFILE"), + // "/test_program" + // )); + + // let devfs_root = devfs::root(); + // let tty_node = devfs_root.lookup("ttyS0").unwrap(); + + // let ioctx = IoContext::new(devfs_root.clone()); + + // // Spawn a test user task + // let proc = + // proc::exec::create_from_memory(USER_PROGRAM, &["user-program", "argument 1", "argument 2"]); + + // match proc { + // Ok(proc) => { + // // Setup I/O for the process + // // let mut io = proc.io.lock(); + // // io.set_file(RawFd::STDOUT, todo!()).unwrap(); + // { + // let mut io = proc.io.lock(); + // io.set_ioctx(ioctx); + // let stdout = tty_node.open(OpenFlags::new().write()).unwrap(); + // let stderr = stdout.clone(); + + // io.set_file(RawFd::STDOUT, stdout).unwrap(); + // io.set_file(RawFd::STDERR, stderr).unwrap(); + // } + + // proc.enqueue_somewhere(); + // } + // Err(err) => { + // warnln!("Failed to create user process: {:?}", err); + // } + // }; + + Process::current().exit(0); +} diff --git a/src/mem/device.rs b/src/mem/device.rs new file mode 100644 index 00000000..898ddda8 --- /dev/null +++ b/src/mem/device.rs @@ -0,0 +1,76 @@ +//! Facilities for mapping devices to virtual address space +use core::{marker::PhantomData, mem::size_of, ops::Deref}; + +use abi::error::Error; + +use crate::arch::{Architecture, ARCHITECTURE}; + +/// Generic MMIO access mapping +#[derive(Clone)] +#[allow(unused)] +pub struct DeviceMemory { + name: &'static str, + base: usize, + size: usize, +} + +/// MMIO wrapper for `T` +pub struct DeviceMemoryIo { + mmio: DeviceMemory, + _pd: PhantomData, +} + +impl DeviceMemory { + /// Maps the device to some virtual memory address and constructs a wrapper for that range. + /// + /// # Safety + /// + /// The caller is responsible for making sure the (phys, size) range is valid and actually + /// points to some device's MMIO. The caller must also make sure no aliasing for that range is + /// possible. + pub unsafe fn map(name: &'static str, phys: usize, size: usize) -> Result { + if size > 0x1000 { + todo!("Device memory mappings larger than 4K"); + } + + let base = ARCHITECTURE.map_device_pages(phys, 1)?; + + Ok(Self { name, base, size }) + } +} + +impl DeviceMemoryIo { + /// Maps the `T` struct at `phys` to some virtual memory address and provides a [Deref]able + /// wrapper to it. + /// + /// # Safety + /// + /// The caller is responsible for making sure the `phys` address points to a MMIO region which + /// is at least `size_of::()` and no aliasing for that region is possible. + pub unsafe fn map(name: &'static str, phys: usize) -> Result { + DeviceMemory::map(name, phys, size_of::()).map(|t| Self::new(t)) + } + + /// Constructs a device MMIO wrapper from given [DeviceMemory] mapping. + /// + /// # Safety + /// + /// The caller must ensure `mmio` actually points to a device of type `T`. + pub unsafe fn new(mmio: DeviceMemory) -> Self { + assert!(mmio.size >= size_of::()); + // TODO check align + + Self { + mmio, + _pd: PhantomData, + } + } +} + +impl Deref for DeviceMemoryIo { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*(self.mmio.base as *const T) } + } +} diff --git a/src/mem/heap.rs b/src/mem/heap.rs new file mode 100644 index 00000000..e8fa441e --- /dev/null +++ b/src/mem/heap.rs @@ -0,0 +1,51 @@ +//! Kernel's global heap allocator +use core::{ + alloc::{GlobalAlloc, Layout}, + ptr::{null_mut, NonNull}, +}; + +use linked_list_allocator::Heap; +use spinning_top::Spinlock; + +struct KernelAllocator { + inner: Spinlock, +} + +impl KernelAllocator { + const fn empty() -> Self { + Self { + inner: Spinlock::new(Heap::empty()), + } + } + + unsafe fn init(&self, base: usize, size: usize) { + self.inner.lock().init(base as _, size); + } +} + +unsafe impl GlobalAlloc for KernelAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + match self.inner.lock().allocate_first_fit(layout) { + Ok(v) => v.as_ptr(), + Err(_) => null_mut(), + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let ptr = NonNull::new(ptr).unwrap(); + self.inner.lock().deallocate(ptr, layout) + } +} + +#[global_allocator] +static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); + +/// Sets up kernel's global heap with given memory range. +/// +/// # Safety +/// +/// The caller must ensure the range is valid and mapped virtual memory. +pub unsafe fn init_heap(heap_base: usize, heap_size: usize) { + debugln!("Heap: {:#x}..{:#x}", heap_base, heap_base + heap_size); + GLOBAL_HEAP.init(heap_base, heap_size); +} diff --git a/src/mem/mod.rs b/src/mem/mod.rs new file mode 100644 index 00000000..a84e6da3 --- /dev/null +++ b/src/mem/mod.rs @@ -0,0 +1,90 @@ +//! Memory management utilities and types +use crate::{ + arch::{Architecture, ArchitectureImpl, PlatformImpl}, + device::platform::Platform, +}; + +pub mod device; +pub mod heap; +pub mod phys; +pub mod table; + +/// Kernel's physical load address +pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; +/// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + +/// [KERNEL_VIRT_OFFSET]) +pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; + +/// Interface for converting between address spaces. +/// +/// # Safety +/// +/// An incorrect implementation can produce invalid address. +pub unsafe trait ConvertAddress { + /// Convert the address into a virtual one + /// + /// # Panics + /// + /// Panics if the address is already a virtual one + /// + /// # Safety + /// + /// An incorrect implementation can produce invalid address. + unsafe fn virtualize(self) -> Self; + /// Convert the address into a physical one + /// + /// # Panics + /// + /// Panics if the address is already a physical one + /// + /// # Safety + /// + /// An incorrect implementation can produce invalid address. + unsafe fn physicalize(self) -> Self; +} + +unsafe impl ConvertAddress for usize { + #[inline(always)] + unsafe fn virtualize(self) -> Self { + #[cfg(debug_assertions)] + if self > KERNEL_VIRT_OFFSET { + todo!(); + } + + self + KERNEL_VIRT_OFFSET + } + + #[inline(always)] + unsafe fn physicalize(self) -> Self { + #[cfg(debug_assertions)] + if self < KERNEL_VIRT_OFFSET { + todo!(); + } + + self - KERNEL_VIRT_OFFSET + } +} + +unsafe impl ConvertAddress for *mut T { + #[inline(always)] + unsafe fn virtualize(self) -> Self { + (self as usize).virtualize() as Self + } + + #[inline(always)] + unsafe fn physicalize(self) -> Self { + (self as usize).physicalize() as Self + } +} + +unsafe impl ConvertAddress for *const T { + #[inline(always)] + unsafe fn virtualize(self) -> Self { + (self as usize).virtualize() as Self + } + + #[inline(always)] + unsafe fn physicalize(self) -> Self { + (self as usize).physicalize() as Self + } +} diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs new file mode 100644 index 00000000..d38a6d16 --- /dev/null +++ b/src/mem/phys/manager.rs @@ -0,0 +1,95 @@ +//! Physical memory manager implementation +use core::mem::size_of; + +use abi::error::Error; + +use super::{Page, PageUsage}; + +/// Physical memory management interface +pub struct PhysicalMemoryManager { + pages: &'static mut [Page], + offset: usize, +} + +impl PhysicalMemoryManager { + /// Constructs a [PhysicalMemoryManager] with page tracking array placed at given + /// `base`..`base+size` range. Physical addresses allocated are offset by the given value. + /// + /// # Safety + /// + /// Addresses are not checked. The caller is responsible for making sure (base, size) ranges do + /// not alias/overlap, they're accessible through virtual memory and that the offset is a + /// meaningful value. + pub unsafe fn new(offset: usize, base: usize, size: usize) -> PhysicalMemoryManager { + // TODO check alignment + let page_count = size / size_of::(); + let pages = core::slice::from_raw_parts_mut(base as *mut _, page_count); + + for page in pages.iter_mut() { + *page = Page { + usage: PageUsage::Reserved, + refcount: 0, + }; + } + + PhysicalMemoryManager { pages, offset } + } + + /// Allocates a single page, marking it as used with `usage` + pub fn alloc_page(&mut self, usage: PageUsage) -> Result { + assert_ne!(usage, PageUsage::Available); + assert_ne!(usage, PageUsage::Reserved); + + for index in 0..self.pages.len() { + if self.pages[index].usage == PageUsage::Available { + self.pages[index].usage = PageUsage::Used; + return Ok(index * 4096 + self.offset); + } + } + + Err(Error::OutOfMemory) + } + + /// Allocates a contiguous range of physical pages, marking it as used with `usage` + pub fn alloc_contiguous_pages( + &mut self, + count: usize, + usage: PageUsage, + ) -> Result { + assert_ne!(usage, PageUsage::Available); + assert_ne!(usage, PageUsage::Reserved); + assert_ne!(count, 0); + + 'l0: for i in 0..self.pages.len() { + for j in 0..count { + if self.pages[i + j].usage != PageUsage::Available { + continue 'l0; + } + } + for j in 0..count { + let page = &mut self.pages[i + j]; + assert!(page.usage == PageUsage::Available); + page.usage = usage; + page.refcount = 1; + } + return Ok(self.offset + i * 0x1000); + } + + Err(Error::OutOfMemory) + } + + /// Marks a previously reserved page as available. + /// + /// # Panics + /// + /// Will panic if the address does not point to a valid, reserved (and unallocated) page. + pub fn add_available_page(&mut self, addr: usize) { + assert!(addr >= self.offset); + let index = (addr - self.offset) / 4096; + + assert_eq!(self.pages[index].usage, PageUsage::Reserved); + assert_eq!(self.pages[index].refcount, 0); + + self.pages[index].usage = PageUsage::Available; + } +} diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs new file mode 100644 index 00000000..4d9b666d --- /dev/null +++ b/src/mem/phys/mod.rs @@ -0,0 +1,199 @@ +//! Physical memory management facilities +use core::{iter::StepBy, mem::size_of, ops::Range}; + +use abi::error::Error; +use spinning_top::Spinlock; + +use crate::{ + absolute_address, + mem::{ + phys::reserved::{is_reserved, reserve_region}, + ConvertAddress, KERNEL_PHYS_BASE, + }, + util::OneTimeInit, +}; + +use self::manager::PhysicalMemoryManager; + +pub mod manager; +pub mod reserved; + +/// Represents the way in which the page is used (or not) +#[derive(PartialEq, Clone, Copy, Debug)] +#[repr(u32)] +pub enum PageUsage { + /// Page is not available for allocation or use + Reserved = 0, + /// Regular page available for allocation + Available, + /// Page is used by some kernel facility + Used, +} + +/// Page descriptor structure for the page management array +#[repr(C)] +pub struct Page { + usage: PageUsage, + refcount: u32, +} + +/// Defines an usable memory region +#[derive(Clone, Copy, Debug)] +pub struct PhysicalMemoryRegion { + /// Start of the region + pub base: usize, + /// Length of the region + pub size: usize, +} + +impl PhysicalMemoryRegion { + /// Returns the end address of the region + pub const fn end(&self) -> usize { + self.base + self.size + } + + /// Returns an address range covered by the region + pub const fn range(&self) -> Range { + self.base..self.end() + } + + /// Provides an iterator over the pages in the region + pub const fn pages(&self) -> StepBy> { + self.range().step_by(0x1000) + } +} + +/// Global physical memory manager +pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); + +/// Allocates a single physical page from the global manager +pub fn alloc_page(usage: PageUsage) -> Result { + PHYSICAL_MEMORY.get().lock().alloc_page(usage) +} + +/// Allocates a contiguous range of physical pages from the global manager +pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result { + PHYSICAL_MEMORY + .get() + .lock() + .alloc_contiguous_pages(count, usage) +} + +fn physical_memory_range>( + it: I, +) -> Option<(usize, usize)> { + let mut start = usize::MAX; + let mut end = usize::MIN; + + for reg in it { + if reg.base < start { + start = reg.base; + } + if reg.base + reg.size > end { + end = reg.base + reg.size; + } + } + + if start == usize::MAX || end == usize::MIN { + None + } else { + Some((start, end)) + } +} + +fn find_contiguous_region>( + it: I, + count: usize, +) -> Option { + for region in it { + let mut collected = 0; + let mut base_addr = None; + + for addr in region.pages() { + if is_reserved(addr) { + collected = 0; + base_addr = None; + continue; + } + if base_addr.is_none() { + base_addr = Some(addr); + } + collected += 1; + if collected == count { + return base_addr; + } + } + } + todo!() +} + +/// Initializes physical memory manager from given available memory region iterator. +/// +/// 1. Finds a non-reserved range to place the page tracking array. +/// 2. Adds all non-reserved pages to the manager. +/// +/// # Safety +/// +/// The caller must ensure this function has not been called before and that the regions +/// are valid and actually available. +pub unsafe fn init_from_iter + Clone>( + it: I, +) -> Result<(), Error> { + let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap(); + let total_count = (phys_end - phys_start) / 0x1000; + let pages_array_size = total_count * size_of::(); + + debugln!("Initializing physical memory manager"); + debugln!("Total tracked pages: {}", total_count); + + // Reserve memory regions from which allocation is forbidden + reserve_region("kernel", kernel_physical_memory_region()); + + let pages_array_base = find_contiguous_region(it.clone(), (pages_array_size + 0xFFF) / 0x1000) + .ok_or(Error::OutOfMemory)?; + + debugln!( + "Placing page tracking at {:#x}", + pages_array_base.virtualize() + ); + + reserve_region( + "pages", + PhysicalMemoryRegion { + base: pages_array_base, + size: (pages_array_size + 0xFFF) & !0xFFF, + }, + ); + + let mut manager = + PhysicalMemoryManager::new(phys_start, pages_array_base.virtualize(), pages_array_size); + let mut page_count = 0; + + for region in it { + for page in region.pages() { + if is_reserved(page) { + continue; + } + + manager.add_available_page(page); + page_count += 1; + } + } + + infoln!("{} available pages", page_count); + + PHYSICAL_MEMORY.init(Spinlock::new(manager)); + Ok(()) +} + +fn kernel_physical_memory_region() -> PhysicalMemoryRegion { + extern "C" { + static __kernel_size: u8; + } + let size = absolute_address!(__kernel_size); + + PhysicalMemoryRegion { + base: KERNEL_PHYS_BASE, + size, + } +} diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs new file mode 100644 index 00000000..4971dddc --- /dev/null +++ b/src/mem/phys/reserved.rs @@ -0,0 +1,33 @@ +//! Utilities for handling reserved memory regions + +use crate::util::StaticVector; + +use super::PhysicalMemoryRegion; + +static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); + +/// Marks a region of physical memory as reserved. +/// +/// # Safety +/// +/// Can only be called from initialization code **before** physical memory manager is initialized. +pub unsafe fn reserve_region(reason: &str, region: PhysicalMemoryRegion) { + debugln!( + "Reserve {:?} memory: {:#x}..{:#x}", + reason, + region.base, + region.end() + ); + + RESERVED_MEMORY.push(region); +} + +/// Returns `true` if `addr` refers to any reserved memory region +pub fn is_reserved(addr: usize) -> bool { + for region in unsafe { RESERVED_MEMORY.iter() } { + if region.range().contains(&addr) { + return true; + } + } + false +} diff --git a/src/mem/table.rs b/src/mem/table.rs new file mode 100644 index 00000000..0456bca3 --- /dev/null +++ b/src/mem/table.rs @@ -0,0 +1,40 @@ +//! Virtual memory table interface +use abi::error::Error; + +pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; + +/// Interface for virtual memory address space management +pub trait VirtualMemoryManager { + /// Allocates a region of virtual memory inside the address space and maps it to physical + /// memory pages with given attributes + fn allocate( + &self, + hint: Option, + len: usize, + attrs: PageAttributes, + ) -> Result; + + /// Releases the virtual memory region from the address space and the pages it refers to + fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error>; +} + +/// Interface for non-terminal tables to retrieve the next level of address translation tables +pub trait NextPageTable { + /// Type for the next-level page table + type NextLevel; + + /// Tries looking up a next-level table at given index, allocating and mapping one if it is not + /// present there + fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error>; + /// Returns a mutable reference to a next-level table at `index`, if present + fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel>; +} + +/// Interface for a single level of address translation +#[const_trait] +pub trait EntryLevel: Copy { + /// Returns the index into a page table for a given address + fn index(addr: usize) -> usize; + /// Returns the offset of an address from the page start at current level + fn page_offset(addr: usize) -> usize; +} diff --git a/src/panic.rs b/src/panic.rs new file mode 100644 index 00000000..8bdcb080 --- /dev/null +++ b/src/panic.rs @@ -0,0 +1,77 @@ +//! Kernel panic handler code +use core::sync::atomic::{AtomicBool, Ordering}; + +use crate::{ + arch::{Architecture, ArchitectureImpl, CpuMessage, PLATFORM}, + debug::{debug_internal, LogLevel}, + device::{interrupt::IpiDeliveryTarget, platform::Platform}, + sync::SpinFence, +}; + +// Just a fence to ensure secondary panics don't trash the screen +static PANIC_FINISHED_FENCE: SpinFence = SpinFence::new(); + +/// Panic handler for CPUs other than the one that initiated it +pub fn panic_secondary() -> ! { + unsafe { + ArchitectureImpl::set_interrupt_mask(true); + } + + PANIC_FINISHED_FENCE.wait_one(); + + log_print_raw!(LogLevel::Fatal, "X"); + + loop { + ArchitectureImpl::wait_for_interrupt(); + } +} + +#[panic_handler] +fn panic_handler(pi: &core::panic::PanicInfo) -> ! { + unsafe { + ArchitectureImpl::set_interrupt_mask(true); + } + static PANIC_HAPPENED: AtomicBool = AtomicBool::new(false); + + if PANIC_HAPPENED + .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) + .is_ok() + { + // Let other CPUs know we're screwed + unsafe { + PLATFORM + .interrupt_controller() + .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) + .ok(); + } + + log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); + log_print_raw!(LogLevel::Fatal, "Kernel panic "); + + if let Some(location) = pi.location() { + log_print_raw!( + LogLevel::Fatal, + "at {}:{}:", + location.file(), + location.line() + ); + } else { + log_print_raw!(LogLevel::Fatal, ":"); + } + + log_print_raw!(LogLevel::Fatal, "\n"); + + if let Some(msg) = pi.message() { + debug_internal(*msg, LogLevel::Fatal); + log_print_raw!(LogLevel::Fatal, "\n"); + } + log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); + + log_print_raw!(LogLevel::Fatal, "X"); + PANIC_FINISHED_FENCE.signal(); + } + + loop { + ArchitectureImpl::wait_for_interrupt(); + } +} diff --git a/src/proc/exec.rs b/src/proc/exec.rs new file mode 100644 index 00000000..b265304c --- /dev/null +++ b/src/proc/exec.rs @@ -0,0 +1,101 @@ +//! Binary execution functions +use core::mem::size_of; + +use abi::error::Error; +use alloc::rc::Rc; + +use crate::{ + arch::aarch64::context::TaskContext, + mem::{ + phys::{self, PageUsage}, + table::{AddressSpace, PageAttributes}, + ConvertAddress, + }, + proc, + task::process::Process, +}; + +fn setup_args(space: &mut AddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { + // arg data len + let args_size: usize = args.iter().map(|x| x.len()).sum(); + // 1 + arg ptr:len count + let args_ptr_size = (1 + args.len() * 2) * size_of::(); + + let total_size = args_size + args_ptr_size; + + if total_size > 0x1000 { + todo!(); + } + + debugln!("arg data size = {}", args_size); + + let phys_page = phys::alloc_page(PageUsage::Used)?; + // TODO check if this doesn't overwrite anything + space.map_page(virt, phys_page, PageAttributes::AP_BOTH_READWRITE)?; + + let write = unsafe { phys_page.virtualize() }; + + let mut offset = args_ptr_size; + + unsafe { + (write as *mut usize).write_volatile(args.len()); + } + + for i in 0..args.len() { + // Place the argument pointer + let ptr_place = write + (i * 2 + 1) * size_of::(); + let len_place = ptr_place + size_of::(); + unsafe { + (ptr_place as *mut usize).write_volatile(virt + offset); + (len_place as *mut usize).write_volatile(args[i].len()); + } + offset += args[i].len(); + } + + // Place the argument data + unsafe { + let arg_data_slice = + core::slice::from_raw_parts_mut((write + args_ptr_size) as *mut u8, args_size); + let mut offset = 0; + for &s in args { + arg_data_slice[offset..offset + s.len()].copy_from_slice(s.as_bytes()); + offset += s.len(); + } + } + + Ok(()) +} + +/// Sets up a userspace structure from a slice defining an ELF binary +pub fn create_from_memory(data: &[u8], args: &[&str]) -> Result, Error> { + const USER_STACK_PAGES: usize = 8; + + let mut space = AddressSpace::new_empty()?; + let elf_entry = proc::load_elf_from_memory(&mut space, data); + + let virt_stack_base = 0x10000000; + // 0x1000 of guard page + let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; + + for i in 0..USER_STACK_PAGES { + let phys = phys::alloc_page(PageUsage::Used)?; + space.map_page( + virt_stack_base + i * 0x1000, + phys, + PageAttributes::AP_BOTH_READWRITE, + )?; + } + + setup_args(&mut space, virt_args_base, args)?; + + debugln!("Entry: {:#x}", elf_entry); + + let context = TaskContext::user( + elf_entry, + virt_args_base, + space.physical_address(), + virt_stack_base + USER_STACK_PAGES * 0x1000, + )?; + + Ok(Process::new_with_context(Some(space), context)) +} diff --git a/src/proc/io.rs b/src/proc/io.rs new file mode 100644 index 00000000..e8de1fbe --- /dev/null +++ b/src/proc/io.rs @@ -0,0 +1,71 @@ +//! Process I/O management +use abi::{error::Error, io::RawFd}; +use alloc::collections::BTreeMap; +use vfs::{FileRef, IoContext}; + +/// I/O context of a process, contains information like root, current directory and file +/// descriptor table +pub struct ProcessIo { + ioctx: Option, + files: BTreeMap, +} + +impl ProcessIo { + /// Constructs an uninitialized I/O context + pub fn new() -> Self { + Self { + ioctx: None, + files: BTreeMap::new(), + } + } + + /// Returns a file given descriptor refers to + pub fn file(&self, fd: RawFd) -> Result { + self.files + .get(&fd) + .cloned() + .ok_or_else(|| Error::InvalidFile) + } + + /// Sets the inner I/O context + pub fn set_ioctx(&mut self, ioctx: IoContext) { + self.ioctx.replace(ioctx); + } + + /// Inserts a file into the descriptor table. Returns error if the file is already present for + /// given descriptor. + pub fn set_file(&mut self, fd: RawFd, file: FileRef) -> Result<(), Error> { + if self.files.contains_key(&fd) { + todo!(); + } + + self.files.insert(fd, file); + Ok(()) + } + + /// Allocates a slot for a file and returns it + pub fn place_file(&mut self, file: FileRef) -> Result { + for idx in 0..64 { + let fd = RawFd(idx); + if !self.files.contains_key(&fd) { + self.files.insert(fd, file); + return Ok(fd); + } + } + todo!(); + } + + /// Closes the file and removes it from the table + pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { + let file = self.files.remove(&fd); + if file.is_none() { + todo!(); + } + Ok(()) + } + + /// Returns the inner I/O context reference + pub fn ioctx(&mut self) -> &mut IoContext { + self.ioctx.as_mut().unwrap() + } +} diff --git a/src/proc/mod.rs b/src/proc/mod.rs new file mode 100644 index 00000000..e61e6f8f --- /dev/null +++ b/src/proc/mod.rs @@ -0,0 +1,91 @@ +//! Internal management for processes + +pub mod exec; +pub mod io; +pub mod wait; + +use aarch64_cpu::registers::TTBR0_EL1; +use elf::{ + abi::{PF_W, PF_X, PT_LOAD}, + endian::AnyEndian, + ElfBytes, +}; +use tock_registers::interfaces::Writeable; + +use crate::{ + arch::aarch64::table::tlb_flush_vaae1, + mem::{ + phys::{self, PageUsage}, + table::{AddressSpace, PageAttributes}, + }, +}; + +fn load_segment(space: &mut AddressSpace, addr: usize, data: &[u8], memsz: usize, elf_attrs: u32) { + let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { + (0, 0) => PageAttributes::AP_BOTH_READONLY, + (_, 0) => PageAttributes::AP_BOTH_READWRITE, + (0, _) => PageAttributes::AP_BOTH_READONLY, + (_, _) => PageAttributes::AP_BOTH_READWRITE, + }; + + let aligned_start = addr & !0xFFF; + let aligned_end = (addr + memsz + 0xFFF) & !0xFFF; + + // Map and write pages + for page in (aligned_start..aligned_end).step_by(0x1000) { + if let Some(_phys) = space.translate(page) { + todo!(); + } else { + let phys = phys::alloc_page(PageUsage::Used).unwrap(); + space + .map_page(page, phys, PageAttributes::AP_BOTH_READWRITE) + .unwrap(); + + debugln!("MAP (alloc) {:#x} -> {:#x}", page, phys); + tlb_flush_vaae1(page); + } + } + + unsafe { + // Write the data + let dst = core::slice::from_raw_parts_mut(addr as *mut u8, memsz); + dst[..data.len()].copy_from_slice(data); + + // Zero the rest + dst[data.len()..memsz].fill(0); + } + + // Map the region as readonly + for page in (aligned_start..aligned_end).step_by(0x1000) { + let phys = space.translate(page).unwrap(); + space.map_page(page, phys, attrs).unwrap(); + } +} + +/// Loads an ELF image into the address space from a slice +pub fn load_elf_from_memory(space: &mut AddressSpace, src: &[u8]) -> usize { + // Map the address space temporarily + TTBR0_EL1.set(space.physical_address() as u64); + + let elf = ElfBytes::::minimal_parse(src).unwrap(); + + for phdr in elf.segments().unwrap() { + if phdr.p_type != PT_LOAD { + continue; + } + + debugln!("LOAD {:#x}", phdr.p_vaddr); + let data = &src[phdr.p_offset as usize..(phdr.p_offset + phdr.p_filesz) as usize]; + load_segment( + space, + phdr.p_vaddr as usize, + data, + phdr.p_memsz as usize, + phdr.p_flags, + ); + } + + TTBR0_EL1.set_baddr(0); + + elf.ehdr.e_entry as usize +} diff --git a/src/proc/wait.rs b/src/proc/wait.rs new file mode 100644 index 00000000..70c2de0f --- /dev/null +++ b/src/proc/wait.rs @@ -0,0 +1,174 @@ +//! Wait channel implementation +use core::time::Duration; + +use abi::error::Error; +use alloc::{collections::LinkedList, rc::Rc}; + +use crate::{ + arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, task::process::Process, +}; + +/// Defines whether the wait channel is available for a specific task +#[derive(Clone, Copy, Debug)] +pub enum WaitStatus { + /// Wait on the channel was interrupted + Interrupted, + /// Channel did not yet signal availability + Pending, + /// Channel has data available + Done, +} + +/// Wait notification channel +pub struct Wait { + queue: IrqSafeSpinlock>>, + // Used for tracing waits + #[allow(dead_code)] + name: &'static str, +} + +struct Timeout { + process: Rc, + deadline: Duration, +} + +impl Wait { + /// Constructs a new wait notification channel + pub const fn new(name: &'static str) -> Self { + Self { + name, + queue: IrqSafeSpinlock::new(LinkedList::new()), + } + } + + /// Wakes up tasks waiting for availability on this channel, but no more than `limit` + pub fn wakeup_some(&self, mut limit: usize) -> usize { + let mut queue = self.queue.lock(); + let mut count = 0; + while limit != 0 && !queue.is_empty() { + let proc = queue.pop_front().unwrap(); + + { + let mut tick_lock = TICK_LIST.lock(); + let mut cursor = tick_lock.cursor_front_mut(); + + while let Some(item) = cursor.current() { + if proc.id() == item.process.id() { + cursor.remove_current(); + break; + } else { + cursor.move_next(); + } + } + + drop(tick_lock); + + unsafe { + proc.set_wait_status(WaitStatus::Done); + } + proc.enqueue_somewhere(); + } + + limit -= 1; + count += 1; + } + + count + } + + /// Wakes up all tasks waiting on this channel + pub fn wakeup_all(&self) { + self.wakeup_some(usize::MAX); + } + + /// Wakes up a single task waiting on this channel + pub fn wakeup_one(&self) { + self.wakeup_some(1); + } + + /// Suspends the task until either the deadline is reached or this channel signals availability + pub fn wait(&'static self, deadline: Option) -> Result<(), Error> { + let process = Process::current(); + let mut queue_lock = self.queue.lock(); + queue_lock.push_back(process.clone()); + unsafe { + process.setup_wait(self); + } + + if let Some(deadline) = deadline { + TICK_LIST.lock().push_back(Timeout { + process: process.clone(), + deadline, + }); + } + + loop { + match process.wait_status() { + WaitStatus::Pending => (), + WaitStatus::Done => return Ok(()), + WaitStatus::Interrupted => todo!(), + } + + drop(queue_lock); + process.suspend(); + + queue_lock = self.queue.lock(); + + if let Some(deadline) = deadline { + let now = PLATFORM.timestamp_source().timestamp()?; + + if now > deadline { + let mut cursor = queue_lock.cursor_front_mut(); + + while let Some(item) = cursor.current() { + if item.id() == process.id() { + cursor.remove_current(); + return Err(Error::TimedOut); + } else { + cursor.move_next(); + } + } + + panic!(); + } + } + } + } +} + +static TICK_LIST: IrqSafeSpinlock> = IrqSafeSpinlock::new(LinkedList::new()); + +/// Suspends current task until given deadline +pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { + static SLEEP_NOTIFY: Wait = Wait::new("sleep"); + let now = PLATFORM.timestamp_source().timestamp()?; + let deadline = now + timeout; + + match SLEEP_NOTIFY.wait(Some(deadline)) { + // Just what we expected + Err(Error::TimedOut) => { + *remaining = Duration::ZERO; + Ok(()) + } + + Ok(_) => panic!("This should not happen"), + Err(e) => Err(e), + } +} + +/// Updates all pending timeouts and wakes up the tasks that have reached theirs +pub fn tick() { + let now = PLATFORM.timestamp_source().timestamp().unwrap(); + let mut list = TICK_LIST.lock(); + let mut cursor = list.cursor_front_mut(); + + while let Some(item) = cursor.current() { + if now > item.deadline { + let t = cursor.remove_current().unwrap(); + + t.process.enqueue_somewhere(); + } else { + cursor.move_next(); + } + } +} diff --git a/src/sync.rs b/src/sync.rs new file mode 100644 index 00000000..042388f8 --- /dev/null +++ b/src/sync.rs @@ -0,0 +1,177 @@ +//! Synchronization primitives +use core::{ + cell::UnsafeCell, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use aarch64_cpu::registers::DAIF; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +/// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is +/// met. +pub struct SpinFence { + value: AtomicUsize, +} + +/// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation +/// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. +pub struct IrqGuard(u64); + +struct SpinlockInner { + value: UnsafeCell, + state: AtomicBool, +} + +struct SpinlockInnerGuard<'a, T> { + lock: &'a SpinlockInner, +} + +/// Spinlock implementation which prevents interrupts to avoid deadlocks when an interrupt handler +/// tries to acquire a lock taken before the IRQ fired. +pub struct IrqSafeSpinlock { + inner: SpinlockInner, +} + +/// Token type allowing safe access to the underlying data of the [IrqSafeSpinlock]. Resumes normal +/// IRQ operation (if enabled before acquiring) when the lifetime is over. +pub struct IrqSafeSpinlockGuard<'a, T> { + // Must come first to ensure the lock is dropped first and only then IRQs are re-enabled + inner: SpinlockInnerGuard<'a, T>, + _irq: IrqGuard, +} + +// Spinlock impls +impl SpinlockInner { + const fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + state: AtomicBool::new(false), + } + } + + fn lock(&self) -> SpinlockInnerGuard { + // Loop until the lock can be acquired + while self + .state + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } + + SpinlockInnerGuard { lock: self } + } +} + +impl<'a, T> Deref for SpinlockInnerGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.lock.value.get() } + } +} + +impl<'a, T> DerefMut for SpinlockInnerGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.lock.value.get() } + } +} + +impl<'a, T> Drop for SpinlockInnerGuard<'a, T> { + fn drop(&mut self) { + self.lock + .state + .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) + .unwrap(); + } +} + +unsafe impl Sync for SpinlockInner {} +unsafe impl Send for SpinlockInner {} + +// IrqSafeSpinlock impls +impl IrqSafeSpinlock { + /// Wraps the value in a spinlock primitive + pub const fn new(value: T) -> Self { + Self { + inner: SpinlockInner::new(value), + } + } + + /// Attempts to acquire a lock. IRQs will be disabled until the lock is released. + pub fn lock(&self) -> IrqSafeSpinlockGuard { + // Disable IRQs to avoid IRQ handler trying to acquire the same lock + let irq_guard = IrqGuard::acquire(); + + // Acquire the inner lock + let inner = self.inner.lock(); + + IrqSafeSpinlockGuard { + inner, + _irq: irq_guard, + } + } +} + +impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +// IrqGuard impls +impl IrqGuard { + /// Saves the current IRQ state and masks them + pub fn acquire() -> Self { + let this = Self(DAIF.get()); + DAIF.modify(DAIF::I::SET); + this + } +} + +impl Drop for IrqGuard { + fn drop(&mut self) { + DAIF.set(self.0); + } +} + +// SpinFence impls +impl SpinFence { + /// Constructs a new [SpinFence] + pub const fn new() -> Self { + Self { + value: AtomicUsize::new(0), + } + } + + /// Resets a fence back to its original state + pub fn reset(&self) { + self.value.store(0, Ordering::Release); + } + + /// "Signals" a fence, incrementing its internal counter by one + pub fn signal(&self) { + self.value.fetch_add(1, Ordering::SeqCst); + } + + /// Waits until the fence is signalled at least the amount of times specified + pub fn wait_all(&self, count: usize) { + while self.value.load(Ordering::Acquire) < count { + core::hint::spin_loop(); + } + } + + /// Waits until the fence is signalled at least once + pub fn wait_one(&self) { + self.wait_all(1); + } +} diff --git a/src/syscall.rs b/src/syscall.rs new file mode 100644 index 00000000..2359a2e5 --- /dev/null +++ b/src/syscall.rs @@ -0,0 +1,145 @@ +//! System function call handlers +use core::time::Duration; + +use abi::{ + error::{Error, IntoSyscallResult}, + io::{OpenFlags, RawFd}, + SyscallFunction, +}; +use vfs::{Read, Write}; + +use crate::{ + mem::table::{PageAttributes, VirtualMemoryManager}, + proc::wait, + task::process::Process, +}; + +fn arg_buffer_ref<'a>(base: usize, len: usize) -> Result<&'a [u8], Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + Ok(unsafe { core::slice::from_raw_parts(base as *const u8, len) }) +} + +fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8], Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + Ok(unsafe { core::slice::from_raw_parts_mut(base as *mut u8, len) }) +} + +fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error> { + let slice = arg_buffer_ref(base, len)?; + Ok(core::str::from_utf8(slice).unwrap()) +} + +fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { + match func { + SyscallFunction::DebugTrace => { + let pid = Process::get_current() + .as_deref() + .map(Process::id) + .unwrap_or(0); + let arg = arg_user_str(args[0] as usize, args[1] as usize)?; + debugln!("[{}] TRACE: {:?}", pid, arg); + + Ok(0) + } + SyscallFunction::Nanosleep => { + let seconds = args[0]; + let nanos = args[1] as u32; + let duration = Duration::new(seconds, nanos); + let mut remaining = Duration::ZERO; + + wait::sleep(duration, &mut remaining).unwrap(); + + Ok(0) + } + SyscallFunction::Exit => { + Process::current().exit(args[0] as _); + panic!(); + } + SyscallFunction::MapMemory => { + let len = args[1] as usize; + + let proc = Process::current(); + let space = proc.address_space(); + + if len & 0xFFF != 0 { + todo!(); + } + + let addr = space.allocate(None, len / 0x1000, PageAttributes::AP_BOTH_READWRITE); + debugln!("mmap({:#x}) = {:x?}", len, addr); + + addr + } + SyscallFunction::UnmapMemory => { + let addr = args[0] as usize; + let len = args[1] as usize; + + let proc = Process::current(); + let space = proc.address_space(); + + if len & 0xFFF != 0 { + todo!(); + } + + debugln!("munmap({:#x}, {:#x})", addr, len); + space.deallocate(addr, len)?; + + Ok(0) + } + SyscallFunction::Write => { + let fd = RawFd(args[0] as u32); + let data = arg_buffer_ref(args[1] as _, args[2] as _)?; + + let proc = Process::current(); + let io = proc.io.lock(); + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); + + file_borrow.write(data) + } + SyscallFunction::Read => { + let fd = RawFd(args[0] as u32); + let data = arg_buffer_mut(args[1] as _, args[2] as _)?; + + let proc = Process::current(); + let io = proc.io.lock(); + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); + + file_borrow.read(data) + } + SyscallFunction::Open => { + let path = arg_user_str(args[0] as usize, args[1] as usize)?; + let opts = OpenFlags(args[2] as u32); + + let proc = Process::current(); + let mut io = proc.io.lock(); + + let file = io.ioctx().open(None, path, opts)?; + let fd = io.place_file(file)?; + + Ok(fd.0 as usize) + } + SyscallFunction::Close => { + let fd = RawFd(args[0] as u32); + + let proc = Process::current(); + let mut io = proc.io.lock(); + io.close_file(fd)?; + Ok(0) + } + } +} + +/// Entrypoint for system calls that takes raw argument values +pub fn raw_syscall_handler(func: u64, args: &[u64]) -> u64 { + let Ok(func) = SyscallFunction::try_from(func as usize) else { + todo!("Undefined syscall: {}", func); + }; + + syscall_handler(func, args).into_syscall_result() as u64 +} diff --git a/src/task/mod.rs b/src/task/mod.rs new file mode 100644 index 00000000..29d86f3d --- /dev/null +++ b/src/task/mod.rs @@ -0,0 +1,110 @@ +//! Multitasking and process/thread management interfaces +use core::sync::atomic::Ordering; + +use aarch64_cpu::registers::MPIDR_EL1; +use abi::error::Error; +use alloc::{rc::Rc, vec::Vec}; +use tock_registers::interfaces::Readable; + +use crate::{ + arch::aarch64::{context::TaskContext, cpu::Cpu, smp::CPU_COUNT}, + kernel_main, + sync::{IrqSafeSpinlock, SpinFence}, + task::sched::CpuQueue, +}; + +use self::process::Process; + +pub mod process; +pub mod sched; + +/// Process identifier alias for clarity +pub type ProcessId = usize; + +/// Wrapper structure to hold all the system's processes +pub struct ProcessList { + data: Vec<(ProcessId, Rc)>, + last_process_id: ProcessId, +} + +impl ProcessList { + /// Constructs an empty process list + pub const fn new() -> Self { + Self { + last_process_id: 0, + data: Vec::new(), + } + } + + /// Inserts a new process into the list. + /// + /// # Safety + /// + /// Only meant to be called from inside the Process impl, as this function does not perform any + /// accounting information updates. + pub unsafe fn push(&mut self, process: Rc) -> ProcessId { + self.last_process_id += 1; + debugln!("Insert process with ID {}", self.last_process_id); + self.data.push((self.last_process_id, process)); + self.last_process_id + } + + /// Looks up a process by its ID + pub fn get(&self, id: ProcessId) -> Option<&Rc> { + self.data + .iter() + .find_map(|(i, p)| if *i == id { Some(p) } else { None }) + } +} + +/// Global shared process list +pub static PROCESSES: IrqSafeSpinlock = IrqSafeSpinlock::new(ProcessList::new()); + +/// Creates a new kernel-space process to execute a closure and queues it to some CPU +pub fn spawn_kernel_closure(f: F) -> Result<(), Error> { + let proc = Process::new_with_context(None, TaskContext::kernel_closure(f)?); + proc.enqueue_somewhere(); + + Ok(()) +} + +/// Sets up CPU queues and gives them some processes to run +pub fn init() -> Result<(), Error> { + let cpu_count = CPU_COUNT.load(Ordering::Acquire); + + // Create a queue for each CPU + sched::init_queues(Vec::from_iter((0..cpu_count).map(|_| CpuQueue::new()))); + + // Spawn kernel main task + spawn_kernel_closure(kernel_main)?; + + Ok(()) +} + +/// Sets up the local CPU queue and switches to some task in it for execution. +/// +/// # Note +/// +/// Any locks held at this point will not be dropped properly, which may lead to a deadlock. +/// +/// # Safety +/// +/// Only safe to call once at the end of non-threaded system initialization. +pub unsafe fn enter() -> ! { + static AP_CAN_ENTER: SpinFence = SpinFence::new(); + + let cpu_id = MPIDR_EL1.get() & 0xF; + + if cpu_id != 0 { + // Wait until BSP allows us to enter + AP_CAN_ENTER.wait_one(); + } else { + AP_CAN_ENTER.signal(); + } + + let queue = CpuQueue::for_cpu(cpu_id as usize); + let cpu = Cpu::local(); + cpu.init_queue(queue); + + queue.enter() +} diff --git a/src/task/process.rs b/src/task/process.rs new file mode 100644 index 00000000..e0c25def --- /dev/null +++ b/src/task/process.rs @@ -0,0 +1,246 @@ +//! Process data structures +use core::sync::atomic::{AtomicU32, Ordering}; + +use alloc::rc::Rc; +use atomic_enum::atomic_enum; + +use crate::{ + arch::aarch64::{context::TaskContext, cpu::Cpu}, + mem::table::AddressSpace, + proc::{ + io::ProcessIo, + wait::{Wait, WaitStatus}, + }, + sync::{IrqGuard, IrqSafeSpinlock}, + util::OneTimeInit, +}; + +use super::{sched::CpuQueue, ProcessId, PROCESSES}; + +/// Represents the states a process can be at some point in time +#[atomic_enum] +#[derive(PartialEq)] +pub enum ProcessState { + /// Process is ready for execution and is present in some CPU's queue + Ready, + /// Process is currently being executed by some CPU + Running, + /// Process is present in a global list, but is not queued for execution until it is resumed + Suspended, + /// Process is terminated and waits to be reaped + Terminated, +} + +struct ProcessInner { + pending_wait: Option<&'static Wait>, + wait_status: WaitStatus, +} + +/// Process data and state structure +pub struct Process { + context: TaskContext, + + // Process state info + id: OneTimeInit, + state: AtomicProcessState, + cpu_id: AtomicU32, + inner: IrqSafeSpinlock, + space: Option, + /// I/O state of the task + pub io: IrqSafeSpinlock, +} + +impl Process { + /// Creates a process from raw architecture-specific [TaskContext]. + /// + /// # Note + /// + /// Has side-effect of allocating a new PID for itself. + pub fn new_with_context(space: Option, context: TaskContext) -> Rc { + let this = Rc::new(Self { + context, + id: OneTimeInit::new(), + state: AtomicProcessState::new(ProcessState::Suspended), + cpu_id: AtomicU32::new(0), + inner: IrqSafeSpinlock::new(ProcessInner { + pending_wait: None, + wait_status: WaitStatus::Done, + }), + space, + io: IrqSafeSpinlock::new(ProcessIo::new()), + }); + + let id = unsafe { PROCESSES.lock().push(this.clone()) }; + this.id.init(id); + + this + } + + /// Returns a reference to the inner architecture-specific [TaskContext]. + pub fn context(&self) -> &TaskContext { + &self.context + } + + /// Returns this process' ID + pub fn id(&self) -> ProcessId { + *self.id.get() + } + + /// Returns the state of the process. + /// + /// # Note + /// + /// Maybe I should remove this and make ALL state changes atomic. + pub fn state(&self) -> ProcessState { + self.state.load(Ordering::Acquire) + } + + /// Atomically updates the state of the process and returns the previous one. + pub fn set_state(&self, state: ProcessState) -> ProcessState { + self.state.swap(state, Ordering::SeqCst) + } + + /// Marks the task as running on the specified CPU. + /// + /// # Safety + /// + /// Only meant to be called from scheduler routines. + pub unsafe fn set_running(&self, cpu: u32) { + self.cpu_id.store(cpu, Ordering::Release); + self.state.store(ProcessState::Running, Ordering::Release); + } + + /// Returns the address space of the task + pub fn address_space(&self) -> &AddressSpace { + self.space.as_ref().unwrap() + } + + /// Selects a suitable CPU queue and submits the process for execution. + /// + /// # Panics + /// + /// Currently, the code will panic if the process is queued/executing on any queue. + pub fn enqueue_somewhere(self: Rc) -> usize { + // Doesn't have to be precise, so even if something changes, we can still be rebalanced + // to another CPU + let (index, queue) = CpuQueue::least_loaded().unwrap(); + + self.enqueue_to(queue); + + index + } + + /// Submits the process to a specific queue. + /// + /// # Panics + /// + /// Currently, the code will panic if the process is queued/executing on any queue. + pub fn enqueue_to(self: Rc, queue: &CpuQueue) { + let current_state = self.state.swap(ProcessState::Ready, Ordering::SeqCst); + + if current_state != ProcessState::Suspended { + todo!("Handle attempt to enqueue an already queued/running/terminated process"); + } + + unsafe { + queue.enqueue(self); + } + } + + /// Marks the process as suspended, blocking it from being run until it's resumed. + /// + /// # Note + /// + /// The process may not halt its execution immediately when this function is called, only when + /// this function is called targeting the *current process* running on *local* CPU. + /// + /// # TODO + /// + /// The code currently does not allow suspension of active processes on either local or other + /// CPUs. + pub fn suspend(&self) { + let _irq = IrqGuard::acquire(); + let current_state = self.state.swap(ProcessState::Suspended, Ordering::SeqCst); + + match current_state { + // NOTE: I'm not sure if the process could've been queued between the store and this + // but most likely not (if I'm not that bad with atomics) + // Do nothing, its queue will just drop the process + ProcessState::Ready => (), + // Do nothing, not in a queue already + ProcessState::Suspended => (), + ProcessState::Terminated => panic!("Process is terminated"), + ProcessState::Running => { + let cpu_id = self.cpu_id.load(Ordering::Acquire); + let local_cpu_id = Cpu::local_id(); + let queue = Cpu::local().queue(); + + if cpu_id == local_cpu_id { + // Suspending a process running on local CPU + unsafe { queue.yield_cpu() } + } else { + todo!(); + } + } + } + } + + /// Sets up a pending wait for the process. + /// + /// # Safety + /// + /// This function is only meant to be called in no-IRQ context and when caller can guarantee + /// the task won't get scheduled to a CPU in such state. + pub unsafe fn setup_wait(&self, wait: &'static Wait) { + let mut inner = self.inner.lock(); + inner.pending_wait.replace(wait); + inner.wait_status = WaitStatus::Pending; + } + + /// Returns current wait status of the task + pub fn wait_status(&self) -> WaitStatus { + self.inner.lock().wait_status + } + + /// Updates the wait status for the task. + /// + /// # Safety + /// + /// This function is only meant to be called on waiting tasks, otherwise atomicity is not + /// guaranteed. + pub unsafe fn set_wait_status(&self, status: WaitStatus) { + self.inner.lock().wait_status = status; + } + + /// Returns the [Process] currently executing on local CPU, None if idling. + pub fn get_current() -> Option> { + let queue = Cpu::local().queue(); + queue.current_process() + } + + /// Wraps [Process::get_current()] for cases when the caller is absolutely sure there is a + /// running process (e.g. the call itself comes from a process). + pub fn current() -> Rc { + Self::get_current().unwrap() + } + + /// Terminate a process + pub fn exit(&self, status: usize) { + let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); + + debugln!("Process {} exited with code {}", self.id(), status); + + match current_state { + ProcessState::Suspended => (), + ProcessState::Ready => todo!(), + ProcessState::Running => unsafe { Cpu::local().queue().yield_cpu() }, + ProcessState::Terminated => todo!(), + } + } +} + +impl Drop for Process { + fn drop(&mut self) { + infoln!("Drop process!"); + } +} diff --git a/src/task/sched.rs b/src/task/sched.rs new file mode 100644 index 00000000..0c498cc1 --- /dev/null +++ b/src/task/sched.rs @@ -0,0 +1,273 @@ +//! Per-CPU queue implementation + +use aarch64_cpu::registers::CNTPCT_EL0; +use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; +use tock_registers::interfaces::Readable; + +use crate::{ + arch::aarch64::{context::TaskContext, cpu::Cpu}, + sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, + util::OneTimeInit, +}; + +use super::{ + process::{Process, ProcessState}, + ProcessId, +}; + +/// Per-CPU statistics +#[derive(Default)] +pub struct CpuQueueStats { + /// Ticks spent idling + pub idle_time: u64, + /// Ticks spent running CPU tasks + pub cpu_time: u64, + + /// Time since last measurement + measure_time: u64, +} + +/// Per-CPU queue's inner data, normally resides under a lock +pub struct CpuQueueInner { + /// Current process, None if idling + pub current: Option>, + /// LIFO queue for processes waiting for execution + pub queue: VecDeque>, + + /// CPU time usage statistics + pub stats: CpuQueueStats, +} + +/// Per-CPU queue +pub struct CpuQueue { + inner: IrqSafeSpinlock, + idle: TaskContext, +} + +static QUEUES: OneTimeInit> = OneTimeInit::new(); + +#[naked] +extern "C" fn __idle(_x: usize) -> ! { + unsafe { + core::arch::asm!("1: nop; b 1b", options(noreturn)); + } +} + +impl CpuQueueStats { + /// Reset the stats to zero values + pub fn reset(&mut self) { + self.cpu_time = 0; + self.idle_time = 0; + } +} + +impl CpuQueueInner { + /// Picks a next task for execution, skipping (dropping) those that were suspended. May return + /// None if the queue is empty or no valid task was found, in which case the scheduler should + /// go idle. + pub fn next_ready_task(&mut self) -> Option> { + while !self.queue.is_empty() { + let task = self.queue.pop_front().unwrap(); + + match task.state() { + ProcessState::Ready => { + return Some(task); + } + // Drop suspended tasks from the queue + ProcessState::Suspended | ProcessState::Terminated => (), + e => panic!("Unexpected process state in CpuQueue: {:?}", e), + } + } + + None + } + + /// Returns an iterator over all the processes in the queue plus the currently running process, + /// if there is one. + pub fn iter(&self) -> impl Iterator> { + Iterator::chain(self.queue.iter(), self.current.iter()) + } +} + +impl CpuQueue { + /// Constructs an empty queue with its own idle task + pub fn new() -> Self { + let idle = TaskContext::kernel(__idle, 0).expect("Could not construct an idle task"); + + Self { + inner: { + IrqSafeSpinlock::new(CpuQueueInner { + current: None, + queue: VecDeque::new(), + stats: CpuQueueStats::default(), + }) + }, + idle, + } + } + + /// Starts queue execution on current CPU. + /// + /// # Safety + /// + /// Only meant to be called from [crate::task::enter()] function. + pub unsafe fn enter(&self) -> ! { + // Start from idle thread to avoid having a Rc stuck here without getting dropped + let t = CNTPCT_EL0.get(); + self.lock().stats.measure_time = t; + + let mut inner = self.inner.lock(); + if let Some(proc) = inner.next_ready_task() { + inner.queue.push_back(proc.clone()); + inner.current = Some(proc.clone()); + proc.set_running(Cpu::local_id()); + + drop(inner); + proc.context().enter(); + } else { + drop(inner); + + self.idle.enter(); + }; + } + + /// Yields CPU execution to the next task in queue (or idle task if there aren't any). + /// + /// # Safety + /// + /// The function is only meant to be called from kernel threads (e.g. if they want to yield + /// CPU execution to wait for something) or interrupt handlers. + pub unsafe fn yield_cpu(&self) { + let mut inner = self.inner.lock(); + + let t = CNTPCT_EL0.get(); + let delta = t - inner.stats.measure_time; + inner.stats.measure_time = t; + + let current = inner.current.clone(); + + if let Some(current) = current.as_ref() { + if current.state() == ProcessState::Running { + current.set_state(ProcessState::Ready); + } + inner.queue.push_back(current.clone()); + + inner.stats.cpu_time += delta; + } else { + inner.stats.idle_time += delta; + } + + let next = inner.next_ready_task(); + + inner.current = next.clone(); + + // Can drop the lock, we hold current and next Rc's + drop(inner); + + let (from, _from_rc) = if let Some(current) = current.as_ref() { + (current.context(), Rc::strong_count(current)) + } else { + (&self.idle, 0) + }; + + let (to, _to_rc) = if let Some(next) = next.as_ref() { + next.set_running(Cpu::local_id()); + (next.context(), Rc::strong_count(next)) + } else { + (&self.idle, 0) + }; + + // if let Some(from) = current.as_ref() { + // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id()); + // } else { + // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); + // } + + // log_print_raw!(crate::debug::LogLevel::Info, " -> "); + + // if let Some(to) = next.as_ref() { + // log_print_raw!(crate::debug::LogLevel::Info, "{}", to.id()); + // } else { + // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); + // } + + // log_print_raw!(crate::debug::LogLevel::Info, "\n"); + + to.switch(from) + } + + /// Pushes the process to the back of the execution queue. + /// + /// # Safety + /// + /// Only meant to be called from Process impl. The function does not set any process accounting + /// information, which may lead to invalid states. + pub unsafe fn enqueue(&self, p: Rc) { + self.inner.lock().queue.push_back(p); + } + + /// Removes process with given PID from the exeuction queue. + pub fn dequeue(&self, _pid: ProcessId) { + todo!(); + } + + /// Returns the queue length at this moment. + /// + /// # Note + /// + /// This value may immediately change. + pub fn len(&self) -> usize { + self.inner.lock().queue.len() + } + + /// Returns `true` if the queue is empty at the moment. + /// + /// # Note + /// + /// This may immediately change. + pub fn is_empty(&self) -> bool { + self.inner.lock().queue.is_empty() + } + + /// Returns a safe reference to the inner data structure. + pub fn lock(&self) -> IrqSafeSpinlockGuard { + self.inner.lock() + } + + /// Returns the process currently being executed. + /// + /// # Note + /// + /// This function should be safe in all kernel thread/interrupt contexts: + /// + /// * (in kthread) the code calling this will still remain on the same thread. + /// * (in irq) the code cannot be interrupted and other CPUs shouldn't change this queue, so it + /// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu] + /// is called. + pub fn current_process(&self) -> Option> { + self.inner.lock().current.clone() + } + + /// Returns a queue for given CPU index + pub fn for_cpu(id: usize) -> &'static CpuQueue { + &QUEUES.get()[id] + } + + /// Returns an iterator over all queues of the system + #[inline] + pub fn all() -> impl Iterator { + QUEUES.get().iter() + } + + /// Picks a queue with the least amount of tasks in it + pub fn least_loaded() -> Option<(usize, &'static CpuQueue)> { + let queues = QUEUES.get(); + + queues.iter().enumerate().min_by_key(|(_, q)| q.len()) + } +} + +/// Initializes the global queue list +pub fn init_queues(queues: Vec) { + QUEUES.init(queues); +} diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 00000000..f05a9c59 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,117 @@ +//! Synchronization utilities +use core::{ + cell::UnsafeCell, + mem::MaybeUninit, + ops::Deref, + panic, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// Statically-allocated "dynamic" vector +pub struct StaticVector { + data: [MaybeUninit; N], + len: usize, +} + +/// Wrapper struct to ensure a value can only be initialized once and used only after that +#[repr(C)] +pub struct OneTimeInit { + value: UnsafeCell>, + state: AtomicBool, +} + +unsafe impl Sync for OneTimeInit {} +unsafe impl Send for OneTimeInit {} + +impl OneTimeInit { + /// Wraps the value in an [OneTimeInit] + pub const fn new() -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + state: AtomicBool::new(false), + } + } + + /// Returns `true` if the value has already been initialized + pub fn is_initialized(&self) -> bool { + self.state.load(Ordering::Acquire) + } + + /// Sets the underlying value of the [OneTimeInit]. If already initialized, panics. + #[track_caller] + pub fn init(&self, value: T) { + if self + .state + .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed) + .is_err() + { + panic!( + "{:?}: Double initialization of OneTimeInit", + panic::Location::caller() + ); + } + + unsafe { + (*self.value.get()).write(value); + } + } + + /// Returns an immutable reference to the underlying value and panics if it hasn't yet been + /// initialized + #[track_caller] + pub fn get(&self) -> &T { + if !self.state.load(Ordering::Acquire) { + panic!( + "{:?}: Attempt to dereference an uninitialized value", + panic::Location::caller() + ); + } + + unsafe { (*self.value.get()).assume_init_ref() } + } +} + +impl StaticVector { + /// Constructs an empty instance of [StaticVector] + pub const fn new() -> Self + where + T: Copy, + { + Self { + data: [MaybeUninit::uninit(); N], + len: 0, + } + } + + /// Appends an item to the vector. + /// + /// # Panics + /// + /// Will panic if the vector is full. + pub fn push(&mut self, value: T) { + if self.len == N { + panic!("Static vector overflow: reached limit of {}", N); + } + + self.data[self.len].write(value); + self.len += 1; + } + + /// Returns the number of items present in the vector + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the vector is empty + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +impl Deref for StaticVector { + type Target = [T]; + + fn deref(&self) -> &Self::Target { + unsafe { MaybeUninit::slice_assume_init_ref(&self.data[..self.len]) } + } +} From 0e6cb88756a6751f88bbed692428fb5c936655fe Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 18 Jul 2023 18:08:54 +0300 Subject: [PATCH 002/211] ***: test commit --- src/main.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index a5afd20c..db79e719 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,11 +15,7 @@ extern crate yggdrasil_abi as abi; -use abi::io::{OpenFlags, RawFd}; use task::process::Process; -use vfs::IoContext; - -use crate::fs::devfs; extern crate alloc; From 058bfddd5840db1c9dd75c3d25a63f60902e1578 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 18 Jul 2023 19:22:30 +0300 Subject: [PATCH 003/211] sched: Fix kernel panic (1 -> idle, idle -> idle) --- src/task/sched.rs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/task/sched.rs b/src/task/sched.rs index 0c498cc1..cd6e5f91 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -115,20 +115,7 @@ impl CpuQueue { // Start from idle thread to avoid having a Rc stuck here without getting dropped let t = CNTPCT_EL0.get(); self.lock().stats.measure_time = t; - - let mut inner = self.inner.lock(); - if let Some(proc) = inner.next_ready_task() { - inner.queue.push_back(proc.clone()); - inner.current = Some(proc.clone()); - proc.set_running(Cpu::local_id()); - - drop(inner); - proc.context().enter(); - } else { - drop(inner); - - self.idle.enter(); - }; + self.idle.enter() } /// Yields CPU execution to the next task in queue (or idle task if there aren't any). @@ -177,6 +164,7 @@ impl CpuQueue { (&self.idle, 0) }; + // log_print_raw!(crate::debug::LogLevel::Info, "{}: ", Cpu::local_id()); // if let Some(from) = current.as_ref() { // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id()); // } else { From 901f31185613d46d2ac3527b9a24f59786fca43f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 18 Jul 2023 23:23:16 +0300 Subject: [PATCH 004/211] vfs: auto-implement Vnode using macros --- lib/vfs/Cargo.toml | 1 + lib/vfs/macros/Cargo.toml | 16 +++++++++ lib/vfs/macros/src/lib.rs | 53 ++++++++++++++++++++++++++++ lib/vfs/macros/src/vnode_impl.rs | 60 ++++++++++++++++++++++++++++++++ lib/vfs/src/char.rs | 21 +++-------- src/syscall.rs | 2 +- 6 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 lib/vfs/macros/Cargo.toml create mode 100644 lib/vfs/macros/src/lib.rs create mode 100644 lib/vfs/macros/src/vnode_impl.rs diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 42c6b09a..733a15cb 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +macros = { path = "macros" } bitflags = "2.3.3" diff --git a/lib/vfs/macros/Cargo.toml b/lib/vfs/macros/Cargo.toml new file mode 100644 index 00000000..721b63a9 --- /dev/null +++ b/lib/vfs/macros/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "macros" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +proc-macro = true + +[dependencies] +lazy_static = "1.4.0" +proc-macro-crate = "1.3.1" +proc-macro2 = "1.0.66" +quote = "1.0.31" +syn = { version = "2.0.26", features = ["full"] } diff --git a/lib/vfs/macros/src/lib.rs b/lib/vfs/macros/src/lib.rs new file mode 100644 index 00000000..b011e3d2 --- /dev/null +++ b/lib/vfs/macros/src/lib.rs @@ -0,0 +1,53 @@ +use std::collections::HashSet; + +use proc_macro::TokenStream; +use proc_macro2::Ident; +use proc_macro_crate::{crate_name, FoundCrate}; +use quote::{quote, ToTokens}; +use syn::{parse_macro_input, ImplItem, ItemImpl}; + +mod vnode_impl; +use vnode_impl::IMPLS; + +#[proc_macro_attribute] +pub fn auto_vnode_impl(attr: TokenStream, input: TokenStream) -> TokenStream { + let current_crate = crate_name("vfs").unwrap(); + + let vfs_crate = match current_crate { + FoundCrate::Itself => quote!(crate), + FoundCrate::Name(name) => quote!( #name ), + }; + let mut impl_item = parse_macro_input!(input as ItemImpl); + let behavior = if attr.is_empty() { + "unimplemented".to_string() + } else { + parse_macro_input!(attr as Ident).to_string() + }; + let behavior = match behavior.as_str() { + "unimplemented" => quote! { unimplemented!() }, + "panic" => quote! { panic!() }, + "error" => quote! { Err(yggdrasil_abi::error::Error::NotImplemented) }, + _ => panic!("Unknown #[auto_vnode_impl] behavior: {:?}", behavior), + }; + + let mut missing_impls: HashSet<&str> = HashSet::from_iter(IMPLS.keys().copied()); + + for item in &impl_item.items { + match item { + ImplItem::Fn(f) => { + let name = &f.sig.ident.to_string(); + missing_impls.remove(name.as_str()); + } + _ => panic!("Unexpected item in Vnode impl"), + } + } + + for &item in &missing_impls { + let f = IMPLS.get(item).unwrap(); + let fn_impl = f(&vfs_crate, &behavior); + + impl_item.items.push(ImplItem::Verbatim(fn_impl)); + } + + impl_item.to_token_stream().into() +} diff --git a/lib/vfs/macros/src/vnode_impl.rs b/lib/vfs/macros/src/vnode_impl.rs new file mode 100644 index 00000000..939d902a --- /dev/null +++ b/lib/vfs/macros/src/vnode_impl.rs @@ -0,0 +1,60 @@ +use std::collections::HashMap; + +use proc_macro2::TokenStream as TS2; +use quote::quote; + +fn impl_open(vfs: &TS2, behavior: &TS2) -> TS2 { + quote! { + fn open(&mut self, _node: &#vfs::VnodeRef, _opts: yggdrasil_abi::io::OpenFlags) + -> Result { + #behavior + } + } +} + +fn impl_close(vfs: &TS2, behavior: &TS2) -> TS2 { + quote! { + fn close(&mut self, _node: &#vfs::VnodeRef) -> Result<(), Error> { + #behavior + } + } +} + +fn impl_create(vfs: &TS2, behavior: &TS2) -> TS2 { + quote! { + fn create(&mut self, _at: &#vfs::VnodeRef, _name: &str, _kind: #vfs::VnodeKind) + -> Result<#vfs::VnodeRef, yggdrasil_abi::error::Error> { + #behavior + } + } +} + +fn impl_write(vfs: &TS2, behavior: &TS2) -> TS2 { + quote! { + fn write(&mut self, _node: &#vfs::VnodeRef, _pos: usize, _data: &[u8]) + -> Result { + #behavior + } + } +} + +fn impl_read(vfs: &TS2, behavior: &TS2) -> TS2 { + quote! { + fn read(&mut self, _node: &#vfs::VnodeRef, _pos: usize, _data: &mut [u8]) + -> Result { + #behavior + } + } +} + +lazy_static::lazy_static! { + pub static ref IMPLS: HashMap<&'static str, fn(&TS2, &TS2) -> TS2> = { + let mut m = HashMap::new(); + m.insert("open".into(), impl_open as _); + m.insert("close".into(), impl_close as _); + m.insert("create".into(), impl_create as _); + m.insert("write".into(), impl_write as _); + m.insert("read".into(), impl_read as _); + m + }; +} diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index 848869f0..91351f7e 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -1,6 +1,7 @@ -use yggdrasil_abi::error::Error; +use macros::auto_vnode_impl; +use yggdrasil_abi::{error::Error, io::OpenFlags}; -use crate::node::{VnodeImpl, VnodeRef}; +use crate::{node::VnodeImpl, VnodeRef}; pub trait CharDevice { fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result; @@ -17,12 +18,9 @@ impl CharDeviceWrapper { } } +#[auto_vnode_impl(error)] impl VnodeImpl for CharDeviceWrapper { - fn open( - &mut self, - _node: &VnodeRef, - _opts: yggdrasil_abi::io::OpenFlags, - ) -> Result { + fn open(&mut self, _node: &VnodeRef, _opts: OpenFlags) -> Result { Ok(0) } @@ -37,13 +35,4 @@ impl VnodeImpl for CharDeviceWrapper { fn write(&mut self, _node: &VnodeRef, _pos: usize, data: &[u8]) -> Result { self.device.write(true, data) } - - fn create( - &mut self, - _at: &VnodeRef, - _name: &str, - _kind: crate::node::VnodeKind, - ) -> Result { - todo!() - } } diff --git a/src/syscall.rs b/src/syscall.rs index 2359a2e5..1596b592 100644 --- a/src/syscall.rs +++ b/src/syscall.rs @@ -4,7 +4,7 @@ use core::time::Duration; use abi::{ error::{Error, IntoSyscallResult}, io::{OpenFlags, RawFd}, - SyscallFunction, + syscall::SyscallFunction, }; use vfs::{Read, Write}; From aa6b2ac46957b59d60974a481c11c19512ea3e97 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 19 Jul 2023 21:58:59 +0300 Subject: [PATCH 005/211] proc: add simple initrd filesystem --- Cargo.toml | 3 +- lib/vfs/Cargo.toml | 2 +- lib/vfs/macros/Cargo.toml | 2 +- lib/vfs/macros/src/lib.rs | 8 +- lib/vfs/macros/src/vnode_impl.rs | 6 +- lib/vfs/src/char.rs | 8 +- lib/vfs/src/file.rs | 35 +++- lib/vfs/src/lib.rs | 37 +++- lib/vfs/src/node.rs | 80 +++++++- src/arch/aarch64/devtree.rs | 14 +- src/arch/aarch64/mod.rs | 51 ++++- src/debug.rs | 20 ++ src/fs/mod.rs | 11 + src/fs/tar.rs | 332 +++++++++++++++++++++++++++++++ src/main.rs | 54 +++-- src/proc/exec.rs | 36 +++- src/proc/mod.rs | 165 ++++++++++++++- src/util.rs | 8 + 18 files changed, 818 insertions(+), 54 deletions(-) create mode 100644 src/fs/tar.rs diff --git a/Cargo.toml b/Cargo.toml index a45603cc..e5f0a7ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } +vfs-macros = { path = "lib/vfs/macros" } aarch64-cpu = "9.3.1" atomic_enum = "0.2.0" @@ -18,4 +19,4 @@ spinning_top = "0.2.5" static_assertions = "1.1.0" tock-registers = "0.8.1" -elf = { version = "0.7.2", default-features = false } +elf = { version = "0.7.2", path = "../../rust-elf", default-features = false, features = ["no_std_stream"] } diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 733a15cb..0802b412 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -macros = { path = "macros" } +vfs-macros = { path = "macros" } bitflags = "2.3.3" diff --git a/lib/vfs/macros/Cargo.toml b/lib/vfs/macros/Cargo.toml index 721b63a9..7959b357 100644 --- a/lib/vfs/macros/Cargo.toml +++ b/lib/vfs/macros/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "macros" +name = "vfs-macros" version = "0.1.0" edition = "2021" diff --git a/lib/vfs/macros/src/lib.rs b/lib/vfs/macros/src/lib.rs index b011e3d2..59296dee 100644 --- a/lib/vfs/macros/src/lib.rs +++ b/lib/vfs/macros/src/lib.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use proc_macro::TokenStream; -use proc_macro2::Ident; +use proc_macro2::{Ident, Span}; use proc_macro_crate::{crate_name, FoundCrate}; use quote::{quote, ToTokens}; use syn::{parse_macro_input, ImplItem, ItemImpl}; @@ -14,9 +14,11 @@ pub fn auto_vnode_impl(attr: TokenStream, input: TokenStream) -> TokenStream { let current_crate = crate_name("vfs").unwrap(); let vfs_crate = match current_crate { - FoundCrate::Itself => quote!(crate), - FoundCrate::Name(name) => quote!( #name ), + FoundCrate::Itself => Ident::new("crate", Span::call_site()), + FoundCrate::Name(name) => Ident::new(&name, Span::call_site()), }; + let vfs_crate = quote! { #vfs_crate }; + let mut impl_item = parse_macro_input!(input as ItemImpl); let behavior = if attr.is_empty() { "unimplemented".to_string() diff --git a/lib/vfs/macros/src/vnode_impl.rs b/lib/vfs/macros/src/vnode_impl.rs index 939d902a..91126cfc 100644 --- a/lib/vfs/macros/src/vnode_impl.rs +++ b/lib/vfs/macros/src/vnode_impl.rs @@ -6,7 +6,7 @@ use quote::quote; fn impl_open(vfs: &TS2, behavior: &TS2) -> TS2 { quote! { fn open(&mut self, _node: &#vfs::VnodeRef, _opts: yggdrasil_abi::io::OpenFlags) - -> Result { + -> Result { #behavior } } @@ -31,7 +31,7 @@ fn impl_create(vfs: &TS2, behavior: &TS2) -> TS2 { fn impl_write(vfs: &TS2, behavior: &TS2) -> TS2 { quote! { - fn write(&mut self, _node: &#vfs::VnodeRef, _pos: usize, _data: &[u8]) + fn write(&mut self, _node: &#vfs::VnodeRef, _pos: u64, _data: &[u8]) -> Result { #behavior } @@ -40,7 +40,7 @@ fn impl_write(vfs: &TS2, behavior: &TS2) -> TS2 { fn impl_read(vfs: &TS2, behavior: &TS2) -> TS2 { quote! { - fn read(&mut self, _node: &#vfs::VnodeRef, _pos: usize, _data: &mut [u8]) + fn read(&mut self, _node: &#vfs::VnodeRef, _pos: u64, _data: &mut [u8]) -> Result { #behavior } diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index 91351f7e..fe3aa635 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -1,4 +1,4 @@ -use macros::auto_vnode_impl; +use vfs_macros::auto_vnode_impl; use yggdrasil_abi::{error::Error, io::OpenFlags}; use crate::{node::VnodeImpl, VnodeRef}; @@ -20,7 +20,7 @@ impl CharDeviceWrapper { #[auto_vnode_impl(error)] impl VnodeImpl for CharDeviceWrapper { - fn open(&mut self, _node: &VnodeRef, _opts: OpenFlags) -> Result { + fn open(&mut self, _node: &VnodeRef, _opts: OpenFlags) -> Result { Ok(0) } @@ -28,11 +28,11 @@ impl VnodeImpl for CharDeviceWrapper { Ok(()) } - fn read(&mut self, _node: &VnodeRef, _pos: usize, data: &mut [u8]) -> Result { + fn read(&mut self, _node: &VnodeRef, _pos: u64, data: &mut [u8]) -> Result { self.device.read(true, data) } - fn write(&mut self, _node: &VnodeRef, _pos: usize, data: &[u8]) -> Result { + fn write(&mut self, _node: &VnodeRef, _pos: u64, data: &[u8]) -> Result { self.device.write(true, data) } } diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs index e3116880..a7a8e9d5 100644 --- a/lib/vfs/src/file.rs +++ b/lib/vfs/src/file.rs @@ -6,7 +6,7 @@ use yggdrasil_abi::error::Error; use crate::{ node::{VnodeKind, VnodeRef}, - Read, Write, + Read, Seek, SeekFrom, Write, }; bitflags! { @@ -20,7 +20,7 @@ pub type FileRef = Rc>; pub struct NormalFile { vnode: VnodeRef, - pos: usize, + pos: u64, } pub enum FileInner { @@ -33,7 +33,7 @@ pub struct File { } impl File { - pub fn normal(vnode: VnodeRef, pos: usize, flags: FileFlags) -> FileRef { + pub fn normal(vnode: VnodeRef, pos: u64, flags: FileFlags) -> FileRef { Rc::new(RefCell::new(Self { inner: FileInner::Normal(NormalFile { vnode, pos }), flags, @@ -51,7 +51,7 @@ impl Write for File { FileInner::Normal(inner) => { let count = inner.vnode.write(inner.pos, data)?; if inner.vnode.kind() != VnodeKind::Char { - inner.pos += count; + inner.pos += count as u64; } Ok(count) } @@ -69,7 +69,7 @@ impl Read for File { FileInner::Normal(inner) => { let count = inner.vnode.read(inner.pos, data)?; if inner.vnode.kind() != VnodeKind::Char { - inner.pos += count; + inner.pos += count as u64; } Ok(count) } @@ -77,6 +77,31 @@ impl Read for File { } } +impl Seek for File { + fn seek(&mut self, pos: SeekFrom) -> Result { + match &mut self.inner { + FileInner::Normal(inner) => { + // TODO check if the file is actually seekable + + let size = inner.vnode.size()?; + let pos = match pos { + SeekFrom::Start(offset) => { + if offset > size { + todo!(); + } + offset + } + SeekFrom::End(0) => size, + _ => todo!(), + }; + inner.pos = pos; + + Ok(pos) + } + } + } +} + impl Drop for File { fn drop(&mut self) { match &mut self.inner { diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 4e005159..ce7a02f6 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -17,8 +17,16 @@ pub(crate) mod node; pub use self::block::BlockDevice; pub use self::char::{CharDevice, CharDeviceWrapper}; pub use file::{File, FileFlags, FileRef}; +pub use fs::Filesystem; pub use ioctx::IoContext; -pub use node::{Vnode, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; +pub use node::{Vnode, VnodeDump, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; + +#[derive(Debug)] +pub enum SeekFrom { + Start(u64), + End(i64), + Current(i64), +} pub trait Write { fn write(&mut self, data: &[u8]) -> Result; @@ -26,4 +34,31 @@ pub trait Write { pub trait Read { fn read(&mut self, data: &mut [u8]) -> Result; + + fn read_exact(&mut self, data: &mut [u8]) -> Result<(), Error> { + default_read_exact(self, data) + } +} + +pub trait Seek { + fn seek(&mut self, pos: SeekFrom) -> Result; +} + +fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Result<(), Error> { + while !buf.is_empty() { + match this.read(buf) { + Ok(0) => break, + Ok(n) => { + let tmp = buf; + buf = &mut tmp[n..]; + } + Err(e) => todo!("default_read_exact: {:?}", e), + } + } + + if !buf.is_empty() { + todo!("default_read_exact unexpected eof") + } else { + Ok(()) + } } diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index bb439309..8c695531 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -11,11 +11,18 @@ use alloc::{ }; use yggdrasil_abi::{error::Error, io::OpenFlags}; -use crate::file::{File, FileFlags, FileRef}; +use crate::{ + file::{File, FileFlags, FileRef}, + fs::Filesystem, +}; pub type VnodeRef = Rc; pub type VnodeWeak = Weak; +pub struct VnodeDump { + node: VnodeRef, +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum VnodeKind { Directory, @@ -34,16 +41,21 @@ pub struct Vnode { tree: RefCell, kind: VnodeKind, data: RefCell>>, + fs: RefCell>>, } pub trait VnodeImpl { fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result; - fn open(&mut self, node: &VnodeRef, opts: OpenFlags) -> Result; + fn open(&mut self, node: &VnodeRef, opts: OpenFlags) -> Result; fn close(&mut self, node: &VnodeRef) -> Result<(), Error>; - fn read(&mut self, node: &VnodeRef, pos: usize, data: &mut [u8]) -> Result; - fn write(&mut self, node: &VnodeRef, pos: usize, data: &[u8]) -> Result; + fn read(&mut self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result; + fn write(&mut self, node: &VnodeRef, pos: u64, data: &[u8]) -> Result; + + fn size(&mut self, _node: &VnodeRef) -> Result { + unimplemented!() + } } impl Vnode { @@ -56,6 +68,7 @@ impl Vnode { }), kind, data: RefCell::new(None), + fs: RefCell::new(None), }) } @@ -74,6 +87,11 @@ impl Vnode { self.data.borrow_mut() } + #[inline] + pub fn fs(&self) -> Option> { + self.fs.borrow().clone() + } + pub fn parent(self: &VnodeRef) -> VnodeRef { match &self.tree.borrow().parent { Some(parent) => parent.upgrade().unwrap(), @@ -85,6 +103,10 @@ impl Vnode { self.data.borrow_mut().replace(data); } + pub fn set_fs(&self, data: Rc) { + self.fs.replace(Some(data)); + } + #[inline] pub fn is_directory(&self) -> bool { self.kind == VnodeKind::Directory @@ -186,7 +208,33 @@ impl Vnode { } } - pub fn write(self: &VnodeRef, pos: usize, buf: &[u8]) -> Result { + pub fn create(self: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + if self.kind != VnodeKind::Directory { + todo!(); + } + if name.contains('/') { + return Err(Error::InvalidArgument); + } + + match self.lookup_or_load(name) { + Err(Error::DoesNotExist) => {} + Ok(_) => return Err(Error::AlreadyExists), + e => return e, + }; + + if let Some(ref mut data) = *self.data() { + let vnode = data.create(self, name, kind)?; + if let Some(fs) = self.fs() { + vnode.set_fs(fs); + } + self.add_child(vnode.clone()); + Ok(vnode) + } else { + Err(Error::NotImplemented) + } + } + + pub fn write(self: &VnodeRef, pos: u64, buf: &[u8]) -> Result { if self.kind == VnodeKind::Directory { todo!(); } @@ -198,7 +246,7 @@ impl Vnode { } } - pub fn read(self: &VnodeRef, pos: usize, buf: &mut [u8]) -> Result { + pub fn read(self: &VnodeRef, pos: u64, buf: &mut [u8]) -> Result { if self.kind == VnodeKind::Directory { todo!(); } @@ -209,6 +257,14 @@ impl Vnode { todo!() } } + + pub fn size(self: &VnodeRef) -> Result { + if let Some(ref mut data) = *self.data() { + data.size(self) + } else { + todo!(); + } + } } impl fmt::Debug for Vnode { @@ -223,3 +279,15 @@ impl fmt::Debug for Vnode { write!(f, "[{} {}]", prefix, self.name) } } + +impl VnodeDump { + pub fn new(node: VnodeRef) -> Self { + Self { node } + } +} + +impl fmt::Debug for VnodeDump { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.node.dump(f, 0) + } +} diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index f18ad736..3f39a8d0 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -126,7 +126,7 @@ fn find_node<'a>(at: TNode<'a>, path: &str) -> Option> { fn dump_node(node: &TNode, depth: usize, level: LogLevel) { fn indent(level: LogLevel, depth: usize) { for _ in 0..depth { - log_print!(level, " "); + log_print_raw!(level, " "); } } @@ -138,18 +138,18 @@ fn dump_node(node: &TNode, depth: usize, level: LogLevel) { } indent(level, depth); - log_print!(level, "{:?} {{\n", node_name); + log_print_raw!(level, "{:?} {{\n", node_name); for prop in node.props() { indent(level, depth + 1); let name = prop.name().unwrap(); - log_print!(level, "{name:?} = "); + log_print_raw!(level, "{name:?} = "); match name { - "compatible" | "stdout-path" => log_print!(level, "{:?}", prop.str().unwrap()), - _ => log_print!(level, "{:x?}", prop.raw()), + "compatible" | "stdout-path" => log_print_raw!(level, "{:?}", prop.str().unwrap()), + _ => log_print_raw!(level, "{:x?}", prop.raw()), } - log_print!(level, "\n"); + log_print_raw!(level, "\n"); } for child in node.children() { @@ -157,5 +157,5 @@ fn dump_node(node: &TNode, depth: usize, level: LogLevel) { } indent(level, depth); - log_print!(level, "}}\n"); + log_print_raw!(level, "}}\n"); } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 9bfe55b7..13feb4f1 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -4,6 +4,7 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1}; use abi::error::Error; +use fdt_rs::prelude::PropReader; use plat_qemu::PLATFORM; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; @@ -15,7 +16,7 @@ use crate::{ }, debug, device::platform::Platform, - fs::devfs, + fs::{devfs, Initrd, INITRD_DATA}, mem::{ heap, phys::{self, reserved::reserve_region, PageUsage, PhysicalMemoryRegion}, @@ -135,6 +136,16 @@ impl AArch64 { unsafe fn init_physical_memory(&self, dtb_phys: usize) -> Result<(), Error> { let dt = self.device_tree(); + if let Some(initrd) = INITRD_DATA.try_get() { + reserve_region( + "initrd", + PhysicalMemoryRegion { + base: initrd.phys_page_start, + size: initrd.phys_page_len, + }, + ); + } + reserve_region( "dtb", PhysicalMemoryRegion { @@ -148,6 +159,41 @@ impl AArch64 { } } +fn setup_initrd() { + let dt = ARCHITECTURE.device_tree(); + let Some(chosen) = dt.node_by_path("/chosen") else { + return; + }; + + let Some(initrd_start) = devtree::find_prop(&chosen, "linux,initrd-start") else { + return; + }; + let Some(initrd_end) = devtree::find_prop(&chosen, "linux,initrd-end") else { + return; + }; + + let initrd_start = initrd_start.u64(0).unwrap() as usize; + let initrd_end = initrd_end.u64(0).unwrap() as usize; + + let start_aligned = initrd_start & !0xFFF; + let end_aligned = initrd_end & !0xFFF; + + let data = unsafe { + core::slice::from_raw_parts( + initrd_start.virtualize() as *const _, + initrd_end - initrd_start, + ) + }; + + let initrd = Initrd { + phys_page_start: start_aligned, + phys_page_len: end_aligned - start_aligned, + data, + }; + + INITRD_DATA.init(initrd); +} + /// AArch64 kernel main entry point pub fn kernel_main(dtb_phys: usize) -> ! { // NOTE it is critical that the code does not panic until the debug is set up, otherwise no @@ -167,6 +213,9 @@ pub fn kernel_main(dtb_phys: usize) -> ! { exception::init_exceptions(); + // Setup initrd + setup_initrd(); + debugln!("Initializing {} platform", PLATFORM.name()); unsafe { ARCHITECTURE diff --git a/src/debug.rs b/src/debug.rs index 456c315a..67ed93ea 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -99,6 +99,26 @@ impl fmt::Write for DebugPrinter { } } +pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { + for (i, b) in data.iter().enumerate() { + if i % 16 == 0 { + log_print_raw!(level, "{:X}: ", addr_offset + i) + } + + log_print_raw!(level, "{:02X}", data[i]); + + if i % 16 == 15 { + log_print_raw!(level, "\n"); + } else if i % 2 == 1 { + log_print_raw!(level, " "); + } + } + + if data.len() % 16 != 0 { + log_print_raw!(level, "\n"); + } +} + /// Initializes the debug logging faclities. /// /// # Panics diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 65c874be..5998278b 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,3 +1,14 @@ //! Filesystem implementations +use crate::util::OneTimeInit; + +pub struct Initrd { + pub phys_page_start: usize, + pub phys_page_len: usize, + pub data: &'static [u8], +} + +pub static INITRD_DATA: OneTimeInit = OneTimeInit::new(); + pub mod devfs; +pub mod tar; diff --git a/src/fs/tar.rs b/src/fs/tar.rs new file mode 100644 index 00000000..92ed38ee --- /dev/null +++ b/src/fs/tar.rs @@ -0,0 +1,332 @@ +use abi::error::Error; +use alloc::{boxed::Box, rc::Rc}; +use vfs::{Filesystem, Vnode, VnodeImpl, VnodeKind, VnodeRef}; +use vfs_macros::auto_vnode_impl; +use yggdrasil_abi::io::OpenFlags; + +use crate::util::OneTimeInit; + +#[repr(C)] +pub struct OctalField { + data: [u8; N], +} + +#[repr(C)] +pub struct TarString { + data: [u8; N], +} + +pub struct TarIterator<'a> { + data: &'a [u8], + offset: usize, + zero_blocks: usize, +} + +#[repr(packed)] +pub struct TarEntry { + name: TarString<100>, + mode: OctalField<8>, + uid: OctalField<8>, + gid: OctalField<8>, + size: OctalField<12>, + mtime: OctalField<12>, + checksum: OctalField<8>, + type_: u8, + link_name: [u8; 100], + magic: [u8; 8], + user: [u8; 32], + group: [u8; 32], + dev_major: OctalField<8>, + dev_minor: OctalField<8>, + prefix: [u8; 155], + __pad: [u8; 12], +} + +impl<'a> TarIterator<'a> { + pub const fn new(data: &'a [u8]) -> Self { + Self { + data, + offset: 0, + zero_blocks: 0, + } + } +} + +impl<'a> Iterator for TarIterator<'a> { + type Item = Result<(&'a TarEntry, Option<&'a [u8]>), Error>; + + fn next(&mut self) -> Option { + loop { + if self.offset + 512 > self.data.len() { + break None; + } + + let hdr_ptr = &self.data[self.offset..]; + let hdr = unsafe { core::mem::transmute::<*const u8, &TarEntry>(hdr_ptr.as_ptr()) }; + + if hdr.is_empty() { + if self.zero_blocks == 1 { + self.offset = self.data.len(); + return None; + } + self.zero_blocks += 1; + continue; + } + + let (data, size_aligned) = match hdr.type_ { + 0 | b'0' => { + let size = usize::from(&hdr.size); + + if self.offset + 512 + size > self.data.len() { + return Some(Err(Error::InvalidArgument)); + } + + let data = &self.data[self.offset + 512..self.offset + 512 + size]; + let size_aligned = (size + 511) & !511; + + (Some(data), size_aligned) + } + // Directory + b'5' => (None, 0), + _ => todo!("Unknown node kind: {}", hdr.type_), + }; + self.offset += size_aligned + 512; + + break Some(Ok((hdr, data))); + } + } +} + +impl From<&OctalField> for usize { + fn from(value: &OctalField) -> Self { + let mut acc = 0; + for i in 0..N { + if value.data[i] == 0 { + break; + } + acc <<= 3; + acc |= (value.data[i] - b'0') as usize; + } + acc + } +} + +impl TarString { + pub fn as_str(&self) -> Result<&str, Error> { + core::str::from_utf8(&self.data[..self.len()]).map_err(|_| Error::InvalidArgument) + } + + pub fn len(&self) -> usize { + for i in 0..N { + if self.data[i] == 0 { + return i; + } + } + N + } +} + +impl TarEntry { + pub fn is_empty(&self) -> bool { + self.name.data[0] == 0 + } + + pub fn node_kind(&self) -> VnodeKind { + match self.type_ { + 0 | b'0' => VnodeKind::Regular, + b'5' => VnodeKind::Directory, + _ => todo!(), + } + } +} + +pub struct TarFilesystem { + root: OneTimeInit, +} + +impl Filesystem for TarFilesystem { + fn dev(self: Rc) -> Option<&'static dyn vfs::BlockDevice> { + todo!() + } + + fn root(self: Rc) -> Result { + self.root.try_get().cloned().ok_or(Error::DoesNotExist) + } + + fn data(&self) -> Option> { + todo!() + } +} + +struct DirInode; +struct RegularInode { + data: &'static [u8], +} + +#[auto_vnode_impl] +impl VnodeImpl for DirInode { + fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + let child = Vnode::new(name, kind); + match kind { + VnodeKind::Directory => child.set_data(Box::new(DirInode)), + VnodeKind::Regular => (), + _ => todo!(), + } + Ok(child) + } +} + +#[auto_vnode_impl] +impl VnodeImpl for RegularInode { + fn open(&mut self, node: &VnodeRef, opts: OpenFlags) -> Result { + if opts.is_write() { + panic!("TODO: tarfs write"); + } + + Ok(0) + } + + fn close(&mut self, node: &VnodeRef) -> Result<(), Error> { + Ok(()) + } + + fn read(&mut self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + let pos = pos as usize; + if pos > self.data.len() { + return Err(Error::InvalidFile); + } + let mut rem = core::cmp::min(self.data.len() - pos, data.len()); + data[..rem].copy_from_slice(&self.data[pos..pos + rem]); + Ok(rem) + } + + fn size(&mut self, _node: &VnodeRef) -> Result { + Ok(self.data.len() as u64) + } +} + +impl TarFilesystem { + fn make_path( + self: &Rc, + at: &VnodeRef, + path: &str, + create: bool, + ) -> Result { + debugln!("make_path {:?}", path); + if path.is_empty() { + return Ok(at.clone()); + } + let (element, rest) = abi::path::split_left(path); + assert!(!element.is_empty()); + + let node = at.lookup(element); + let node = match node { + Some(node) => node, + None => { + if !create { + debugln!("path {:?} does not exist", path); + return Err(Error::DoesNotExist); + } + + infoln!("Create {:?}", element); + at.create(element, VnodeKind::Directory)? + } + }; + + if rest.is_empty() { + Ok(node) + } else { + self.make_path(&node, rest, create) + } + } + + fn create_node_initial( + self: &Rc, + name: &str, + hdr: &TarEntry, + data: Option<&[u8]>, + ) -> VnodeRef { + assert!(!name.is_empty()); + assert!(!name.contains('/')); + + let kind = hdr.node_kind(); + let node = Vnode::new(name, kind); + node.set_fs(self.clone()); + + match kind { + VnodeKind::Directory => node.set_data(Box::new(DirInode)), + VnodeKind::Regular => {} + _ => todo!(), + } + + node + } + + fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { + let root = Vnode::new("", VnodeKind::Directory); + root.set_fs(self.clone()); + root.set_data(Box::new(DirInode)); + + // 1. Create paths in tar + for item in TarIterator::new(tar_data) { + let Ok((hdr, data)) = item else { + warnln!("Tar image is truncated"); + return Err(Error::InvalidArgument); + }; + + let path = hdr.name.as_str()?.trim_matches('/'); + infoln!("path = {:?}", path); + let (dirname, filename) = abi::path::split_right(path); + let parent = self.make_path(&root, dirname, true)?; + let node = self.create_node_initial(filename, hdr, data); + + parent.add_child(node); + } + + // 2. Associate files with their data + for item in TarIterator::new(tar_data) { + let Ok((hdr, data)) = item else { + panic!("Unreachable"); + }; + if hdr.node_kind() == VnodeKind::Regular { + let data = data.unwrap(); + let path = hdr.name.as_str()?.trim_matches('/'); + let node = self.make_path(&root, path, false)?; + node.set_data(Box::new(RegularInode { data })); + } + } + + Ok(root) + } + + pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { + let fs = Rc::new(TarFilesystem { + root: OneTimeInit::new(), + }); + let root = fs.from_slice_internal(tar_data)?; + fs.root.init(root); + + Ok(fs) + } +} + +// pub fn init() { +// let Some(initrd) = INITRD_DATA.try_get() else { +// warnln!("No initrd found"); +// return; +// }; +// +// let fs = match TarFilesystem::from_slice(initrd.data) { +// Ok(fs) => fs, +// Err(err) => { +// warnln!("Could not initialize tar filesystem: {:?}", err); +// return; +// } +// }; +// +// let r = fs.root().unwrap(); +// let dump = VnodeDump::new(r); +// infoln!("{:?}", dump); +// +// todo!() +// } diff --git a/src/main.rs b/src/main.rs index db79e719..f966da6b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,15 @@ extern crate yggdrasil_abi as abi; +use core::ops::DerefMut; + +use abi::{ + error::Error, + io::{OpenFlags, RawFd}, +}; +use fs::{devfs, tar::TarFilesystem, INITRD_DATA}; use task::process::Process; +use vfs::{Filesystem, IoContext, Read, VnodeRef}; extern crate alloc; @@ -34,6 +42,12 @@ pub mod syscall; pub mod task; pub mod util; +fn setup_root() -> Result { + let initrd_data = INITRD_DATA.get(); + let fs = TarFilesystem::from_slice(initrd_data.data)?; + fs.root() +} + /// Entry point for common kernel code. /// /// # Note @@ -41,15 +55,40 @@ pub mod util; /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kernel_main() { + let root = match setup_root() { + Ok(root) => root, + Err(err) => { + warnln!("Could not setup root from initrd: {:?}", err); + return; + } + }; + + let devfs_root = devfs::root(); + let tty_node = devfs_root.lookup("ttyS0").unwrap(); + + let ioctx = IoContext::new(root); + let node = ioctx.find(None, "/init", false).unwrap(); + let file = node.open(OpenFlags::new().read()).unwrap(); + + { + let user_init = proc::exec::load_elf(file, &["/init"]).unwrap(); + let mut io = user_init.io.lock(); + io.set_ioctx(ioctx); + let stdout = tty_node.open(OpenFlags::new().write()).unwrap(); + let stderr = stdout.clone(); + + io.set_file(RawFd::STDOUT, stdout).unwrap(); + io.set_file(RawFd::STDERR, stderr).unwrap(); + drop(io); + user_init.enqueue_somewhere(); + } + // static USER_PROGRAM: &[u8] = include_bytes!(concat!( // "../../target/aarch64-unknown-yggdrasil/", // env!("PROFILE"), // "/test_program" // )); - // let devfs_root = devfs::root(); - // let tty_node = devfs_root.lookup("ttyS0").unwrap(); - // let ioctx = IoContext::new(devfs_root.clone()); // // Spawn a test user task @@ -61,15 +100,6 @@ pub fn kernel_main() { // // Setup I/O for the process // // let mut io = proc.io.lock(); // // io.set_file(RawFd::STDOUT, todo!()).unwrap(); - // { - // let mut io = proc.io.lock(); - // io.set_ioctx(ioctx); - // let stdout = tty_node.open(OpenFlags::new().write()).unwrap(); - // let stderr = stdout.clone(); - - // io.set_file(RawFd::STDOUT, stdout).unwrap(); - // io.set_file(RawFd::STDERR, stderr).unwrap(); - // } // proc.enqueue_somewhere(); // } diff --git a/src/proc/exec.rs b/src/proc/exec.rs index b265304c..9c2d33fb 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -3,6 +3,7 @@ use core::mem::size_of; use abi::error::Error; use alloc::rc::Rc; +use vfs::{FileRef, Read, Seek}; use crate::{ arch::aarch64::context::TaskContext, @@ -66,13 +67,13 @@ fn setup_args(space: &mut AddressSpace, virt: usize, args: &[&str]) -> Result<() Ok(()) } -/// Sets up a userspace structure from a slice defining an ELF binary -pub fn create_from_memory(data: &[u8], args: &[&str]) -> Result, Error> { +fn setup_binary( + mut space: AddressSpace, + entry: usize, + args: &[&str], +) -> Result, Error> { const USER_STACK_PAGES: usize = 8; - let mut space = AddressSpace::new_empty()?; - let elf_entry = proc::load_elf_from_memory(&mut space, data); - let virt_stack_base = 0x10000000; // 0x1000 of guard page let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; @@ -88,10 +89,16 @@ pub fn create_from_memory(data: &[u8], args: &[&str]) -> Result, Err setup_args(&mut space, virt_args_base, args)?; - debugln!("Entry: {:#x}", elf_entry); + debugln!( + "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", + entry, + virt_stack_base, + virt_stack_base + USER_STACK_PAGES * 0x1000, + virt_args_base + ); let context = TaskContext::user( - elf_entry, + entry, virt_args_base, space.physical_address(), virt_stack_base + USER_STACK_PAGES * 0x1000, @@ -99,3 +106,18 @@ pub fn create_from_memory(data: &[u8], args: &[&str]) -> Result, Err Ok(Process::new_with_context(Some(space), context)) } + +pub fn load_elf(file: FileRef, args: &[&str]) -> Result, Error> { + let mut space = AddressSpace::new_empty()?; + let elf_entry = proc::load_elf_from_file(&mut space, file)?; + + setup_binary(space, elf_entry, args) +} + +/// Sets up a userspace structure from a slice defining an ELF binary +pub fn create_from_memory(data: &[u8], args: &[&str]) -> Result, Error> { + let mut space = AddressSpace::new_empty()?; + let elf_entry = proc::load_elf_from_memory(&mut space, data); + + setup_binary(space, elf_entry, args) +} diff --git a/src/proc/mod.rs b/src/proc/mod.rs index e61e6f8f..d4691f24 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -4,22 +4,76 @@ pub mod exec; pub mod io; pub mod wait; -use aarch64_cpu::registers::TTBR0_EL1; +use aarch64_cpu::registers::{TCR_EL1::A1::TTBR0, TTBR0_EL1}; +use alloc::rc::Rc; use elf::{ abi::{PF_W, PF_X, PT_LOAD}, endian::AnyEndian, - ElfBytes, + ElfBytes, ElfStream, ParseError, }; use tock_registers::interfaces::Writeable; +use vfs::{FileRef, Read, Seek, SeekFrom}; +use yggdrasil_abi::error::Error; use crate::{ arch::aarch64::table::tlb_flush_vaae1, + debug::hex_dump, mem::{ phys::{self, PageUsage}, table::{AddressSpace, PageAttributes}, + ConvertAddress, }, }; +#[derive(Clone, Copy)] +pub struct FileReader<'a> { + file: &'a FileRef, +} + +impl elf::io_traits::Read for FileReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> Result { + todo!() + } + + fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), elf::io_traits::StreamError> { + self.file + .borrow_mut() + .read_exact(buf) + .map_err(conv_stream_error) + } +} + +impl elf::io_traits::Seek for FileReader<'_> { + fn seek(&mut self, pos: elf::io_traits::SeekFrom) -> Result { + self.file + .borrow_mut() + .seek(conv_seek_from(pos)) + .map_err(conv_stream_error) + } +} + +#[inline] +fn conv_stream_error(v: Error) -> elf::io_traits::StreamError { + elf::io_traits::StreamError { + message: "Elf read error", + } +} + +#[inline] +fn conv_seek_from(v: elf::io_traits::SeekFrom) -> SeekFrom { + match v { + elf::io_traits::SeekFrom::End(off) => SeekFrom::End(off), + elf::io_traits::SeekFrom::Start(off) => SeekFrom::Start(off), + _ => todo!(), + } +} + +#[inline] +fn from_parse_error(v: ParseError) -> Error { + warnln!("ELF loading error: {:?}", v); + Error::InvalidFile +} + fn load_segment(space: &mut AddressSpace, addr: usize, data: &[u8], memsz: usize, elf_attrs: u32) { let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { (0, 0) => PageAttributes::AP_BOTH_READONLY, @@ -62,6 +116,113 @@ fn load_segment(space: &mut AddressSpace, addr: usize, data: &[u8], memsz: usize } } +fn load_bytes( + space: &mut AddressSpace, + addr: usize, + mut src: F, + len: usize, + elf_attrs: u32, +) -> Result<(), Error> +where + F: FnMut(usize, &mut [u8]) -> Result<(), Error>, +{ + // TODO check for crazy addresses here + + let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { + (0, 0) => PageAttributes::AP_BOTH_READONLY, + (_, 0) => PageAttributes::AP_BOTH_READWRITE, + (0, _) => PageAttributes::AP_BOTH_READONLY, + (_, _) => PageAttributes::AP_BOTH_READWRITE, + }; + + let dst_page_off = addr & 0xFFF; + let dst_page_aligned = addr & !0xFFF; + let mut off = 0usize; + let mut rem = len; + + while rem != 0 { + let page_idx = (dst_page_off + off) / 0x1000; + let page_off = (dst_page_off + off) % 0x1000; + let count = core::cmp::min(rem, 0x1000 - page_off); + + let virt_page = dst_page_aligned + page_idx * 0x1000; + assert_eq!(virt_page & 0xFFF, 0); + if let Some(_) = space.translate(virt_page) { + // Handle these cases + todo!(); + } + + let phys_page = phys::alloc_page(PageUsage::Used)?; + debugln!("map {:#x} -> {:#x}", virt_page, phys_page); + space.map_page(virt_page, phys_page, attrs)?; + + let dst_slice = unsafe { + let addr = (phys_page + page_off).virtualize(); + + core::slice::from_raw_parts_mut(addr as *mut u8, count) + }; + + src(off, dst_slice)?; + debugln!("{:#x} (off = {}):", virt_page + page_off, page_off); + // hex_dump( + // crate::debug::LogLevel::Debug, + // virt_page + page_off, + // dst_slice, + // ); + + rem -= count; + off += count; + } + + Ok(()) +} + +pub fn load_elf_from_file(space: &mut AddressSpace, file: FileRef) -> Result { + let file = FileReader { file: &file }; + + let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; + + for phdr in elf.segments() { + if phdr.p_type != PT_LOAD { + continue; + } + + debugln!("LOAD {:#x}", phdr.p_vaddr); + + if phdr.p_filesz > 0 { + load_bytes( + space, + phdr.p_vaddr as usize, + |off, dst| { + let mut source = file.file.borrow_mut(); + source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + source.read_exact(dst) + }, + phdr.p_filesz as usize, + phdr.p_flags, + )?; + } + + if phdr.p_memsz > phdr.p_filesz { + let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; + let len = (phdr.p_memsz - phdr.p_filesz) as usize; + + load_bytes( + space, + addr, + |_, dst| { + dst.fill(0); + Ok(()) + }, + len, + phdr.p_flags, + )?; + } + } + + Ok(elf.ehdr.e_entry as usize) +} + /// Loads an ELF image into the address space from a slice pub fn load_elf_from_memory(space: &mut AddressSpace, src: &[u8]) -> usize { // Map the address space temporarily diff --git a/src/util.rs b/src/util.rs index f05a9c59..cda37d61 100644 --- a/src/util.rs +++ b/src/util.rs @@ -69,6 +69,14 @@ impl OneTimeInit { unsafe { (*self.value.get()).assume_init_ref() } } + + pub fn try_get(&self) -> Option<&T> { + if self.state.load(Ordering::Acquire) { + Some(self.get()) + } else { + None + } + } } impl StaticVector { From 74cd9daed788411167cc880ac3a690688fcdcfa7 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 20 Jul 2023 11:59:53 +0300 Subject: [PATCH 006/211] proc: add mount/unmount system calls --- lib/vfs/src/debug.rs | 23 +++++++ lib/vfs/src/ioctx.rs | 101 ++++++++++++++++++++++------- lib/vfs/src/lib.rs | 7 ++ lib/vfs/src/node.rs | 79 ++++++++++++++++++++-- src/debug.rs | 5 ++ src/fs/mod.rs | 16 ++++- src/main.rs | 29 +-------- src/mem/phys/mod.rs | 4 +- src/syscall/arg.rs | 32 +++++++++ src/{syscall.rs => syscall/mod.rs} | 54 ++++++++++----- 10 files changed, 271 insertions(+), 79 deletions(-) create mode 100644 lib/vfs/src/debug.rs create mode 100644 src/syscall/arg.rs rename src/{syscall.rs => syscall/mod.rs} (76%) diff --git a/lib/vfs/src/debug.rs b/lib/vfs/src/debug.rs new file mode 100644 index 00000000..677c9f9e --- /dev/null +++ b/lib/vfs/src/debug.rs @@ -0,0 +1,23 @@ +use core::fmt; + +pub(crate) static mut DEBUG_HOOK: Option<&'static dyn Fn(fmt::Arguments)> = None; + +#[macro_export] +macro_rules! debug_raw { + ($($arg:tt)+) => { + $crate::debug::_debug_print(format_args!($($arg)+)) + }; +} + +#[macro_export] +macro_rules! debugln { + ($($arg:tt)+) => { + debug_raw!("[VFS] {}\n", format_args!($($arg)+)) + }; +} + +pub(crate) fn _debug_print(args: fmt::Arguments) { + if let Some(hook) = unsafe { DEBUG_HOOK.as_mut() } { + hook(args); + } +} diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index 007e302d..f47a4b94 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -15,7 +15,13 @@ impl IoContext { } } - fn _find(&self, mut at: VnodeRef, path: &str, follow: bool) -> Result { + fn _find( + &self, + mut at: VnodeRef, + path: &str, + follow: bool, + follow_mount: bool, + ) -> Result { let mut element; let mut rest = path; @@ -35,18 +41,44 @@ impl IoContext { } } - // TODO resolve link target + if follow || follow_mount { + while let Some(target) = at.target() { + assert!(at.is_mountpoint()); + if at.is_mountpoint() && !follow_mount { + break; + } + + debugln!("resolve parent: {:?} -> {:?}", at, target); + at = target; + } + } + + if !at.is_directory() { + return Err(Error::NotADirectory); + } if element.is_empty() && rest.is_empty() { return Ok(at); } - let node = at.lookup_or_load(element)?; + let mut node = at.lookup_or_load(element)?; + + if follow || follow_mount { + while let Some(target) = node.target() { + assert!(node.is_mountpoint()); + if node.is_mountpoint() && !follow_mount { + break; + } + + debugln!("resolve node: {:?} -> {:?}", node, target); + node = target; + } + } if rest.is_empty() { Ok(node) } else { - self._find(node, rest, follow) + self._find(node, rest, follow, follow_mount) } } @@ -55,7 +87,10 @@ impl IoContext { at: Option, mut path: &str, follow: bool, + follow_mount: bool, ) -> Result { + debugln!("_find {:?} in {:?}", path, at); + let at = if path.starts_with('/') { path = path.trim_start_matches('/'); self.root.clone() @@ -65,7 +100,7 @@ impl IoContext { self.cwd.clone() }; - self._find(at, path, follow) + self._find(at, path, follow, follow_mount) } pub fn open( @@ -74,7 +109,7 @@ impl IoContext { path: &str, opts: OpenFlags, ) -> Result { - let node = match self.find(at.clone(), path, true) { + let node = match self.find(at.clone(), path, true, true) { Err(Error::DoesNotExist) => { // TODO check for create option return Err(Error::DoesNotExist); @@ -84,11 +119,15 @@ impl IoContext { node.open(opts) } + + pub fn root(&self) -> &VnodeRef { + &self.root + } } #[cfg(test)] mod tests { - use abi::error::Error; + use yggdrasil_abi::error::Error; use crate::{node::VnodeRef, IoContext}; use std::fmt; @@ -115,7 +154,7 @@ mod tests { impl fmt::Debug for DumpNode<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.node.dump(f, 0) + self.node.dump(f, 0, true) } } @@ -137,79 +176,93 @@ mod tests { // Absolute lookups assert_eq!( - ctx.find(None, "/file1.txt", false).unwrap().name(), + ctx.find(None, "/file1.txt", false, false).unwrap().name(), "file1.txt" ); assert_eq!( - ctx.find(None, "/file3.txt", false).unwrap_err(), + ctx.find(None, "/file3.txt", false, false).unwrap_err(), Error::DoesNotExist ); assert_eq!( - ctx.find(None, "/dir1/file3.txt", false).unwrap().name(), + ctx.find(None, "/dir1/file3.txt", false, false) + .unwrap() + .name(), "file3.txt" ); // Non-absolute lookups from root assert_eq!( - ctx.find(None, "file1.txt", false).unwrap().name(), + ctx.find(None, "file1.txt", false, false).unwrap().name(), "file1.txt" ); assert_eq!( - ctx.find(None, "dir1/file3.txt", false).unwrap().name(), + ctx.find(None, "dir1/file3.txt", false, false) + .unwrap() + .name(), "file3.txt" ); // Absolute lookups from non-root - let cwd = ctx.find(None, "/dir1", false).unwrap(); + let cwd = ctx.find(None, "/dir1", false, false).unwrap(); assert_eq!( - ctx.find(Some(cwd.clone()), "/file1.txt", false) + ctx.find(Some(cwd.clone()), "/file1.txt", false, false) .unwrap() .name(), "file1.txt" ); assert_eq!( - ctx.find(Some(cwd.clone()), "/dir1/file3.txt", false) + ctx.find(Some(cwd.clone()), "/dir1/file3.txt", false, false) .unwrap() .name(), "file3.txt" ); assert_eq!( - ctx.find(Some(cwd.clone()), "/file3.txt", false) + ctx.find(Some(cwd.clone()), "/file3.txt", false, false) .unwrap_err(), Error::DoesNotExist ); assert_eq!( - ctx.find(Some(cwd.clone()), "/dir2", false).unwrap_err(), + ctx.find(Some(cwd.clone()), "/dir2", false, false) + .unwrap_err(), Error::DoesNotExist ); // Non-absolute lookups in non-root assert_eq!( - ctx.find(Some(cwd.clone()), "file3.txt", false) + ctx.find(Some(cwd.clone()), "file3.txt", false, false) .unwrap() .name(), "file3.txt" ); assert_eq!( - ctx.find(Some(cwd.clone()), "././file3.txt", false) + ctx.find(Some(cwd.clone()), "././file3.txt", false, false) .unwrap() .name(), "file3.txt" ); assert_eq!( - ctx.find(Some(cwd.clone()), "../dir1/file3.txt", false) + ctx.find(Some(cwd.clone()), "../dir1/file3.txt", false, false) .unwrap() .name(), "file3.txt" ); assert_eq!( - ctx.find(Some(cwd.clone()), ".", false).unwrap().name(), + ctx.find(Some(cwd.clone()), ".", false, false) + .unwrap() + .name(), "dir1" ); - assert_eq!(ctx.find(Some(cwd.clone()), "..", false).unwrap().name(), ""); assert_eq!( - ctx.find(Some(cwd.clone()), "../..", false).unwrap().name(), + ctx.find(Some(cwd.clone()), "..", false, false) + .unwrap() + .name(), + "" + ); + assert_eq!( + ctx.find(Some(cwd.clone()), "../..", false, false) + .unwrap() + .name(), "" ); } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index ce7a02f6..78f9b245 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -7,6 +7,9 @@ extern crate alloc; #[cfg(test)] extern crate std; +#[macro_use] +pub(crate) mod debug; + pub(crate) mod block; pub(crate) mod char; pub(crate) mod file; @@ -62,3 +65,7 @@ fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Res Ok(()) } } + +pub unsafe fn init_debug_hook(hook: &'static dyn Fn(core::fmt::Arguments)) { + debug::DEBUG_HOOK.replace(hook); +} diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index 8c695531..b9396808 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -42,6 +42,7 @@ pub struct Vnode { kind: VnodeKind, data: RefCell>>, fs: RefCell>>, + target: RefCell>, } pub trait VnodeImpl { @@ -69,6 +70,7 @@ impl Vnode { kind, data: RefCell::new(None), fs: RefCell::new(None), + target: RefCell::new(None), }) } @@ -82,6 +84,11 @@ impl Vnode { self.kind } + #[inline] + pub fn target(&self) -> Option { + self.target.borrow().clone() + } + #[inline] pub fn data(&self) -> RefMut>> { self.data.borrow_mut() @@ -92,6 +99,11 @@ impl Vnode { self.fs.borrow().clone() } + #[inline] + pub fn set_target(&self, target: Option) { + self.target.replace(target); + } + pub fn parent(self: &VnodeRef) -> VnodeRef { match &self.tree.borrow().parent { Some(parent) => parent.upgrade().unwrap(), @@ -112,6 +124,11 @@ impl Vnode { self.kind == VnodeKind::Directory } + #[inline] + pub fn is_mountpoint(&self) -> bool { + self.is_directory() && self.target.borrow().is_some() + } + // Cache tree operations pub fn add_child(self: &VnodeRef, child: VnodeRef) { let parent_weak = Rc::downgrade(self); @@ -126,9 +143,11 @@ impl Vnode { parent_borrow.children.push(child); } - pub fn dump(&self, f: &mut fmt::Formatter<'_>, depth: usize) -> fmt::Result { - for _ in 0..depth { - f.write_str(" ")?; + pub fn dump(&self, f: &mut fmt::Formatter<'_>, depth: usize, indent: bool) -> fmt::Result { + if indent { + for _ in 0..depth { + f.write_str(" ")?; + } } write!(f, "{:?}", self.name)?; @@ -136,12 +155,20 @@ impl Vnode { match self.kind { VnodeKind::Directory => { let tree = self.tree.borrow(); + let target = self.target(); + + if let Some(target) = target { + assert_eq!(tree.children.len(), 0); + f.write_str(" -> ")?; + return target.dump(f, depth, false); + } + if tree.children.is_empty() { f.write_str(" []")?; } else { f.write_str(" [\n")?; for child in tree.children.iter() { - child.dump(f, depth + 1)?; + child.dump(f, depth + 1, true)?; f.write_str("\n")?; } for _ in 0..depth { @@ -265,6 +292,48 @@ impl Vnode { todo!(); } } + + pub fn mount(self: &VnodeRef, fs_root: VnodeRef) -> Result<(), Error> { + if !self.is_directory() { + return Err(Error::NotADirectory); + } + if !fs_root.is_directory() { + todo!("Filesystem root is not a directory"); + } + if self.target.borrow().is_some() { + todo!("Target mountpoint is busy"); + } + + { + let mut child_borrow = fs_root.tree.borrow_mut(); + if child_borrow.parent.is_some() { + todo!("Filesystem is already mounted somewhere else"); + } + child_borrow.parent = Some(Rc::downgrade(self)); + } + self.target.replace(Some(fs_root)); + + Ok(()) + } + + pub fn unmount_target(self: &VnodeRef) -> Result<(), Error> { + if !self.is_directory() { + return Err(Error::NotADirectory); + } + let Some(fs_root) = self.target.take() else { + todo!(); + }; + + { + let mut target_borrow = fs_root.tree.borrow_mut(); + let Some(parent) = target_borrow.parent.take() else { + todo!() + }; + assert!(Rc::ptr_eq(self, &parent.upgrade().unwrap())); + } + + Ok(()) + } } impl fmt::Debug for Vnode { @@ -288,6 +357,6 @@ impl VnodeDump { impl fmt::Debug for VnodeDump { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.node.dump(f, 0) + self.node.dump(f, 0, true) } } diff --git a/src/debug.rs b/src/debug.rs index 67ed93ea..a514a0c1 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -128,6 +128,11 @@ pub fn init() { DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { sink: PLATFORM.primary_serial().unwrap(), })); + unsafe { + vfs::init_debug_hook(&move |args| { + debug_internal(args, LogLevel::Debug); + }); + } } #[doc = "hide"] diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 5998278b..14a99e67 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,7 +1,13 @@ //! Filesystem implementations +use vfs::VnodeRef; +use yggdrasil_abi::{error::Error, io::MountOptions}; + use crate::util::OneTimeInit; +pub mod devfs; +pub mod tar; + pub struct Initrd { pub phys_page_start: usize, pub phys_page_len: usize, @@ -10,5 +16,11 @@ pub struct Initrd { pub static INITRD_DATA: OneTimeInit = OneTimeInit::new(); -pub mod devfs; -pub mod tar; +pub fn create_filesystem(options: &MountOptions) -> Result { + let fs_name = options.filesystem.unwrap(); + + match fs_name { + "devfs" => Ok(devfs::root().clone()), + _ => todo!(), + } +} diff --git a/src/main.rs b/src/main.rs index f966da6b..14a05aa7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,8 +15,6 @@ extern crate yggdrasil_abi as abi; -use core::ops::DerefMut; - use abi::{ error::Error, io::{OpenFlags, RawFd}, @@ -67,7 +65,7 @@ pub fn kernel_main() { let tty_node = devfs_root.lookup("ttyS0").unwrap(); let ioctx = IoContext::new(root); - let node = ioctx.find(None, "/init", false).unwrap(); + let node = ioctx.find(None, "/init", true, true).unwrap(); let file = node.open(OpenFlags::new().read()).unwrap(); { @@ -83,30 +81,5 @@ pub fn kernel_main() { user_init.enqueue_somewhere(); } - // static USER_PROGRAM: &[u8] = include_bytes!(concat!( - // "../../target/aarch64-unknown-yggdrasil/", - // env!("PROFILE"), - // "/test_program" - // )); - - // let ioctx = IoContext::new(devfs_root.clone()); - - // // Spawn a test user task - // let proc = - // proc::exec::create_from_memory(USER_PROGRAM, &["user-program", "argument 1", "argument 2"]); - - // match proc { - // Ok(proc) => { - // // Setup I/O for the process - // // let mut io = proc.io.lock(); - // // io.set_file(RawFd::STDOUT, todo!()).unwrap(); - - // proc.enqueue_somewhere(); - // } - // Err(err) => { - // warnln!("Failed to create user process: {:?}", err); - // } - // }; - Process::current().exit(0); } diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 4d9b666d..1a2dc4ab 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -53,12 +53,12 @@ impl PhysicalMemoryRegion { } /// Returns an address range covered by the region - pub const fn range(&self) -> Range { + pub fn range(&self) -> Range { self.base..self.end() } /// Provides an iterator over the pages in the region - pub const fn pages(&self) -> StepBy> { + pub fn pages(&self) -> StepBy> { self.range().step_by(0x1000) } } diff --git a/src/syscall/arg.rs b/src/syscall/arg.rs new file mode 100644 index 00000000..f7dc3e52 --- /dev/null +++ b/src/syscall/arg.rs @@ -0,0 +1,32 @@ +use core::alloc::Layout; + +use yggdrasil_abi::error::Error; + +pub(super) fn arg_buffer_ref<'a>(base: usize, len: usize) -> Result<&'a [u8], Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + Ok(unsafe { core::slice::from_raw_parts(base as *const u8, len) }) +} + +pub(super) fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8], Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + Ok(unsafe { core::slice::from_raw_parts_mut(base as *mut u8, len) }) +} + +pub(super) fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error> { + let slice = arg_buffer_ref(base, len)?; + Ok(core::str::from_utf8(slice).unwrap()) +} + +pub(super) fn arg_user_ref<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { + let layout = Layout::new::(); + if addr % layout.align() != 0 { + todo!("Misaligned argument"); + } + // TODO check that addr actually points to mapped (and user-accessible) memory + let value = unsafe { core::mem::transmute::<_, &'a T>(addr) }; + Ok(value) +} diff --git a/src/syscall.rs b/src/syscall/mod.rs similarity index 76% rename from src/syscall.rs rename to src/syscall/mod.rs index 1596b592..746f37e8 100644 --- a/src/syscall.rs +++ b/src/syscall/mod.rs @@ -7,31 +7,17 @@ use abi::{ syscall::SyscallFunction, }; use vfs::{Read, Write}; +use yggdrasil_abi::io::{MountOptions, UnmountOptions}; use crate::{ + fs, mem::table::{PageAttributes, VirtualMemoryManager}, proc::wait, task::process::Process, }; -fn arg_buffer_ref<'a>(base: usize, len: usize) -> Result<&'a [u8], Error> { - if base + len > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } - Ok(unsafe { core::slice::from_raw_parts(base as *const u8, len) }) -} - -fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8], Error> { - if base + len > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } - Ok(unsafe { core::slice::from_raw_parts_mut(base as *mut u8, len) }) -} - -fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error> { - let slice = arg_buffer_ref(base, len)?; - Ok(core::str::from_utf8(slice).unwrap()) -} +mod arg; +use arg::*; fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { match func { @@ -132,6 +118,38 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result io.close_file(fd)?; Ok(0) } + SyscallFunction::Mount => { + let options = arg_user_ref::(args[0] as usize)?; + + let proc = Process::current(); + let mut io = proc.io.lock(); + + let target_node = io.ioctx().find(None, options.target, true, false)?; + if !target_node.is_directory() { + return Err(Error::NotADirectory); + } + + let fs_root = fs::create_filesystem(options)?; + + target_node.mount(fs_root)?; + + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); + + Ok(0) + } + SyscallFunction::Unmount => { + let options = arg_user_ref::(args[0] as usize)?; + + let proc = Process::current(); + let mut io = proc.io.lock(); + + let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; + mountpoint.unmount_target()?; + + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); + + Ok(0) + } } } From 0bdc30afdc23baf9bb73aa7a410f56daa03d943b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 20 Jul 2023 12:38:58 +0300 Subject: [PATCH 007/211] proc: update ABI dependency crate --- src/syscall/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 746f37e8..63b0a8ce 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -2,12 +2,15 @@ use core::time::Duration; use abi::{ - error::{Error, IntoSyscallResult}, + error::Error, io::{OpenFlags, RawFd}, syscall::SyscallFunction, }; use vfs::{Read, Write}; -use yggdrasil_abi::io::{MountOptions, UnmountOptions}; +use yggdrasil_abi::{ + error::SyscallResult, + io::{MountOptions, UnmountOptions}, +}; use crate::{ fs, From d3e3d3add891f7e5593025b215bbdc5ee0f78614 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 20 Jul 2023 15:40:49 +0300 Subject: [PATCH 008/211] proc: reimplement OpenOptions --- Cargo.toml | 1 - lib/vfs/Cargo.toml | 1 - lib/vfs/macros/Cargo.toml | 16 --------- lib/vfs/macros/src/lib.rs | 55 ----------------------------- lib/vfs/macros/src/vnode_impl.rs | 60 -------------------------------- lib/vfs/src/char.rs | 14 +++++--- lib/vfs/src/ioctx.rs | 11 ++++-- lib/vfs/src/node.rs | 42 ++++++++++++++++------ src/fs/tar.rs | 11 +++--- src/main.rs | 10 +++--- src/syscall/mod.rs | 7 ++-- 11 files changed, 63 insertions(+), 165 deletions(-) delete mode 100644 lib/vfs/macros/Cargo.toml delete mode 100644 lib/vfs/macros/src/lib.rs delete mode 100644 lib/vfs/macros/src/vnode_impl.rs diff --git a/Cargo.toml b/Cargo.toml index e5f0a7ab..485ff936 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } -vfs-macros = { path = "lib/vfs/macros" } aarch64-cpu = "9.3.1" atomic_enum = "0.2.0" diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 0802b412..42c6b09a 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -7,5 +7,4 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -vfs-macros = { path = "macros" } bitflags = "2.3.3" diff --git a/lib/vfs/macros/Cargo.toml b/lib/vfs/macros/Cargo.toml deleted file mode 100644 index 7959b357..00000000 --- a/lib/vfs/macros/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "vfs-macros" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[lib] -proc-macro = true - -[dependencies] -lazy_static = "1.4.0" -proc-macro-crate = "1.3.1" -proc-macro2 = "1.0.66" -quote = "1.0.31" -syn = { version = "2.0.26", features = ["full"] } diff --git a/lib/vfs/macros/src/lib.rs b/lib/vfs/macros/src/lib.rs deleted file mode 100644 index 59296dee..00000000 --- a/lib/vfs/macros/src/lib.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::collections::HashSet; - -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use proc_macro_crate::{crate_name, FoundCrate}; -use quote::{quote, ToTokens}; -use syn::{parse_macro_input, ImplItem, ItemImpl}; - -mod vnode_impl; -use vnode_impl::IMPLS; - -#[proc_macro_attribute] -pub fn auto_vnode_impl(attr: TokenStream, input: TokenStream) -> TokenStream { - let current_crate = crate_name("vfs").unwrap(); - - let vfs_crate = match current_crate { - FoundCrate::Itself => Ident::new("crate", Span::call_site()), - FoundCrate::Name(name) => Ident::new(&name, Span::call_site()), - }; - let vfs_crate = quote! { #vfs_crate }; - - let mut impl_item = parse_macro_input!(input as ItemImpl); - let behavior = if attr.is_empty() { - "unimplemented".to_string() - } else { - parse_macro_input!(attr as Ident).to_string() - }; - let behavior = match behavior.as_str() { - "unimplemented" => quote! { unimplemented!() }, - "panic" => quote! { panic!() }, - "error" => quote! { Err(yggdrasil_abi::error::Error::NotImplemented) }, - _ => panic!("Unknown #[auto_vnode_impl] behavior: {:?}", behavior), - }; - - let mut missing_impls: HashSet<&str> = HashSet::from_iter(IMPLS.keys().copied()); - - for item in &impl_item.items { - match item { - ImplItem::Fn(f) => { - let name = &f.sig.ident.to_string(); - missing_impls.remove(name.as_str()); - } - _ => panic!("Unexpected item in Vnode impl"), - } - } - - for &item in &missing_impls { - let f = IMPLS.get(item).unwrap(); - let fn_impl = f(&vfs_crate, &behavior); - - impl_item.items.push(ImplItem::Verbatim(fn_impl)); - } - - impl_item.to_token_stream().into() -} diff --git a/lib/vfs/macros/src/vnode_impl.rs b/lib/vfs/macros/src/vnode_impl.rs deleted file mode 100644 index 91126cfc..00000000 --- a/lib/vfs/macros/src/vnode_impl.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::collections::HashMap; - -use proc_macro2::TokenStream as TS2; -use quote::quote; - -fn impl_open(vfs: &TS2, behavior: &TS2) -> TS2 { - quote! { - fn open(&mut self, _node: &#vfs::VnodeRef, _opts: yggdrasil_abi::io::OpenFlags) - -> Result { - #behavior - } - } -} - -fn impl_close(vfs: &TS2, behavior: &TS2) -> TS2 { - quote! { - fn close(&mut self, _node: &#vfs::VnodeRef) -> Result<(), Error> { - #behavior - } - } -} - -fn impl_create(vfs: &TS2, behavior: &TS2) -> TS2 { - quote! { - fn create(&mut self, _at: &#vfs::VnodeRef, _name: &str, _kind: #vfs::VnodeKind) - -> Result<#vfs::VnodeRef, yggdrasil_abi::error::Error> { - #behavior - } - } -} - -fn impl_write(vfs: &TS2, behavior: &TS2) -> TS2 { - quote! { - fn write(&mut self, _node: &#vfs::VnodeRef, _pos: u64, _data: &[u8]) - -> Result { - #behavior - } - } -} - -fn impl_read(vfs: &TS2, behavior: &TS2) -> TS2 { - quote! { - fn read(&mut self, _node: &#vfs::VnodeRef, _pos: u64, _data: &mut [u8]) - -> Result { - #behavior - } - } -} - -lazy_static::lazy_static! { - pub static ref IMPLS: HashMap<&'static str, fn(&TS2, &TS2) -> TS2> = { - let mut m = HashMap::new(); - m.insert("open".into(), impl_open as _); - m.insert("close".into(), impl_close as _); - m.insert("create".into(), impl_create as _); - m.insert("write".into(), impl_write as _); - m.insert("read".into(), impl_read as _); - m - }; -} diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index fe3aa635..6da845f9 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -1,5 +1,7 @@ -use vfs_macros::auto_vnode_impl; -use yggdrasil_abi::{error::Error, io::OpenFlags}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, +}; use crate::{node::VnodeImpl, VnodeRef}; @@ -18,9 +20,13 @@ impl CharDeviceWrapper { } } -#[auto_vnode_impl(error)] impl VnodeImpl for CharDeviceWrapper { - fn open(&mut self, _node: &VnodeRef, _opts: OpenFlags) -> Result { + fn open( + &mut self, + _node: &VnodeRef, + _opts: OpenOptions, + _mode: FileMode, + ) -> Result { Ok(0) } diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index f47a4b94..1f721d9d 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -1,4 +1,8 @@ -use yggdrasil_abi::{error::Error, io::OpenFlags, path}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, + path, +}; use crate::{file::FileRef, node::VnodeRef}; @@ -107,7 +111,8 @@ impl IoContext { &self, at: Option, path: &str, - opts: OpenFlags, + opts: OpenOptions, + mode: FileMode, ) -> Result { let node = match self.find(at.clone(), path, true, true) { Err(Error::DoesNotExist) => { @@ -117,7 +122,7 @@ impl IoContext { o => o, }?; - node.open(opts) + node.open(opts, mode) } pub fn root(&self) -> &VnodeRef { diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index b9396808..ece2c418 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -9,7 +9,10 @@ use alloc::{ string::String, vec::Vec, }; -use yggdrasil_abi::{error::Error, io::OpenFlags}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, +}; use crate::{ file::{File, FileFlags, FileRef}, @@ -46,16 +49,33 @@ pub struct Vnode { } pub trait VnodeImpl { - fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result; + fn create(&mut self, _at: &VnodeRef, _name: &str, _kind: VnodeKind) -> Result { + Err(Error::NotImplemented) + } - fn open(&mut self, node: &VnodeRef, opts: OpenFlags) -> Result; - fn close(&mut self, node: &VnodeRef) -> Result<(), Error>; + fn open( + &mut self, + _node: &VnodeRef, + _opts: OpenOptions, + _mode: FileMode, + ) -> Result { + Err(Error::NotImplemented) + } - fn read(&mut self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result; - fn write(&mut self, node: &VnodeRef, pos: u64, data: &[u8]) -> Result; + fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + Err(Error::NotImplemented) + } + + fn read(&mut self, _node: &VnodeRef, _pos: u64, _data: &mut [u8]) -> Result { + Err(Error::NotImplemented) + } + + fn write(&mut self, _node: &VnodeRef, _pos: u64, _data: &[u8]) -> Result { + Err(Error::NotImplemented) + } fn size(&mut self, _node: &VnodeRef) -> Result { - unimplemented!() + Err(Error::NotImplemented) } } @@ -205,13 +225,13 @@ impl Vnode { } // Node operations - pub fn open(self: &VnodeRef, flags: OpenFlags) -> Result { + pub fn open(self: &VnodeRef, flags: OpenOptions, mode: FileMode) -> Result { let mut open_flags = FileFlags::empty(); - if flags.is_read() { + if flags.contains(OpenOptions::READ) { open_flags |= FileFlags::READ; } - if flags.is_write() { + if flags.contains(OpenOptions::WRITE) { open_flags |= FileFlags::WRITE; } @@ -220,7 +240,7 @@ impl Vnode { } if let Some(ref mut data) = *self.data() { - let pos = data.open(self, flags)?; + let pos = data.open(self, flags, mode)?; Ok(File::normal(self.clone(), pos, open_flags)) } else { todo!() diff --git a/src/fs/tar.rs b/src/fs/tar.rs index 92ed38ee..44bc8930 100644 --- a/src/fs/tar.rs +++ b/src/fs/tar.rs @@ -1,8 +1,7 @@ -use abi::error::Error; +use abi::{error::Error, io::FileMode}; use alloc::{boxed::Box, rc::Rc}; use vfs::{Filesystem, Vnode, VnodeImpl, VnodeKind, VnodeRef}; -use vfs_macros::auto_vnode_impl; -use yggdrasil_abi::io::OpenFlags; +use yggdrasil_abi::io::OpenOptions; use crate::util::OneTimeInit; @@ -163,7 +162,6 @@ struct RegularInode { data: &'static [u8], } -#[auto_vnode_impl] impl VnodeImpl for DirInode { fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { let child = Vnode::new(name, kind); @@ -176,10 +174,9 @@ impl VnodeImpl for DirInode { } } -#[auto_vnode_impl] impl VnodeImpl for RegularInode { - fn open(&mut self, node: &VnodeRef, opts: OpenFlags) -> Result { - if opts.is_write() { + fn open(&mut self, node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { + if opts.contains(OpenOptions::WRITE) { panic!("TODO: tarfs write"); } diff --git a/src/main.rs b/src/main.rs index 14a05aa7..3e97a92e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,11 +17,11 @@ extern crate yggdrasil_abi as abi; use abi::{ error::Error, - io::{OpenFlags, RawFd}, + io::{FileMode, OpenOptions, RawFd}, }; use fs::{devfs, tar::TarFilesystem, INITRD_DATA}; use task::process::Process; -use vfs::{Filesystem, IoContext, Read, VnodeRef}; +use vfs::{Filesystem, IoContext, VnodeRef}; extern crate alloc; @@ -66,13 +66,15 @@ pub fn kernel_main() { let ioctx = IoContext::new(root); let node = ioctx.find(None, "/init", true, true).unwrap(); - let file = node.open(OpenFlags::new().read()).unwrap(); + let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); { let user_init = proc::exec::load_elf(file, &["/init"]).unwrap(); let mut io = user_init.io.lock(); io.set_ioctx(ioctx); - let stdout = tty_node.open(OpenFlags::new().write()).unwrap(); + let stdout = tty_node + .open(OpenOptions::WRITE, FileMode::empty()) + .unwrap(); let stderr = stdout.clone(); io.set_file(RawFd::STDOUT, stdout).unwrap(); diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 63b0a8ce..6c39c17c 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -3,7 +3,7 @@ use core::time::Duration; use abi::{ error::Error, - io::{OpenFlags, RawFd}, + io::{FileMode, OpenOptions, RawFd}, syscall::SyscallFunction, }; use vfs::{Read, Write}; @@ -103,12 +103,13 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } SyscallFunction::Open => { let path = arg_user_str(args[0] as usize, args[1] as usize)?; - let opts = OpenFlags(args[2] as u32); + let opts = OpenOptions::from(args[2] as u32); + let mode = FileMode::from(args[3] as u32); let proc = Process::current(); let mut io = proc.io.lock(); - let file = io.ioctx().open(None, path, opts)?; + let file = io.ioctx().open(None, path, opts, mode)?; let fd = io.place_file(file)?; Ok(fd.0 as usize) From 98c4e9152ec54ff136940907a3dbd3965f87f7df Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 20 Jul 2023 16:19:23 +0300 Subject: [PATCH 009/211] proc: implement basic line discipline --- src/device/tty.rs | 107 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 6 deletions(-) diff --git a/src/device/tty.rs b/src/device/tty.rs index df3ca931..799ddbae 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -1,5 +1,8 @@ //! Terminal driver implementation -use abi::error::Error; +use abi::{ + error::Error, + io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, +}; use crate::{proc::wait::Wait, sync::IrqSafeSpinlock}; @@ -18,6 +21,7 @@ pub struct CharRing { wait_read: Wait, wait_write: Wait, inner: IrqSafeSpinlock>, + config: IrqSafeSpinlock, } /// Terminal device interface @@ -37,26 +41,88 @@ pub trait TtyDevice: SerialDevice { /// Sends a single byte to the terminal fn line_send(&self, byte: u8) -> Result<(), Error> { + let config = self.ring().config.lock(); + + if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + self.send(b'\r').ok(); + } + self.send(byte) } /// Receives a single byte from the terminal - fn recv_byte(&self, byte: u8) { + fn recv_byte(&self, mut byte: u8) { let ring = self.ring(); + let config = ring.config.lock(); + + if byte == b'\r' && config.input.contains(TerminalInputOptions::CR_TO_NL) { + byte = b'\n'; + } + + if byte == b'\n' { + let echo = config.line.contains(TerminalLineOptions::ECHO) + || config + .line + .contains(TerminalLineOptions::CANONICAL | TerminalLineOptions::ECHO_NL); + if config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + self.send(b'\r').ok(); + } + self.send(byte).ok(); + } else if config.line.contains(TerminalLineOptions::ECHO) { + // TODO proper control + if byte.is_ascii_control() { + self.send(b'^').ok(); + self.send(byte + 0x40).ok(); + } else { + self.send(byte).ok(); + } + } + + // TODO handle signals + ring.putc(byte, false).ok(); } /// Reads and processes data from the terminal fn line_read(&'static self, data: &mut [u8]) -> Result { let ring = self.ring(); + let mut config = ring.config.lock(); if data.is_empty() { return Ok(0); } - let byte = ring.getc()?; - data[0] = byte; - Ok(1) + if !config.is_canonical() { + let byte = ring.getc()?; + data[0] = byte; + Ok(1) + } else { + let mut rem = data.len(); + let mut off = 0; + + // Run until either end of buffer or return condition is reached + while rem != 0 { + drop(config); + let byte = ring.getc()?; + config = ring.config.lock(); + + if byte == config.chars.eof && config.is_canonical() { + break; + } + + // TODO handle special characters + + data[off] = byte; + off += 1; + rem -= 1; + + if byte == b'\n' || byte == b'\r' { + break; + } + } + + Ok(off) + } } /// Processes and writes the data to the terminal @@ -109,13 +175,42 @@ impl CharRing { }), wait_read: Wait::new("char_ring_read"), wait_write: Wait::new("char_ring_write"), + config: IrqSafeSpinlock::new(TerminalOptions::const_default()), } } /// Returns `true` if the buffer has data to read pub fn is_readable(&self) -> bool { let inner = self.inner.lock(); - inner.is_readable() || inner.flags != 0 + let config = self.config.lock(); + + if config.is_canonical() { + let mut rd = inner.rd; + let mut count = 0usize; + + loop { + let readable = if rd <= inner.wr { + (inner.wr - rd) > 0 + } else { + (inner.wr + (N - rd)) > 0 + }; + + if !readable { + break; + } + + let byte = inner.data[rd]; + if byte == b'\n' { + count += 1; + } + + rd = (rd + 1) % N; + } + + count != 0 || inner.flags != 0 + } else { + inner.is_readable() || inner.flags != 0 + } } /// Reads a single character from the buffer, blocking until available From 9e0555f7a3fe6908830bd8cd23618da857c754b2 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 20 Jul 2023 18:14:30 +0300 Subject: [PATCH 010/211] proc: /init is responsible for setting up std*** --- src/main.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3e97a92e..3cefd594 100644 --- a/src/main.rs +++ b/src/main.rs @@ -72,13 +72,6 @@ pub fn kernel_main() { let user_init = proc::exec::load_elf(file, &["/init"]).unwrap(); let mut io = user_init.io.lock(); io.set_ioctx(ioctx); - let stdout = tty_node - .open(OpenOptions::WRITE, FileMode::empty()) - .unwrap(); - let stderr = stdout.clone(); - - io.set_file(RawFd::STDOUT, stdout).unwrap(); - io.set_file(RawFd::STDERR, stderr).unwrap(); drop(io); user_init.enqueue_somewhere(); } From 69a413f6bb23972955222596bf3997172463a1ec Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 20 Jul 2023 18:51:56 +0300 Subject: [PATCH 011/211] ***: fix clippy and check warnings --- Cargo.toml | 6 +- lib/vfs/src/ioctx.rs | 5 +- lib/vfs/src/lib.rs | 3 + lib/vfs/src/node.rs | 45 ++++---- src/arch/aarch64/mod.rs | 4 +- src/debug.rs | 3 +- src/device/tty.rs | 4 +- src/fs/mod.rs | 6 + src/fs/tar.rs | 91 ++++++--------- src/main.rs | 7 +- src/proc/elf.rs | 158 +++++++++++++++++++++++++ src/proc/exec.rs | 31 ++--- src/proc/io.rs | 12 +- src/proc/mod.rs | 248 +--------------------------------------- src/util.rs | 2 + 15 files changed, 258 insertions(+), 367 deletions(-) create mode 100644 src/proc/elf.rs diff --git a/Cargo.toml b/Cargo.toml index 485ff936..062ebf14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,4 +18,8 @@ spinning_top = "0.2.5" static_assertions = "1.1.0" tock-registers = "0.8.1" -elf = { version = "0.7.2", path = "../../rust-elf", default-features = false, features = ["no_std_stream"] } +[dependencies.elf] +version = "0.7.2" +git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" +default-features = false +features = ["no_std_stream"] diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index 1f721d9d..838c11c4 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -20,7 +20,6 @@ impl IoContext { } fn _find( - &self, mut at: VnodeRef, path: &str, follow: bool, @@ -82,7 +81,7 @@ impl IoContext { if rest.is_empty() { Ok(node) } else { - self._find(node, rest, follow, follow_mount) + Self::_find(node, rest, follow, follow_mount) } } @@ -104,7 +103,7 @@ impl IoContext { self.cwd.clone() }; - self._find(at, path, follow, follow_mount) + Self::_find(at, path, follow, follow_mount) } pub fn open( diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 78f9b245..10937641 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -66,6 +66,9 @@ fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Res } } +/// # Safety +/// +/// Unsafe: only meant to be called early in kernel init sequence pub unsafe fn init_debug_hook(hook: &'static dyn Fn(core::fmt::Arguments)) { debug::DEBUG_HOOK.replace(hook); } diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index ece2c418..82088b3a 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -172,32 +172,29 @@ impl Vnode { write!(f, "{:?}", self.name)?; - match self.kind { - VnodeKind::Directory => { - let tree = self.tree.borrow(); - let target = self.target(); + if self.is_directory() { + let tree = self.tree.borrow(); + let target = self.target(); - if let Some(target) = target { - assert_eq!(tree.children.len(), 0); - f.write_str(" -> ")?; - return target.dump(f, depth, false); - } - - if tree.children.is_empty() { - f.write_str(" []")?; - } else { - f.write_str(" [\n")?; - for child in tree.children.iter() { - child.dump(f, depth + 1, true)?; - f.write_str("\n")?; - } - for _ in 0..depth { - f.write_str(" ")?; - } - f.write_str("]")?; - } + if let Some(target) = target { + assert_eq!(tree.children.len(), 0); + f.write_str(" -> ")?; + return target.dump(f, depth, false); + } + + if tree.children.is_empty() { + f.write_str(" []")?; + } else { + f.write_str(" [\n")?; + for child in tree.children.iter() { + child.dump(f, depth + 1, true)?; + f.write_str("\n")?; + } + for _ in 0..depth { + f.write_str(" ")?; + } + f.write_str("]")?; } - _ => (), } Ok(()) diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 13feb4f1..02385a64 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -45,11 +45,11 @@ pub mod smp; pub mod table; pub mod timer; -pub(self) const BOOT_STACK_SIZE: usize = 65536; +const BOOT_STACK_SIZE: usize = 65536; #[derive(Clone, Copy)] #[repr(C, align(0x20))] -pub(self) struct KernelStack { +struct KernelStack { data: [u8; BOOT_STACK_SIZE], } diff --git a/src/debug.rs b/src/debug.rs index a514a0c1..f2e6e051 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -99,13 +99,14 @@ impl fmt::Write for DebugPrinter { } } +/// Prints a hex-dump of a slice, appending a virtual address offset to the output pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { for (i, b) in data.iter().enumerate() { if i % 16 == 0 { log_print_raw!(level, "{:X}: ", addr_offset + i) } - log_print_raw!(level, "{:02X}", data[i]); + log_print_raw!(level, "{:02X}", b); if i % 16 == 15 { log_print_raw!(level, "\n"); diff --git a/src/device/tty.rs b/src/device/tty.rs index 799ddbae..63a11baf 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -60,10 +60,12 @@ pub trait TtyDevice: SerialDevice { } if byte == b'\n' { - let echo = config.line.contains(TerminalLineOptions::ECHO) + // TODO implement proper echo here + let _echo = config.line.contains(TerminalLineOptions::ECHO) || config .line .contains(TerminalLineOptions::CANONICAL | TerminalLineOptions::ECHO_NL); + if config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { self.send(b'\r').ok(); } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 14a99e67..a0c3e58f 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -8,14 +8,20 @@ use crate::util::OneTimeInit; pub mod devfs; pub mod tar; +/// Describes in-memory filesystem image used as initial root pub struct Initrd { + /// Page-aligned start address of the initrd pub phys_page_start: usize, + /// Page-aligned length pub phys_page_len: usize, + /// Safe reference to the initrd data slice pub data: &'static [u8], } +/// Holds reference to the data of initrd as well as its page-aligned physical memory region pub static INITRD_DATA: OneTimeInit = OneTimeInit::new(); +/// Constructs an instance of a filesystem for given set of [MountOptions] pub fn create_filesystem(options: &MountOptions) -> Result { let fs_name = options.filesystem.unwrap(); diff --git a/src/fs/tar.rs b/src/fs/tar.rs index 44bc8930..c1d2358f 100644 --- a/src/fs/tar.rs +++ b/src/fs/tar.rs @@ -1,3 +1,4 @@ +//! In-memory filesystem implementation use abi::{error::Error, io::FileMode}; use alloc::{boxed::Box, rc::Rc}; use vfs::{Filesystem, Vnode, VnodeImpl, VnodeKind, VnodeRef}; @@ -6,38 +7,38 @@ use yggdrasil_abi::io::OpenOptions; use crate::util::OneTimeInit; #[repr(C)] -pub struct OctalField { +struct OctalField { data: [u8; N], } #[repr(C)] -pub struct TarString { +struct TarString { data: [u8; N], } -pub struct TarIterator<'a> { +struct TarIterator<'a> { data: &'a [u8], offset: usize, zero_blocks: usize, } #[repr(packed)] -pub struct TarEntry { +struct TarEntry { name: TarString<100>, - mode: OctalField<8>, - uid: OctalField<8>, - gid: OctalField<8>, + _mode: OctalField<8>, + _uid: OctalField<8>, + _gid: OctalField<8>, size: OctalField<12>, - mtime: OctalField<12>, - checksum: OctalField<8>, + _mtime: OctalField<12>, + _checksum: OctalField<8>, type_: u8, - link_name: [u8; 100], - magic: [u8; 8], - user: [u8; 32], - group: [u8; 32], - dev_major: OctalField<8>, - dev_minor: OctalField<8>, - prefix: [u8; 155], + _link_name: TarString<100>, + _magic: [u8; 8], + _user: TarString<32>, + _group: TarString<32>, + _dev_major: OctalField<8>, + _dev_minor: OctalField<8>, + _prefix: TarString<155>, __pad: [u8; 12], } @@ -61,7 +62,7 @@ impl<'a> Iterator for TarIterator<'a> { } let hdr_ptr = &self.data[self.offset..]; - let hdr = unsafe { core::mem::transmute::<*const u8, &TarEntry>(hdr_ptr.as_ptr()) }; + let hdr = unsafe { &*(hdr_ptr.as_ptr() as *const TarEntry) }; if hdr.is_empty() { if self.zero_blocks == 1 { @@ -139,6 +140,7 @@ impl TarEntry { } } +/// tar-image based in-memory filesystem pub struct TarFilesystem { root: OneTimeInit, } @@ -175,7 +177,7 @@ impl VnodeImpl for DirInode { } impl VnodeImpl for RegularInode { - fn open(&mut self, node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { + fn open(&mut self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { if opts.contains(OpenOptions::WRITE) { panic!("TODO: tarfs write"); } @@ -183,16 +185,16 @@ impl VnodeImpl for RegularInode { Ok(0) } - fn close(&mut self, node: &VnodeRef) -> Result<(), Error> { + fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { Ok(()) } - fn read(&mut self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + fn read(&mut self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { let pos = pos as usize; if pos > self.data.len() { return Err(Error::InvalidFile); } - let mut rem = core::cmp::min(self.data.len() - pos, data.len()); + let rem = core::cmp::min(self.data.len() - pos, data.len()); data[..rem].copy_from_slice(&self.data[pos..pos + rem]); Ok(rem) } @@ -207,6 +209,7 @@ impl TarFilesystem { self: &Rc, at: &VnodeRef, path: &str, + kind: VnodeKind, create: bool, ) -> Result { debugln!("make_path {:?}", path); @@ -226,27 +229,25 @@ impl TarFilesystem { } infoln!("Create {:?}", element); - at.create(element, VnodeKind::Directory)? + let node = self.create_node_initial(element, kind); + at.add_child(node.clone()); + + node } }; if rest.is_empty() { Ok(node) } else { - self.make_path(&node, rest, create) + assert!(node.is_directory()); + self.make_path(&node, rest, kind, create) } } - fn create_node_initial( - self: &Rc, - name: &str, - hdr: &TarEntry, - data: Option<&[u8]>, - ) -> VnodeRef { + fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { assert!(!name.is_empty()); assert!(!name.contains('/')); - let kind = hdr.node_kind(); let node = Vnode::new(name, kind); node.set_fs(self.clone()); @@ -266,7 +267,7 @@ impl TarFilesystem { // 1. Create paths in tar for item in TarIterator::new(tar_data) { - let Ok((hdr, data)) = item else { + let Ok((hdr, _)) = item else { warnln!("Tar image is truncated"); return Err(Error::InvalidArgument); }; @@ -274,8 +275,8 @@ impl TarFilesystem { let path = hdr.name.as_str()?.trim_matches('/'); infoln!("path = {:?}", path); let (dirname, filename) = abi::path::split_right(path); - let parent = self.make_path(&root, dirname, true)?; - let node = self.create_node_initial(filename, hdr, data); + let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; + let node = self.create_node_initial(filename, hdr.node_kind()); parent.add_child(node); } @@ -288,7 +289,7 @@ impl TarFilesystem { if hdr.node_kind() == VnodeKind::Regular { let data = data.unwrap(); let path = hdr.name.as_str()?.trim_matches('/'); - let node = self.make_path(&root, path, false)?; + let node = self.make_path(&root, path, VnodeKind::Directory, false)?; node.set_data(Box::new(RegularInode { data })); } } @@ -296,6 +297,7 @@ impl TarFilesystem { Ok(root) } + /// Constructs a filesystem tree from a tar image in memory pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { let fs = Rc::new(TarFilesystem { root: OneTimeInit::new(), @@ -306,24 +308,3 @@ impl TarFilesystem { Ok(fs) } } - -// pub fn init() { -// let Some(initrd) = INITRD_DATA.try_get() else { -// warnln!("No initrd found"); -// return; -// }; -// -// let fs = match TarFilesystem::from_slice(initrd.data) { -// Ok(fs) => fs, -// Err(err) => { -// warnln!("Could not initialize tar filesystem: {:?}", err); -// return; -// } -// }; -// -// let r = fs.root().unwrap(); -// let dump = VnodeDump::new(r); -// infoln!("{:?}", dump); -// -// todo!() -// } diff --git a/src/main.rs b/src/main.rs index 3cefd594..91f56fd7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,9 +17,9 @@ extern crate yggdrasil_abi as abi; use abi::{ error::Error, - io::{FileMode, OpenOptions, RawFd}, + io::{FileMode, OpenOptions}, }; -use fs::{devfs, tar::TarFilesystem, INITRD_DATA}; +use fs::{tar::TarFilesystem, INITRD_DATA}; use task::process::Process; use vfs::{Filesystem, IoContext, VnodeRef}; @@ -61,9 +61,6 @@ pub fn kernel_main() { } }; - let devfs_root = devfs::root(); - let tty_node = devfs_root.lookup("ttyS0").unwrap(); - let ioctx = IoContext::new(root); let node = ioctx.find(None, "/init", true, true).unwrap(); let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); diff --git a/src/proc/elf.rs b/src/proc/elf.rs new file mode 100644 index 00000000..c74b6d26 --- /dev/null +++ b/src/proc/elf.rs @@ -0,0 +1,158 @@ +//! ELF binary format support +use elf::{ + abi::{PF_W, PF_X, PT_LOAD}, + endian::AnyEndian, + ElfStream, ParseError, +}; +use vfs::{FileRef, Read, Seek, SeekFrom}; +use yggdrasil_abi::error::Error; + +use crate::mem::{ + phys::{self, PageUsage}, + table::{AddressSpace, PageAttributes}, + ConvertAddress, +}; + +#[derive(Clone, Copy)] +struct FileReader<'a> { + file: &'a FileRef, +} + +impl elf::io_traits::InputStream for FileReader<'_> { + fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), elf::io_traits::StreamError> { + self.file + .borrow_mut() + .read_exact(buf) + .map_err(conv_stream_error) + } + + fn seek(&mut self, pos: elf::io_traits::SeekFrom) -> Result { + self.file + .borrow_mut() + .seek(conv_seek_from(pos)) + .map_err(conv_stream_error) + } +} + +#[inline] +fn conv_stream_error(_v: Error) -> elf::io_traits::StreamError { + elf::io_traits::StreamError { + message: "Elf read error", + } +} + +#[inline] +fn conv_seek_from(v: elf::io_traits::SeekFrom) -> SeekFrom { + match v { + elf::io_traits::SeekFrom::End(off) => SeekFrom::End(off), + elf::io_traits::SeekFrom::Start(off) => SeekFrom::Start(off), + _ => todo!(), + } +} + +#[inline] +fn from_parse_error(v: ParseError) -> Error { + warnln!("ELF loading error: {:?}", v); + Error::InvalidFile +} + +fn load_bytes( + space: &AddressSpace, + addr: usize, + mut src: F, + len: usize, + elf_attrs: u32, +) -> Result<(), Error> +where + F: FnMut(usize, &mut [u8]) -> Result<(), Error>, +{ + // TODO check for crazy addresses here + + let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { + (0, 0) => PageAttributes::AP_BOTH_READONLY, + (_, 0) => PageAttributes::AP_BOTH_READWRITE, + (0, _) => PageAttributes::AP_BOTH_READONLY, + (_, _) => PageAttributes::AP_BOTH_READWRITE, + }; + + let dst_page_off = addr & 0xFFF; + let dst_page_aligned = addr & !0xFFF; + let mut off = 0usize; + let mut rem = len; + + while rem != 0 { + let page_idx = (dst_page_off + off) / 0x1000; + let page_off = (dst_page_off + off) % 0x1000; + let count = core::cmp::min(rem, 0x1000 - page_off); + + let virt_page = dst_page_aligned + page_idx * 0x1000; + assert_eq!(virt_page & 0xFFF, 0); + if space.translate(virt_page).is_some() { + // Handle these cases + todo!(); + } + + let phys_page = phys::alloc_page(PageUsage::Used)?; + space.map_page(virt_page, phys_page, attrs)?; + + let dst_slice = unsafe { + let addr = (phys_page + page_off).virtualize(); + + core::slice::from_raw_parts_mut(addr as *mut u8, count) + }; + + src(off, dst_slice)?; + + rem -= count; + off += count; + } + + Ok(()) +} + +/// Loads an ELF binary from `file` into the target address space +pub fn load_elf_from_file(space: &AddressSpace, file: FileRef) -> Result { + let file = FileReader { file: &file }; + + let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; + + for phdr in elf.segments() { + if phdr.p_type != PT_LOAD { + continue; + } + + debugln!("LOAD {:#x}", phdr.p_vaddr); + + if phdr.p_filesz > 0 { + load_bytes( + space, + phdr.p_vaddr as usize, + |off, dst| { + let mut source = file.file.borrow_mut(); + source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + source.read_exact(dst) + }, + phdr.p_filesz as usize, + phdr.p_flags, + )?; + } + + if phdr.p_memsz > phdr.p_filesz { + let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; + let len = (phdr.p_memsz - phdr.p_filesz) as usize; + + load_bytes( + space, + addr, + |_, dst| { + dst.fill(0); + Ok(()) + }, + len, + phdr.p_flags, + )?; + } + } + + Ok(elf.ehdr.e_entry as usize) +} diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 9c2d33fb..7e038341 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -3,7 +3,7 @@ use core::mem::size_of; use abi::error::Error; use alloc::rc::Rc; -use vfs::{FileRef, Read, Seek}; +use vfs::FileRef; use crate::{ arch::aarch64::context::TaskContext, @@ -16,7 +16,7 @@ use crate::{ task::process::Process, }; -fn setup_args(space: &mut AddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { +fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { // arg data len let args_size: usize = args.iter().map(|x| x.len()).sum(); // 1 + arg ptr:len count @@ -42,15 +42,15 @@ fn setup_args(space: &mut AddressSpace, virt: usize, args: &[&str]) -> Result<() (write as *mut usize).write_volatile(args.len()); } - for i in 0..args.len() { + for (i, arg) in args.iter().enumerate() { // Place the argument pointer let ptr_place = write + (i * 2 + 1) * size_of::(); let len_place = ptr_place + size_of::(); unsafe { (ptr_place as *mut usize).write_volatile(virt + offset); - (len_place as *mut usize).write_volatile(args[i].len()); + (len_place as *mut usize).write_volatile(arg.len()); } - offset += args[i].len(); + offset += arg.len(); } // Place the argument data @@ -67,11 +67,7 @@ fn setup_args(space: &mut AddressSpace, virt: usize, args: &[&str]) -> Result<() Ok(()) } -fn setup_binary( - mut space: AddressSpace, - entry: usize, - args: &[&str], -) -> Result, Error> { +fn setup_binary(space: AddressSpace, entry: usize, args: &[&str]) -> Result, Error> { const USER_STACK_PAGES: usize = 8; let virt_stack_base = 0x10000000; @@ -87,7 +83,7 @@ fn setup_binary( )?; } - setup_args(&mut space, virt_args_base, args)?; + setup_args(&space, virt_args_base, args)?; debugln!( "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", @@ -107,17 +103,10 @@ fn setup_binary( Ok(Process::new_with_context(Some(space), context)) } +/// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory pub fn load_elf(file: FileRef, args: &[&str]) -> Result, Error> { - let mut space = AddressSpace::new_empty()?; - let elf_entry = proc::load_elf_from_file(&mut space, file)?; - - setup_binary(space, elf_entry, args) -} - -/// Sets up a userspace structure from a slice defining an ELF binary -pub fn create_from_memory(data: &[u8], args: &[&str]) -> Result, Error> { - let mut space = AddressSpace::new_empty()?; - let elf_entry = proc::load_elf_from_memory(&mut space, data); + let space = AddressSpace::new_empty()?; + let elf_entry = proc::elf::load_elf_from_file(&space, file)?; setup_binary(space, elf_entry, args) } diff --git a/src/proc/io.rs b/src/proc/io.rs index e8de1fbe..3dda86b5 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -1,6 +1,6 @@ //! Process I/O management use abi::{error::Error, io::RawFd}; -use alloc::collections::BTreeMap; +use alloc::collections::{btree_map::Entry, BTreeMap}; use vfs::{FileRef, IoContext}; /// I/O context of a process, contains information like root, current directory and file @@ -21,10 +21,7 @@ impl ProcessIo { /// Returns a file given descriptor refers to pub fn file(&self, fd: RawFd) -> Result { - self.files - .get(&fd) - .cloned() - .ok_or_else(|| Error::InvalidFile) + self.files.get(&fd).cloned().ok_or(Error::InvalidFile) } /// Sets the inner I/O context @@ -47,8 +44,9 @@ impl ProcessIo { pub fn place_file(&mut self, file: FileRef) -> Result { for idx in 0..64 { let fd = RawFd(idx); - if !self.files.contains_key(&fd) { - self.files.insert(fd, file); + + if let Entry::Vacant(e) = self.files.entry(fd) { + e.insert(file); return Ok(fd); } } diff --git a/src/proc/mod.rs b/src/proc/mod.rs index d4691f24..5c7e917e 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -1,252 +1,6 @@ //! Internal management for processes +pub mod elf; pub mod exec; pub mod io; pub mod wait; - -use aarch64_cpu::registers::{TCR_EL1::A1::TTBR0, TTBR0_EL1}; -use alloc::rc::Rc; -use elf::{ - abi::{PF_W, PF_X, PT_LOAD}, - endian::AnyEndian, - ElfBytes, ElfStream, ParseError, -}; -use tock_registers::interfaces::Writeable; -use vfs::{FileRef, Read, Seek, SeekFrom}; -use yggdrasil_abi::error::Error; - -use crate::{ - arch::aarch64::table::tlb_flush_vaae1, - debug::hex_dump, - mem::{ - phys::{self, PageUsage}, - table::{AddressSpace, PageAttributes}, - ConvertAddress, - }, -}; - -#[derive(Clone, Copy)] -pub struct FileReader<'a> { - file: &'a FileRef, -} - -impl elf::io_traits::Read for FileReader<'_> { - fn read(&mut self, buf: &mut [u8]) -> Result { - todo!() - } - - fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), elf::io_traits::StreamError> { - self.file - .borrow_mut() - .read_exact(buf) - .map_err(conv_stream_error) - } -} - -impl elf::io_traits::Seek for FileReader<'_> { - fn seek(&mut self, pos: elf::io_traits::SeekFrom) -> Result { - self.file - .borrow_mut() - .seek(conv_seek_from(pos)) - .map_err(conv_stream_error) - } -} - -#[inline] -fn conv_stream_error(v: Error) -> elf::io_traits::StreamError { - elf::io_traits::StreamError { - message: "Elf read error", - } -} - -#[inline] -fn conv_seek_from(v: elf::io_traits::SeekFrom) -> SeekFrom { - match v { - elf::io_traits::SeekFrom::End(off) => SeekFrom::End(off), - elf::io_traits::SeekFrom::Start(off) => SeekFrom::Start(off), - _ => todo!(), - } -} - -#[inline] -fn from_parse_error(v: ParseError) -> Error { - warnln!("ELF loading error: {:?}", v); - Error::InvalidFile -} - -fn load_segment(space: &mut AddressSpace, addr: usize, data: &[u8], memsz: usize, elf_attrs: u32) { - let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { - (0, 0) => PageAttributes::AP_BOTH_READONLY, - (_, 0) => PageAttributes::AP_BOTH_READWRITE, - (0, _) => PageAttributes::AP_BOTH_READONLY, - (_, _) => PageAttributes::AP_BOTH_READWRITE, - }; - - let aligned_start = addr & !0xFFF; - let aligned_end = (addr + memsz + 0xFFF) & !0xFFF; - - // Map and write pages - for page in (aligned_start..aligned_end).step_by(0x1000) { - if let Some(_phys) = space.translate(page) { - todo!(); - } else { - let phys = phys::alloc_page(PageUsage::Used).unwrap(); - space - .map_page(page, phys, PageAttributes::AP_BOTH_READWRITE) - .unwrap(); - - debugln!("MAP (alloc) {:#x} -> {:#x}", page, phys); - tlb_flush_vaae1(page); - } - } - - unsafe { - // Write the data - let dst = core::slice::from_raw_parts_mut(addr as *mut u8, memsz); - dst[..data.len()].copy_from_slice(data); - - // Zero the rest - dst[data.len()..memsz].fill(0); - } - - // Map the region as readonly - for page in (aligned_start..aligned_end).step_by(0x1000) { - let phys = space.translate(page).unwrap(); - space.map_page(page, phys, attrs).unwrap(); - } -} - -fn load_bytes( - space: &mut AddressSpace, - addr: usize, - mut src: F, - len: usize, - elf_attrs: u32, -) -> Result<(), Error> -where - F: FnMut(usize, &mut [u8]) -> Result<(), Error>, -{ - // TODO check for crazy addresses here - - let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { - (0, 0) => PageAttributes::AP_BOTH_READONLY, - (_, 0) => PageAttributes::AP_BOTH_READWRITE, - (0, _) => PageAttributes::AP_BOTH_READONLY, - (_, _) => PageAttributes::AP_BOTH_READWRITE, - }; - - let dst_page_off = addr & 0xFFF; - let dst_page_aligned = addr & !0xFFF; - let mut off = 0usize; - let mut rem = len; - - while rem != 0 { - let page_idx = (dst_page_off + off) / 0x1000; - let page_off = (dst_page_off + off) % 0x1000; - let count = core::cmp::min(rem, 0x1000 - page_off); - - let virt_page = dst_page_aligned + page_idx * 0x1000; - assert_eq!(virt_page & 0xFFF, 0); - if let Some(_) = space.translate(virt_page) { - // Handle these cases - todo!(); - } - - let phys_page = phys::alloc_page(PageUsage::Used)?; - debugln!("map {:#x} -> {:#x}", virt_page, phys_page); - space.map_page(virt_page, phys_page, attrs)?; - - let dst_slice = unsafe { - let addr = (phys_page + page_off).virtualize(); - - core::slice::from_raw_parts_mut(addr as *mut u8, count) - }; - - src(off, dst_slice)?; - debugln!("{:#x} (off = {}):", virt_page + page_off, page_off); - // hex_dump( - // crate::debug::LogLevel::Debug, - // virt_page + page_off, - // dst_slice, - // ); - - rem -= count; - off += count; - } - - Ok(()) -} - -pub fn load_elf_from_file(space: &mut AddressSpace, file: FileRef) -> Result { - let file = FileReader { file: &file }; - - let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; - - for phdr in elf.segments() { - if phdr.p_type != PT_LOAD { - continue; - } - - debugln!("LOAD {:#x}", phdr.p_vaddr); - - if phdr.p_filesz > 0 { - load_bytes( - space, - phdr.p_vaddr as usize, - |off, dst| { - let mut source = file.file.borrow_mut(); - source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; - source.read_exact(dst) - }, - phdr.p_filesz as usize, - phdr.p_flags, - )?; - } - - if phdr.p_memsz > phdr.p_filesz { - let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; - let len = (phdr.p_memsz - phdr.p_filesz) as usize; - - load_bytes( - space, - addr, - |_, dst| { - dst.fill(0); - Ok(()) - }, - len, - phdr.p_flags, - )?; - } - } - - Ok(elf.ehdr.e_entry as usize) -} - -/// Loads an ELF image into the address space from a slice -pub fn load_elf_from_memory(space: &mut AddressSpace, src: &[u8]) -> usize { - // Map the address space temporarily - TTBR0_EL1.set(space.physical_address() as u64); - - let elf = ElfBytes::::minimal_parse(src).unwrap(); - - for phdr in elf.segments().unwrap() { - if phdr.p_type != PT_LOAD { - continue; - } - - debugln!("LOAD {:#x}", phdr.p_vaddr); - let data = &src[phdr.p_offset as usize..(phdr.p_offset + phdr.p_filesz) as usize]; - load_segment( - space, - phdr.p_vaddr as usize, - data, - phdr.p_memsz as usize, - phdr.p_flags, - ); - } - - TTBR0_EL1.set_baddr(0); - - elf.ehdr.e_entry as usize -} diff --git a/src/util.rs b/src/util.rs index cda37d61..daec101b 100644 --- a/src/util.rs +++ b/src/util.rs @@ -70,6 +70,8 @@ impl OneTimeInit { unsafe { (*self.value.get()).assume_init_ref() } } + /// Returns an immutable reference to the underlying value and [None] if the value hasn't yet + /// been initialized pub fn try_get(&self) -> Option<&T> { if self.state.load(Ordering::Acquire) { Some(self.get()) From 64233db3df05899620d4e6639d27e9d847377345 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 21 Jul 2023 17:09:43 +0300 Subject: [PATCH 012/211] fs: implement in-memory writable filesystem --- Cargo.toml | 1 + lib/memfs/Cargo.toml | 15 ++ lib/memfs/src/block.rs | 260 +++++++++++++++++++ lib/memfs/src/bvec.rs | 459 ++++++++++++++++++++++++++++++++++ lib/memfs/src/dir.rs | 32 +++ lib/memfs/src/file.rs | 37 +++ lib/memfs/src/lib.rs | 287 +++++++++++++++++++++ lib/memfs/src/tar.rs | 136 ++++++++++ lib/memfs/test/dir1/test1.txt | 1 + lib/memfs/test/test1.txt | 1 + lib/memfs/test/test_image.tar | Bin 0 -> 10240 bytes src/fs/mod.rs | 31 ++- src/fs/tar.rs | 310 ----------------------- src/main.rs | 5 +- 14 files changed, 1261 insertions(+), 314 deletions(-) create mode 100644 lib/memfs/Cargo.toml create mode 100644 lib/memfs/src/block.rs create mode 100644 lib/memfs/src/bvec.rs create mode 100644 lib/memfs/src/dir.rs create mode 100644 lib/memfs/src/file.rs create mode 100644 lib/memfs/src/lib.rs create mode 100644 lib/memfs/src/tar.rs create mode 100644 lib/memfs/test/dir1/test1.txt create mode 100644 lib/memfs/test/test1.txt create mode 100644 lib/memfs/test/test_image.tar delete mode 100644 src/fs/tar.rs diff --git a/Cargo.toml b/Cargo.toml index 062ebf14..7ae70ef8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } +memfs = { path = "lib/memfs" } aarch64-cpu = "9.3.1" atomic_enum = "0.2.0" diff --git a/lib/memfs/Cargo.toml b/lib/memfs/Cargo.toml new file mode 100644 index 00000000..4369a818 --- /dev/null +++ b/lib/memfs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "memfs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +vfs = { path = "../vfs" } +static_assertions = "1.1.0" + +[features] +default = [] +test-io = [] diff --git a/lib/memfs/src/block.rs b/lib/memfs/src/block.rs new file mode 100644 index 00000000..ec68ef93 --- /dev/null +++ b/lib/memfs/src/block.rs @@ -0,0 +1,260 @@ +//! Block management interfaces and structures +use core::{ + marker::PhantomData, + mem::{size_of, MaybeUninit}, + ops::{Deref, DerefMut}, + ptr::NonNull, +}; + +use yggdrasil_abi::error::Error; + +/// Number of bytes in a block +pub const SIZE: usize = 4096; +/// Maximum number of indirection pointers a block can hold +pub const ENTRY_COUNT: usize = SIZE / size_of::(); + +/// Interface for a block allocator +/// +/// # Safety +/// +/// This trait is unsafe to implement because it has to provide and accept raw data pointers of +/// exactly [SIZE]. +pub unsafe trait BlockAllocator: 'static { + /// Allocates a contiguous block of size [SIZE] + fn alloc() -> Result, Error>; + + /// Dealocates a block. + /// + /// # Safety + /// + /// Unsafe: accepts arbitrary data pointers. + unsafe fn dealloc(block: NonNull); +} + +#[repr(transparent)] +struct BlockRaw<'a, A: BlockAllocator> { + inner: Option<&'a mut [u8; SIZE]>, + _pd: PhantomData, +} + +/// Block containing file data +#[repr(transparent)] +pub struct BlockData<'a, A: BlockAllocator> { + inner: BlockRaw<'a, A>, +} + +/// Block containing indirection pointers to other blocks +#[repr(transparent)] +pub struct BlockIndirect<'a, A: BlockAllocator> { + inner: BlockRaw<'a, A>, +} + +impl<'a, A: BlockAllocator> BlockRaw<'a, A> { + const fn null() -> Self { + Self { + inner: None, + _pd: PhantomData, + } + } + + fn new() -> Result { + let ptr = A::alloc()?; + unsafe { Ok(Self::from_raw(ptr)) } + } + + unsafe fn from_raw(ptr: NonNull) -> Self { + Self { + inner: Some(&mut *(ptr.as_ptr() as *mut _)), + _pd: PhantomData, + } + } + + unsafe fn as_uninit_indirect_mut( + &mut self, + ) -> &'a mut [MaybeUninit>; ENTRY_COUNT] { + unsafe { &mut *(self.inner.as_ref().unwrap().as_ptr() as *mut _) } + } + + #[inline] + fn is_null(&self) -> bool { + self.inner.is_none() + } +} + +impl Drop for BlockRaw<'_, A> { + fn drop(&mut self) { + if let Some(inner) = self.inner.take() { + unsafe { + A::dealloc(NonNull::new_unchecked(inner as *mut _)); + } + } + } +} + +// Data block +impl<'a, A: BlockAllocator> BlockData<'a, A> { + /// Dummy entry representing a missing block + pub const fn null() -> Self { + Self { + inner: BlockRaw::null(), + } + } + + /// Allocates a new block for data + pub fn new() -> Result { + Ok(Self { + inner: BlockRaw::new()?, + }) + } + + /// Replaces self with a null block and drops any data that might've been allocated + pub fn set_null(&mut self) { + self.inner = BlockRaw::null(); + } + + /// Returns `true` if the block this structure refers to has not yet been allocated + #[inline] + pub fn is_null(&self) -> bool { + self.inner.is_null() + } +} + +impl Deref for BlockData<'_, A> { + type Target = [u8; SIZE]; + + fn deref(&self) -> &Self::Target { + self.inner.inner.as_ref().unwrap() + } +} + +impl DerefMut for BlockData<'_, A> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.inner.as_mut().unwrap() + } +} + +// Indirect block +impl<'a, A: BlockAllocator> BlockIndirect<'a, A> { + /// Dummy entry representing a missing block + pub const fn null() -> Self { + Self { + inner: BlockRaw::null(), + } + } + + /// Allocates a new indirection block + pub fn new() -> Result { + let mut inner = BlockRaw::new()?; + for item in unsafe { inner.as_uninit_indirect_mut() } { + item.write(BlockData::null()); + } + Ok(Self { inner }) + } + + /// Returns `true` if the block this structure refers to has not yet been allocated + #[inline] + pub fn is_null(&self) -> bool { + self.inner.is_null() + } +} + +impl<'a, A: BlockAllocator> Deref for BlockIndirect<'a, A> { + type Target = [BlockData<'a, A>; ENTRY_COUNT]; + + fn deref(&self) -> &Self::Target { + unsafe { &*(self.inner.inner.as_ref().unwrap().as_ptr() as *const _) } + } +} + +impl<'a, A: BlockAllocator> DerefMut for BlockIndirect<'a, A> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *(self.inner.inner.as_mut().unwrap().as_mut_ptr() as *mut _) } + } +} + +impl<'a, A: BlockAllocator> Drop for BlockIndirect<'a, A> { + fn drop(&mut self) { + if self.is_null() { + return; + } + + for item in self.iter_mut() { + item.set_null(); + } + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::Ordering; + use std::vec::Vec; + + use crate::block::{BlockData, BlockIndirect}; + + #[test] + fn block_indirect_allocation() { + test_allocator_with_counter!(A_COUNTER, A); + + const N: usize = 7; + const M: usize = 13; + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + + { + let mut indirect = Vec::new(); + + // Allocate indirect blocks + { + for _ in 0..N { + indirect.push(BlockIndirect::::new().unwrap()); + } + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), N); + + // Allocate L1 indirection blocks + { + for l1_block in indirect.iter_mut() { + for i in 0..M { + let l0_block = BlockData::new().unwrap(); + + l1_block[i] = l0_block; + } + } + } + + // N * M data blocks and N indirection blocks + assert_eq!(A_COUNTER.load(Ordering::Acquire), N * M + N); + + // Drop 1 indirect block for test + indirect.pop(); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), (N - 1) * M + (N - 1)); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + } + + #[test] + fn block_allocation() { + test_allocator_with_counter!(A_COUNTER, A); + + const N: usize = 13; + { + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + + { + let mut s = Vec::new(); + + for _ in 0..N { + let mut block = BlockData::::new().unwrap(); + block.fill(1); + s.push(block); + } + assert_eq!(A_COUNTER.load(Ordering::Acquire), N); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + } + } +} diff --git a/lib/memfs/src/bvec.rs b/lib/memfs/src/bvec.rs new file mode 100644 index 00000000..f731115b --- /dev/null +++ b/lib/memfs/src/bvec.rs @@ -0,0 +1,459 @@ +//! Block vector management structures +use core::{ + cmp::Ordering, + mem::MaybeUninit, + ops::{Index, IndexMut}, +}; + +use yggdrasil_abi::error::Error; + +use crate::block::{self, BlockAllocator, BlockData, BlockIndirect}; + +// 16.125M total +const L0_BLOCKS: usize = 32; // 128K in L0 +const L1_BLOCKS: usize = 8; // 16M in L1 + +/// Block vector for efficient in-memory files +pub struct BVec<'a, A: BlockAllocator> { + capacity: usize, + size: usize, + l0: [BlockData<'a, A>; L0_BLOCKS], + l1: [BlockIndirect<'a, A>; L1_BLOCKS], +} + +impl<'a, A: BlockAllocator> BVec<'a, A> { + /// Creates an empty block vector. + /// + /// # Note + /// + /// The function is guaranteed to make no allocations before the vector is actually written to. + pub fn new() -> Self { + let mut l0 = MaybeUninit::uninit_array(); + let mut l1 = MaybeUninit::uninit_array(); + + for it in l0.iter_mut() { + it.write(BlockData::null()); + } + + for it in l1.iter_mut() { + it.write(BlockIndirect::null()); + } + + Self { + capacity: 0, + size: 0, + l0: unsafe { MaybeUninit::array_assume_init(l0) }, + l1: unsafe { MaybeUninit::array_assume_init(l1) }, + } + } + + /// Returns the size of the data inside this vector + #[inline] + pub const fn size(&self) -> usize { + self.size + } + + fn grow_l1(&mut self, old_l1_cap: usize, new_l1_cap: usize) -> Result<(), Error> { + for i in old_l1_cap..new_l1_cap { + assert!(self.l1[i].is_null()); + self.l1[i] = BlockIndirect::new()?; + } + Ok(()) + } + + fn shrink_l1(&mut self, old_l1_cap: usize, new_l1_cap: usize) { + debug_assert!(new_l1_cap <= old_l1_cap); + for i in new_l1_cap..old_l1_cap { + assert!(!self.l1[i].is_null()); + self.l1[i] = BlockIndirect::null(); + } + } + + #[inline] + fn caps(cap: usize) -> (usize, usize) { + let l0_cap = core::cmp::min(cap, L0_BLOCKS); + let l1_cap = if cap > L0_BLOCKS { + core::cmp::min( + (cap - L0_BLOCKS + block::ENTRY_COUNT - 1) / block::ENTRY_COUNT, + L1_BLOCKS, + ) + } else { + 0 + }; + (l0_cap, l1_cap) + } + + /// Resizes the vector to hold exactly `new_capacity` data blocks + pub fn resize(&mut self, new_capacity: usize) -> Result<(), Error> { + // TODO handle L2 capacity + match new_capacity.cmp(&self.capacity) { + Ordering::Less => { + let (_, new_l1_cap) = Self::caps(new_capacity); + let (_, old_l1_cap) = Self::caps(self.capacity); + + // Shrink data blocks + for index in new_capacity..self.capacity { + let block = &mut self[index]; + assert!(!block.is_null()); + block.set_null(); + } + + // Shrink L1 blocks + self.shrink_l1(old_l1_cap, new_l1_cap); + } + + Ordering::Greater => { + let (_, new_l1_cap) = Self::caps(new_capacity); + let (_, old_l1_cap) = Self::caps(self.capacity); + + // Allocate L1 indirection blocks + assert!(new_l1_cap >= old_l1_cap); + if new_l1_cap > old_l1_cap { + self.grow_l1(old_l1_cap, new_l1_cap)?; + } + + // Grow data blocks + for index in self.capacity..new_capacity { + let block = unsafe { self.index_unchecked_mut(index) }; + assert!(block.is_null()); + *block = BlockData::new()?; + } + } + + Ordering::Equal => (), + } + + self.capacity = new_capacity; + Ok(()) + } + + fn ensure_write_capacity(&mut self, pos: usize, need_to_write: usize) -> Result<(), Error> { + let current_capacity = self.capacity * block::SIZE; + let need_capacity = + (core::cmp::max(pos + need_to_write, self.size) + block::SIZE - 1) / block::SIZE; + + if need_capacity > current_capacity { + self.resize(need_capacity) + } else { + Ok(()) + } + } + + /// Writes data to the vector, growing it if needed + pub fn write(&mut self, pos: u64, data: &[u8]) -> Result { + let mut pos = pos as usize; + // if pos > self.size { + // return Err(Error::InvalidFile); + // } + + let mut rem = data.len(); + let mut doff = 0usize; + + self.ensure_write_capacity(pos, rem)?; + + if pos + rem > self.size { + self.size = pos + rem; + } + + while rem > 0 { + let index = pos / block::SIZE; + let offset = pos % block::SIZE; + let count = core::cmp::min(rem, block::SIZE - offset); + + let block = &mut self[index]; + + let dst = &mut block[offset..offset + count]; + let src = &data[doff..doff + count]; + dst.copy_from_slice(src); + + doff += count; + pos += count; + rem -= count; + } + + Ok(doff) + } + + /// Reads data from the vector + pub fn read(&self, pos: u64, data: &mut [u8]) -> Result { + let mut pos = pos as usize; + if pos > self.size { + return Err(Error::InvalidFile); + } + + let mut rem = core::cmp::min(self.size - pos, data.len()); + let mut doff = 0usize; + + while rem > 0 { + let index = pos / block::SIZE; + let offset = pos % block::SIZE; + let count = core::cmp::min(block::SIZE - offset, rem); + + let block = &self[index]; + + let src = &block[offset..offset + count]; + let dst = &mut data[doff..doff + count]; + + dst.copy_from_slice(src); + + doff += count; + pos += count; + rem -= count; + } + + Ok(doff) + } + + unsafe fn index_unchecked(&self, mut index: usize) -> &BlockData<'a, A> { + if index < L0_BLOCKS { + return &self.l0[index]; + } + index -= L0_BLOCKS; + if index < L1_BLOCKS * block::ENTRY_COUNT { + let l1i = index / block::ENTRY_COUNT; + let l0i = index % block::ENTRY_COUNT; + + let l1r = &self.l1[l1i]; + assert!(!l1r.is_null()); + + return &l1r[l0i]; + } + + todo!(); + } + + unsafe fn index_unchecked_mut(&mut self, mut index: usize) -> &mut BlockData<'a, A> { + if index < L0_BLOCKS { + return &mut self.l0[index]; + } + index -= L0_BLOCKS; + if index < L1_BLOCKS * block::ENTRY_COUNT { + let l1i = index / block::ENTRY_COUNT; + let l0i = index % block::ENTRY_COUNT; + + let l1r = &mut self.l1[l1i]; + assert!(!l1r.is_null()); + + return &mut l1r[l0i]; + } + + todo!() + } +} + +impl<'a, A: BlockAllocator> Index for BVec<'a, A> { + type Output = BlockData<'a, A>; + + fn index(&self, index: usize) -> &Self::Output { + if index > self.capacity { + panic!( + "Block index out of bounds: capacity={}, index={}", + self.capacity, index + ); + } + + unsafe { self.index_unchecked(index) } + } +} + +impl<'a, A: BlockAllocator> IndexMut for BVec<'a, A> { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + if index > self.capacity { + panic!( + "Block index out of bounds: capacity={}, index={}", + self.capacity, index + ); + } + + unsafe { self.index_unchecked_mut(index) } + } +} + +impl<'a, A: BlockAllocator> TryFrom<&[u8]> for BVec<'a, A> { + type Error = Error; + + fn try_from(value: &[u8]) -> Result { + let mut res = Self::new(); + res.write(0, value)?; + assert_eq!(res.size(), value.len()); + Ok(res) + } +} + +#[cfg(test)] +mod bvec_allocation { + use core::sync::atomic::Ordering; + + use crate::{ + block, + bvec::{BVec, L0_BLOCKS, L1_BLOCKS}, + }; + + #[test] + fn bvec_grow_shrink() { + test_allocator_with_counter!(A_COUNTER, A); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + + { + let mut bvec = BVec::::new(); + + assert_eq!( + A_COUNTER.load(Ordering::Acquire), + 0, + "BVec should not allocate on creation" + ); + + const N: usize = 123; + bvec.resize(N).unwrap(); + + // N data blocks (12 in L0 + 111 in L1) + assert_eq!(A_COUNTER.load(Ordering::Acquire), N + 1); + + // Test the index interface + for i in 0..N { + assert!(!bvec[i].is_null(), "Index {} must be allocated", i); + } + + // Test the data structure + for i in 0..L0_BLOCKS { + assert!(!bvec.l0[i].is_null()); + } + assert!(!bvec.l1[0].is_null()); + for i in L0_BLOCKS..N { + let l1i = (i - L0_BLOCKS) / block::ENTRY_COUNT; + let l0i = (i - L0_BLOCKS) % block::ENTRY_COUNT; + + let l1r = &bvec.l1[l1i]; + assert!(!l1r.is_null()); + assert!(!l1r[l0i].is_null()); + } + + for i in 1..L1_BLOCKS { + assert!(bvec.l1[i].is_null()); + } + + // Shrink to 100 blocks, test if L1 is still allocated + const M: usize = 100; + bvec.resize(M).unwrap(); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), M + 1); + + // Test the index interface + for i in 0..M { + assert!(!bvec[i].is_null(), "Index {} must be allocated", i); + } + + // Test the data structure + for i in 0..L0_BLOCKS { + assert!(!bvec.l0[i].is_null()); + } + assert!(!bvec.l1[0].is_null()); + for i in L0_BLOCKS..M { + let l1i = (i - L0_BLOCKS) / block::ENTRY_COUNT; + let l0i = (i - L0_BLOCKS) % block::ENTRY_COUNT; + + let l1r = &bvec.l1[l1i]; + assert!(!l1r.is_null()); + assert!(!l1r[l0i].is_null()); + } + for i in M..N { + let l1i = (i - L0_BLOCKS) / block::ENTRY_COUNT; + let l0i = (i - L0_BLOCKS) % block::ENTRY_COUNT; + + let l1r = &bvec.l1[l1i]; + assert!(!l1r.is_null()); + assert!(l1r[l0i].is_null()); + } + + for i in 1..L1_BLOCKS { + assert!(bvec.l1[i].is_null()); + } + + // Shrink to 13 blocks, test if L1 got deallocated + const O: usize = 13; + bvec.resize(O).unwrap(); + + assert_eq!(A_COUNTER.load(Ordering::Acquire), O); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + } +} + +#[cfg(all(test, feature = "test-io"))] +mod bvec_io { + use crate::{block, bvec::L0_BLOCKS}; + + use super::BVec; + + #[test] + fn test_bvec_write() { + test_allocator_with_counter!(A_COUNTER, A); + + { + let data = [1, 2, 3, 4, 5]; + let mut bvec = BVec::::new(); + + // Write at 0 + assert_eq!(bvec.write(0, &data).unwrap(), data.len()); + assert_eq!(bvec.capacity, 1); + assert_eq!(bvec.size(), data.len()); + + assert_eq!(&bvec[0][..bvec.size()], &data[..]); + + // Write at 3 + assert_eq!(bvec.write(3, &data).unwrap(), data.len()); + assert_eq!(bvec.capacity, 1); + assert_eq!(bvec.size(), 3 + data.len()); + + assert_eq!(&bvec[0][..bvec.size()], &[1, 2, 3, 1, 2, 3, 4, 5]); + } + + { + let data = [5, 4, 3, 2, 1]; + let mut bvec = BVec::::new(); + + // Write at the end of L0-region + assert_eq!( + bvec.write((L0_BLOCKS * block::SIZE) as u64, &data).unwrap(), + data.len() + ); + // L0_BLOCKS + 1 L1 data block + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[L0_BLOCKS][..data.len()], &data[..]); + + // Write at zero + assert_eq!(bvec.write(0, &data).unwrap(), data.len()); + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[0][..data.len()], &data[..]); + + // Test write crossing L0 block boundary + assert_eq!( + bvec.write((block::SIZE - 3) as u64, &data).unwrap(), + data.len() + ); + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[0][block::SIZE - 3..], &[5, 4, 3]); + assert_eq!(&bvec[1][..2], &[2, 1]); + + // Test write crossing L0-L1 boundary + assert_eq!( + bvec.write((L0_BLOCKS * block::SIZE) as u64 - 2, &data) + .unwrap(), + data.len() + ); + assert_eq!(bvec.capacity, L0_BLOCKS + 1); + assert_eq!(bvec.size(), L0_BLOCKS * block::SIZE + data.len()); + + assert_eq!(&bvec[L0_BLOCKS - 1][block::SIZE - 2..], &[5, 4]); + assert_eq!(&bvec[L0_BLOCKS][..data.len()], &[3, 2, 1, 2, 1]); + } + } +} diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs new file mode 100644 index 00000000..77f4f796 --- /dev/null +++ b/lib/memfs/src/dir.rs @@ -0,0 +1,32 @@ +use core::marker::PhantomData; + +use alloc::boxed::Box; + +use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef}; +use yggdrasil_abi::error::Error; + +use crate::{block::BlockAllocator, bvec::BVec, file::FileNode}; + +pub(crate) struct DirectoryNode { + _pd: PhantomData, +} + +impl VnodeImpl for DirectoryNode { + fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + let child = Vnode::new(name, kind); + match kind { + VnodeKind::Directory => child.set_data(Box::new(Self { _pd: PhantomData })), + VnodeKind::Regular => child.set_data(Box::new(FileNode { + data: BVec::::new(), + })), + _ => todo!(), + } + Ok(child) + } +} + +impl DirectoryNode { + pub fn new() -> Self { + Self { _pd: PhantomData } + } +} diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs new file mode 100644 index 00000000..57fcf54c --- /dev/null +++ b/lib/memfs/src/file.rs @@ -0,0 +1,37 @@ +use vfs::{VnodeImpl, VnodeRef}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, +}; + +use crate::{block::BlockAllocator, bvec::BVec}; + +pub(crate) struct FileNode { + pub(crate) data: BVec<'static, A>, +} + +impl VnodeImpl for FileNode { + fn open(&mut self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { + if opts.contains(OpenOptions::APPEND) { + Ok(self.data.size() as u64) + } else { + Ok(0) + } + } + + fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + Ok(()) + } + + fn read(&mut self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + self.data.read(pos, data) + } + + fn write(&mut self, _node: &VnodeRef, pos: u64, data: &[u8]) -> Result { + self.data.write(pos, data) + } + + fn size(&mut self, _node: &VnodeRef) -> Result { + Ok(self.data.size() as u64) + } +} diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs new file mode 100644 index 00000000..db5ef084 --- /dev/null +++ b/lib/memfs/src/lib.rs @@ -0,0 +1,287 @@ +//! In-memory filesystem driver +#![no_std] +#![warn(missing_docs)] +#![allow(clippy::new_without_default)] +#![feature( + const_mut_refs, + maybe_uninit_uninit_array, + const_maybe_uninit_uninit_array, + maybe_uninit_array_assume_init +)] + +use core::{ + any::Any, + cell::{Ref, RefCell}, + marker::PhantomData, +}; + +use alloc::{boxed::Box, rc::Rc}; +use block::BlockAllocator; +use vfs::{BlockDevice, Filesystem, Vnode, VnodeKind, VnodeRef}; +use yggdrasil_abi::{error::Error, path}; + +use crate::{bvec::BVec, dir::DirectoryNode, file::FileNode, tar::TarIterator}; + +#[cfg(test)] +extern crate std; + +extern crate alloc; + +#[cfg(test)] +macro_rules! test_allocator_with_counter { + ($counter:ident, $allocator:ident) => { + static $counter: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + + struct $allocator; + + unsafe impl $crate::block::BlockAllocator for $allocator { + fn alloc() -> Result, yggdrasil_abi::error::Error> { + let b = std::boxed::Box::into_raw(std::boxed::Box::new([0; $crate::block::SIZE])); + $counter.fetch_add(1, core::sync::atomic::Ordering::Release); + Ok(unsafe { core::ptr::NonNull::new_unchecked(b as _) }) + } + + unsafe fn dealloc(block: core::ptr::NonNull) { + $counter.fetch_sub(1, core::sync::atomic::Ordering::Release); + drop(std::boxed::Box::from_raw( + block.as_ptr() as *mut [u8; $crate::block::SIZE] + )); + } + } + }; +} + +pub mod block; +pub mod bvec; + +mod dir; +mod file; +mod tar; + +/// In-memory read/write filesystem +pub struct MemoryFilesystem { + root: RefCell>, + _pd: PhantomData, +} + +impl Filesystem for MemoryFilesystem { + fn dev(self: Rc) -> Option<&'static dyn BlockDevice> { + todo!() + } + + fn data(&self) -> Option> { + todo!() + } + + fn root(self: Rc) -> Result { + Ok(self.root.borrow().clone().unwrap()) + } +} + +impl MemoryFilesystem { + fn make_path( + self: &Rc, + at: &VnodeRef, + path: &str, + kind: VnodeKind, + create: bool, + ) -> Result { + if path.is_empty() { + return Ok(at.clone()); + } + let (element, rest) = path::split_left(path); + assert!(!element.is_empty()); + + let node = at.lookup(element); + let node = match node { + Some(node) => node, + None => { + if !create { + return Err(Error::DoesNotExist); + } + + let node = self.create_node_initial(element, kind); + at.add_child(node.clone()); + + node + } + }; + + if rest.is_empty() { + Ok(node) + } else { + assert!(node.is_directory()); + self.make_path(&node, rest, kind, create) + } + } + + fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { + assert!(!name.is_empty()); + assert!(!name.contains('/')); + + let node = Vnode::new(name, kind); + node.set_fs(self.clone()); + + match kind { + VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new())), + VnodeKind::Regular => {} + _ => todo!(), + } + + node + } + + fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { + let root = Vnode::new("", VnodeKind::Directory); + root.set_fs(self.clone()); + root.set_data(Box::new(DirectoryNode::::new())); + + // 1. Create paths in tar + for item in TarIterator::new(tar_data) { + let Ok((hdr, _)) = item else { + return Err(Error::InvalidArgument); + }; + + let path = hdr.name.as_str()?.trim_matches('/'); + let (dirname, filename) = path::split_right(path); + let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; + let node = self.create_node_initial(filename, hdr.node_kind()); + + parent.add_child(node); + } + + // 2. Associate files with their data + for item in TarIterator::new(tar_data) { + let Ok((hdr, data)) = item else { + panic!("Unreachable"); + }; + if hdr.node_kind() == VnodeKind::Regular { + let data = data.unwrap(); + let path = hdr.name.as_str()?.trim_matches('/'); + let node = self.make_path(&root, path, VnodeKind::Directory, false)?; + + let bvec = BVec::::try_from(data)?; + assert_eq!(bvec.size(), data.len()); + node.set_data(Box::new(FileNode { data: bvec })); + } + } + + Ok(root) + } + + /// Constructs a filesystem tree from a tar image in memory + pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { + let fs = Rc::new(Self { + root: RefCell::new(None), + _pd: PhantomData, + }); + let root = fs.from_slice_internal(tar_data)?; + fs.root.replace(Some(root)); + + Ok(fs) + } + + /// Constructs an empty memory filesystem + pub fn empty() -> Rc { + let fs = Rc::new(Self { + root: RefCell::new(None), + _pd: PhantomData, + }); + let root = Vnode::new("", VnodeKind::Directory); + root.set_data(Box::new(DirectoryNode::::new())); + root.set_fs(fs.clone()); + fs.root.replace(Some(root)); + fs + } +} + +#[cfg(test)] +mod tests { + use std::rc::Rc; + use vfs::{Filesystem, IoContext, Read, Seek, SeekFrom, VnodeKind, Write}; + use yggdrasil_abi::io::{FileMode, OpenOptions}; + + use crate::MemoryFilesystem; + + #[test] + fn test_memfs_construction() { + fn check_file(ioctx: &IoContext, path: &str, expected_data: &str) { + let node = ioctx.find(None, path, false, false).unwrap(); + + assert_eq!(node.kind(), VnodeKind::Regular); + assert_eq!(node.size().unwrap(), expected_data.len() as u64); + + let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let mut buf = [0; 512]; + + assert_eq!( + file.borrow_mut().read(&mut buf).unwrap(), + expected_data.len() + ); + + assert_eq!(&buf[..expected_data.len()], expected_data.as_bytes()); + } + + static TEST_IMAGE: &[u8] = include_bytes!("../test/test_image.tar"); + test_allocator_with_counter!(A_COUNTER, A); + + let fs = MemoryFilesystem::::from_slice(TEST_IMAGE).unwrap(); + let root = fs.root().unwrap(); + + let ioctx = IoContext::new(root.clone()); + + assert!(Rc::ptr_eq( + &root, + &ioctx.find(None, "/", false, false).unwrap() + )); + + check_file(&ioctx, "/test1.txt", include_str!("../test/test1.txt")); + } + + #[test] + fn test_memfs_create_and_write() { + test_allocator_with_counter!(A_COUNTER, A); + + let fs = MemoryFilesystem::::empty(); + let root = fs.root().unwrap(); + + let ioctx = IoContext::new(root.clone()); + + // Create, write, seek and read file + { + // TODO CREATE option handling + root.create("test1.txt", VnodeKind::Regular).unwrap(); + + let file = ioctx + .open( + None, + "/test1.txt", + OpenOptions::WRITE | OpenOptions::READ, + FileMode::empty(), + ) + .unwrap(); + + let write_data = [1, 2, 3, 4]; + let mut read_data = [0; 512]; + + let mut file = file.borrow_mut(); + assert_eq!(file.write(&write_data).unwrap(), write_data.len()); + assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); + assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); + assert_eq!(&read_data[..write_data.len()], &write_data[..]); + } + + // Create a directory + { + // TODO read directory + root.create("dir1", VnodeKind::Directory).unwrap(); + + let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); + let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); + assert!(Rc::ptr_eq( + &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), + &node + )); + } + } +} diff --git a/lib/memfs/src/tar.rs b/lib/memfs/src/tar.rs new file mode 100644 index 00000000..a6e7afe0 --- /dev/null +++ b/lib/memfs/src/tar.rs @@ -0,0 +1,136 @@ +use vfs::VnodeKind; +use yggdrasil_abi::error::Error; + +#[repr(C)] +pub(crate) struct OctalField { + data: [u8; N], +} + +#[repr(C)] +pub(crate) struct TarString { + data: [u8; N], +} + +pub(crate) struct TarIterator<'a> { + data: &'a [u8], + offset: usize, + zero_blocks: usize, +} + +#[repr(packed)] +pub(crate) struct TarEntry { + pub name: TarString<100>, + _mode: OctalField<8>, + _uid: OctalField<8>, + _gid: OctalField<8>, + pub size: OctalField<12>, + _mtime: OctalField<12>, + _checksum: OctalField<8>, + type_: u8, + _link_name: TarString<100>, + _magic: [u8; 8], + _user: TarString<32>, + _group: TarString<32>, + _dev_major: OctalField<8>, + _dev_minor: OctalField<8>, + _prefix: TarString<155>, + __pad: [u8; 12], +} + +impl<'a> TarIterator<'a> { + pub const fn new(data: &'a [u8]) -> Self { + Self { + data, + offset: 0, + zero_blocks: 0, + } + } +} + +impl<'a> Iterator for TarIterator<'a> { + type Item = Result<(&'a TarEntry, Option<&'a [u8]>), Error>; + + fn next(&mut self) -> Option { + loop { + if self.offset + 512 > self.data.len() { + break None; + } + + let hdr_ptr = &self.data[self.offset..]; + let hdr = unsafe { &*(hdr_ptr.as_ptr() as *const TarEntry) }; + + if hdr.is_empty() { + if self.zero_blocks == 1 { + self.offset = self.data.len(); + return None; + } + self.zero_blocks += 1; + continue; + } + + let (data, size_aligned) = match hdr.type_ { + 0 | b'0' => { + let size = usize::from(&hdr.size); + + if self.offset + 512 + size > self.data.len() { + return Some(Err(Error::InvalidArgument)); + } + + let data = &self.data[self.offset + 512..self.offset + 512 + size]; + let size_aligned = (size + 511) & !511; + + (Some(data), size_aligned) + } + // Directory + b'5' => (None, 0), + _ => todo!("Unknown node kind: {}", hdr.type_), + }; + self.offset += size_aligned + 512; + + break Some(Ok((hdr, data))); + } + } +} + +impl From<&OctalField> for usize { + fn from(value: &OctalField) -> Self { + let mut acc = 0; + for i in 0..N { + if value.data[i] == 0 { + break; + } + acc <<= 3; + acc |= (value.data[i] - b'0') as usize; + } + acc + } +} + +impl TarString { + pub fn as_str(&self) -> Result<&str, Error> { + core::str::from_utf8(&self.data[..self.len()]).map_err(|_| Error::InvalidArgument) + } + + pub fn len(&self) -> usize { + for i in 0..N { + if self.data[i] == 0 { + return i; + } + } + N + } +} + +impl TarEntry { + pub fn is_empty(&self) -> bool { + self.name.data[0] == 0 + } + + pub fn node_kind(&self) -> VnodeKind { + match self.type_ { + 0 | b'0' => VnodeKind::Regular, + b'5' => VnodeKind::Directory, + _ => todo!(), + } + } +} diff --git a/lib/memfs/test/dir1/test1.txt b/lib/memfs/test/dir1/test1.txt new file mode 100644 index 00000000..ac439756 --- /dev/null +++ b/lib/memfs/test/dir1/test1.txt @@ -0,0 +1 @@ +This is another test file diff --git a/lib/memfs/test/test1.txt b/lib/memfs/test/test1.txt new file mode 100644 index 00000000..9f4b6d8b --- /dev/null +++ b/lib/memfs/test/test1.txt @@ -0,0 +1 @@ +This is a test file diff --git a/lib/memfs/test/test_image.tar b/lib/memfs/test/test_image.tar new file mode 100644 index 0000000000000000000000000000000000000000..15dd01d95793af66dc35379801bbcd144887e5cc GIT binary patch literal 10240 zcmeIz;R=E<6o%pb?kaW#bvAeOM!Eq(BOwfQLEU|>C@QFaEu!8b3^}-s=j_9#8{ER= z9Wsqd)l8GCRBq=~O2fDiv<#|+R7RZhr5LMuk9A&sH^zFf*3_H+Q1{Pwd{=+|>bHD1J_kX$WMms%qzm4m5u<0t; zRkvwp_e{L~raxlK?12CR2q1s}0tg_0 g00IagfB*srAb = OneTimeInit::new(); +/// Implementation of [memfs::block::BlockAllocator] for the kernel +pub struct FileBlockAllocator; + +unsafe impl BlockAllocator for FileBlockAllocator { + fn alloc() -> Result, Error> { + // TODO make this a static assertion + assert_eq!(block::SIZE, 4096); + let page = phys::alloc_page(PageUsage::Used)?; + Ok(unsafe { NonNull::new_unchecked(page.virtualize() as *mut _) }) + } + + unsafe fn dealloc(block: NonNull) { + let page = block.as_ptr() as usize; + assert!(page > mem::KERNEL_VIRT_OFFSET); + todo!("Release physical memory"); + } +} + /// Constructs an instance of a filesystem for given set of [MountOptions] pub fn create_filesystem(options: &MountOptions) -> Result { let fs_name = options.filesystem.unwrap(); diff --git a/src/fs/tar.rs b/src/fs/tar.rs deleted file mode 100644 index c1d2358f..00000000 --- a/src/fs/tar.rs +++ /dev/null @@ -1,310 +0,0 @@ -//! In-memory filesystem implementation -use abi::{error::Error, io::FileMode}; -use alloc::{boxed::Box, rc::Rc}; -use vfs::{Filesystem, Vnode, VnodeImpl, VnodeKind, VnodeRef}; -use yggdrasil_abi::io::OpenOptions; - -use crate::util::OneTimeInit; - -#[repr(C)] -struct OctalField { - data: [u8; N], -} - -#[repr(C)] -struct TarString { - data: [u8; N], -} - -struct TarIterator<'a> { - data: &'a [u8], - offset: usize, - zero_blocks: usize, -} - -#[repr(packed)] -struct TarEntry { - name: TarString<100>, - _mode: OctalField<8>, - _uid: OctalField<8>, - _gid: OctalField<8>, - size: OctalField<12>, - _mtime: OctalField<12>, - _checksum: OctalField<8>, - type_: u8, - _link_name: TarString<100>, - _magic: [u8; 8], - _user: TarString<32>, - _group: TarString<32>, - _dev_major: OctalField<8>, - _dev_minor: OctalField<8>, - _prefix: TarString<155>, - __pad: [u8; 12], -} - -impl<'a> TarIterator<'a> { - pub const fn new(data: &'a [u8]) -> Self { - Self { - data, - offset: 0, - zero_blocks: 0, - } - } -} - -impl<'a> Iterator for TarIterator<'a> { - type Item = Result<(&'a TarEntry, Option<&'a [u8]>), Error>; - - fn next(&mut self) -> Option { - loop { - if self.offset + 512 > self.data.len() { - break None; - } - - let hdr_ptr = &self.data[self.offset..]; - let hdr = unsafe { &*(hdr_ptr.as_ptr() as *const TarEntry) }; - - if hdr.is_empty() { - if self.zero_blocks == 1 { - self.offset = self.data.len(); - return None; - } - self.zero_blocks += 1; - continue; - } - - let (data, size_aligned) = match hdr.type_ { - 0 | b'0' => { - let size = usize::from(&hdr.size); - - if self.offset + 512 + size > self.data.len() { - return Some(Err(Error::InvalidArgument)); - } - - let data = &self.data[self.offset + 512..self.offset + 512 + size]; - let size_aligned = (size + 511) & !511; - - (Some(data), size_aligned) - } - // Directory - b'5' => (None, 0), - _ => todo!("Unknown node kind: {}", hdr.type_), - }; - self.offset += size_aligned + 512; - - break Some(Ok((hdr, data))); - } - } -} - -impl From<&OctalField> for usize { - fn from(value: &OctalField) -> Self { - let mut acc = 0; - for i in 0..N { - if value.data[i] == 0 { - break; - } - acc <<= 3; - acc |= (value.data[i] - b'0') as usize; - } - acc - } -} - -impl TarString { - pub fn as_str(&self) -> Result<&str, Error> { - core::str::from_utf8(&self.data[..self.len()]).map_err(|_| Error::InvalidArgument) - } - - pub fn len(&self) -> usize { - for i in 0..N { - if self.data[i] == 0 { - return i; - } - } - N - } -} - -impl TarEntry { - pub fn is_empty(&self) -> bool { - self.name.data[0] == 0 - } - - pub fn node_kind(&self) -> VnodeKind { - match self.type_ { - 0 | b'0' => VnodeKind::Regular, - b'5' => VnodeKind::Directory, - _ => todo!(), - } - } -} - -/// tar-image based in-memory filesystem -pub struct TarFilesystem { - root: OneTimeInit, -} - -impl Filesystem for TarFilesystem { - fn dev(self: Rc) -> Option<&'static dyn vfs::BlockDevice> { - todo!() - } - - fn root(self: Rc) -> Result { - self.root.try_get().cloned().ok_or(Error::DoesNotExist) - } - - fn data(&self) -> Option> { - todo!() - } -} - -struct DirInode; -struct RegularInode { - data: &'static [u8], -} - -impl VnodeImpl for DirInode { - fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { - let child = Vnode::new(name, kind); - match kind { - VnodeKind::Directory => child.set_data(Box::new(DirInode)), - VnodeKind::Regular => (), - _ => todo!(), - } - Ok(child) - } -} - -impl VnodeImpl for RegularInode { - fn open(&mut self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { - if opts.contains(OpenOptions::WRITE) { - panic!("TODO: tarfs write"); - } - - Ok(0) - } - - fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { - Ok(()) - } - - fn read(&mut self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { - let pos = pos as usize; - if pos > self.data.len() { - return Err(Error::InvalidFile); - } - let rem = core::cmp::min(self.data.len() - pos, data.len()); - data[..rem].copy_from_slice(&self.data[pos..pos + rem]); - Ok(rem) - } - - fn size(&mut self, _node: &VnodeRef) -> Result { - Ok(self.data.len() as u64) - } -} - -impl TarFilesystem { - fn make_path( - self: &Rc, - at: &VnodeRef, - path: &str, - kind: VnodeKind, - create: bool, - ) -> Result { - debugln!("make_path {:?}", path); - if path.is_empty() { - return Ok(at.clone()); - } - let (element, rest) = abi::path::split_left(path); - assert!(!element.is_empty()); - - let node = at.lookup(element); - let node = match node { - Some(node) => node, - None => { - if !create { - debugln!("path {:?} does not exist", path); - return Err(Error::DoesNotExist); - } - - infoln!("Create {:?}", element); - let node = self.create_node_initial(element, kind); - at.add_child(node.clone()); - - node - } - }; - - if rest.is_empty() { - Ok(node) - } else { - assert!(node.is_directory()); - self.make_path(&node, rest, kind, create) - } - } - - fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { - assert!(!name.is_empty()); - assert!(!name.contains('/')); - - let node = Vnode::new(name, kind); - node.set_fs(self.clone()); - - match kind { - VnodeKind::Directory => node.set_data(Box::new(DirInode)), - VnodeKind::Regular => {} - _ => todo!(), - } - - node - } - - fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { - let root = Vnode::new("", VnodeKind::Directory); - root.set_fs(self.clone()); - root.set_data(Box::new(DirInode)); - - // 1. Create paths in tar - for item in TarIterator::new(tar_data) { - let Ok((hdr, _)) = item else { - warnln!("Tar image is truncated"); - return Err(Error::InvalidArgument); - }; - - let path = hdr.name.as_str()?.trim_matches('/'); - infoln!("path = {:?}", path); - let (dirname, filename) = abi::path::split_right(path); - let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; - let node = self.create_node_initial(filename, hdr.node_kind()); - - parent.add_child(node); - } - - // 2. Associate files with their data - for item in TarIterator::new(tar_data) { - let Ok((hdr, data)) = item else { - panic!("Unreachable"); - }; - if hdr.node_kind() == VnodeKind::Regular { - let data = data.unwrap(); - let path = hdr.name.as_str()?.trim_matches('/'); - let node = self.make_path(&root, path, VnodeKind::Directory, false)?; - node.set_data(Box::new(RegularInode { data })); - } - } - - Ok(root) - } - - /// Constructs a filesystem tree from a tar image in memory - pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { - let fs = Rc::new(TarFilesystem { - root: OneTimeInit::new(), - }); - let root = fs.from_slice_internal(tar_data)?; - fs.root.init(root); - - Ok(fs) - } -} diff --git a/src/main.rs b/src/main.rs index 91f56fd7..9ed9318e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,8 @@ use abi::{ error::Error, io::{FileMode, OpenOptions}, }; -use fs::{tar::TarFilesystem, INITRD_DATA}; +use fs::{FileBlockAllocator, INITRD_DATA}; +use memfs::MemoryFilesystem; use task::process::Process; use vfs::{Filesystem, IoContext, VnodeRef}; @@ -42,7 +43,7 @@ pub mod util; fn setup_root() -> Result { let initrd_data = INITRD_DATA.get(); - let fs = TarFilesystem::from_slice(initrd_data.data)?; + let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); fs.root() } From 867edf774864d2a5b74234246ee74cde430b44ba Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 22 Jul 2023 00:39:08 +0300 Subject: [PATCH 013/211] ***: fix incorrect tlbi + add Read/OpenDirectory --- lib/memfs/src/dir.rs | 16 +++++- lib/vfs/src/file.rs | 102 ++++++++++++++++++++++++++++++++++++-- lib/vfs/src/lib.rs | 12 +++++ lib/vfs/src/node.rs | 35 ++++++++++++- src/arch/aarch64/mod.rs | 14 +++--- src/arch/aarch64/table.rs | 26 +++++++--- src/mem/phys/manager.rs | 8 +++ src/mem/phys/mod.rs | 10 +++- src/proc/elf.rs | 2 +- src/proc/exec.rs | 10 ++-- src/syscall/arg.rs | 30 +++++++++++ src/syscall/mod.rs | 47 +++++++++++++++--- 12 files changed, 277 insertions(+), 35 deletions(-) diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs index 77f4f796..995c3f33 100644 --- a/lib/memfs/src/dir.rs +++ b/lib/memfs/src/dir.rs @@ -2,8 +2,11 @@ use core::marker::PhantomData; use alloc::boxed::Box; -use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef}; -use yggdrasil_abi::error::Error; +use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, +}; use crate::{block::BlockAllocator, bvec::BVec, file::FileNode}; @@ -23,6 +26,15 @@ impl VnodeImpl for DirectoryNode { } Ok(child) } + + fn open( + &mut self, + _node: &VnodeRef, + _opts: OpenOptions, + _mode: FileMode, + ) -> Result { + Ok(DIR_POSITION_FROM_CACHE) + } } impl DirectoryNode { diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs index a7a8e9d5..c12442cb 100644 --- a/lib/vfs/src/file.rs +++ b/lib/vfs/src/file.rs @@ -1,12 +1,15 @@ -use core::cell::RefCell; +use core::{cell::RefCell, mem::MaybeUninit}; use alloc::rc::Rc; use bitflags::bitflags; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{ + error::Error, + io::{DirectoryEntry, FileType}, +}; use crate::{ node::{VnodeKind, VnodeRef}, - Read, Seek, SeekFrom, Write, + Read, ReadDirectory, Seek, SeekFrom, Write, DIR_POSITION_FROM_CACHE, }; bitflags! { @@ -18,13 +21,28 @@ bitflags! { pub type FileRef = Rc>; +enum DirectoryPosition { + DiskPosition(u64), + // TODO not the best implementation, but at least somewhat safe? + CachePosition(usize), + Dot, + DotDot, + End, +} + pub struct NormalFile { vnode: VnodeRef, pos: u64, } +pub struct Directory { + vnode: VnodeRef, + pos: DirectoryPosition, +} + pub enum FileInner { Normal(NormalFile), + Directory(Directory), } pub struct File { @@ -39,6 +57,20 @@ impl File { flags, })) } + + pub fn directory(vnode: VnodeRef, pos: u64) -> FileRef { + let pos = if pos == DIR_POSITION_FROM_CACHE { + // Read from cache + DirectoryPosition::Dot + } else { + // Reading from a "physical" directory + todo!() + }; + Rc::new(RefCell::new(Self { + inner: FileInner::Directory(Directory { vnode, pos }), + flags: FileFlags::READ, + })) + } } impl Write for File { @@ -55,6 +87,7 @@ impl Write for File { } Ok(count) } + FileInner::Directory(_) => unimplemented!(), } } } @@ -73,6 +106,7 @@ impl Read for File { } Ok(count) } + FileInner::Directory(_) => Err(Error::IsADirectory), } } } @@ -98,16 +132,78 @@ impl Seek for File { Ok(pos) } + FileInner::Directory(_) => todo!(), } } } +impl ReadDirectory for File { + fn read_dir_entries( + &mut self, + entries: &mut [MaybeUninit], + ) -> Result { + let FileInner::Directory(inner) = &mut self.inner else { + return Err(Error::NotADirectory); + }; + + let mut nread = 0; + let mut rem = entries.len(); + + while rem != 0 { + let mut entry = DirectoryEntry { + name: [0; 256], + ty: FileType::File, + }; + + let next_position = match inner.pos { + DirectoryPosition::End => { + break; + } + DirectoryPosition::Dot => { + entry.name[0] = b'.'; + entry.ty = FileType::Directory; + DirectoryPosition::DotDot + } + DirectoryPosition::DotDot => { + entry.name[..2].copy_from_slice(b".."); + entry.ty = FileType::Directory; + DirectoryPosition::CachePosition(0) + } + DirectoryPosition::CachePosition(index) => { + let child = inner.vnode.child_at(index); + + if let Some(child) = child { + let child_name = child.name(); + entry.name[..child_name.len()].copy_from_slice(child_name.as_bytes()); + entry.ty = FileType::from(child.kind()); + DirectoryPosition::CachePosition(index + 1) + } else { + break; + } + } + DirectoryPosition::DiskPosition(_) => todo!(), + }; + inner.pos = next_position; + + entries[nread].write(entry); + + nread += 1; + rem -= 1; + } + + Ok(nread) + } +} + impl Drop for File { fn drop(&mut self) { match &mut self.inner { FileInner::Normal(inner) => { inner.vnode.close().ok(); } + FileInner::Directory(inner) => { + inner.vnode.close().ok(); + } } } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 10937641..e382838f 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,6 +1,9 @@ #![no_std] +use core::mem::MaybeUninit; + use yggdrasil_abi::error::Error; +use yggdrasil_abi::io::DirectoryEntry; extern crate alloc; @@ -24,6 +27,8 @@ pub use fs::Filesystem; pub use ioctx::IoContext; pub use node::{Vnode, VnodeDump, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; +pub const DIR_POSITION_FROM_CACHE: u64 = u64::MAX; + #[derive(Debug)] pub enum SeekFrom { Start(u64), @@ -47,6 +52,13 @@ pub trait Seek { fn seek(&mut self, pos: SeekFrom) -> Result; } +pub trait ReadDirectory { + fn read_dir_entries( + &mut self, + entries: &mut [MaybeUninit], + ) -> Result; +} + fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Result<(), Error> { while !buf.is_empty() { match this.read(buf) { diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index 82088b3a..0a37b2ec 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -11,12 +11,13 @@ use alloc::{ }; use yggdrasil_abi::{ error::Error, - io::{FileMode, OpenOptions}, + io::{FileMode, FileType, OpenOptions}, }; use crate::{ file::{File, FileFlags, FileRef}, fs::Filesystem, + DIR_POSITION_FROM_CACHE, }; pub type VnodeRef = Rc; @@ -163,6 +164,11 @@ impl Vnode { parent_borrow.children.push(child); } + pub fn child_at(self: &VnodeRef, index: usize) -> Option { + let tree = self.tree.borrow(); + tree.children.get(index).cloned() + } + pub fn dump(&self, f: &mut fmt::Formatter<'_>, depth: usize, indent: bool) -> fmt::Result { if indent { for _ in 0..depth { @@ -244,11 +250,25 @@ impl Vnode { } } + pub fn open_directory(self: &VnodeRef) -> Result { + if !self.is_directory() { + return Err(Error::IsADirectory); + } + + if let Some(ref mut data) = *self.data() { + let pos = data.open(self, OpenOptions::READ, FileMode::empty())?; + Ok(File::directory(self.clone(), pos)) + } else { + // TODO: some options here? + Ok(File::directory(self.clone(), DIR_POSITION_FROM_CACHE)) + } + } + pub fn close(self: &VnodeRef) -> Result<(), Error> { if let Some(ref mut data) = *self.data() { data.close(self) } else { - todo!() + Ok(()) } } @@ -377,3 +397,14 @@ impl fmt::Debug for VnodeDump { self.node.dump(f, 0, true) } } + +impl From for FileType { + fn from(value: VnodeKind) -> Self { + match value { + VnodeKind::Regular => Self::File, + VnodeKind::Directory => Self::Directory, + VnodeKind::Block => Self::Block, + VnodeKind::Char => Self::Char, + } + } +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 02385a64..2525daf1 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -232,13 +232,13 @@ pub fn kernel_main(dtb_phys: usize) -> ! { devfs::init(); PLATFORM.init(true).unwrap(); - let dt = ARCHITECTURE.dt.get(); - if let Err(e) = smp::start_ap_cores(dt) { - errorln!( - "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", - e - ); - } + // let dt = ARCHITECTURE.dt.get(); + // if let Err(e) = smp::start_ap_cores(dt) { + // errorln!( + // "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", + // e + // ); + // } Cpu::init_ipi_queues(); diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index 0a6fd586..c28e0b89 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -5,8 +5,10 @@ use core::{ sync::atomic::{AtomicU8, Ordering}, }; +use aarch64_cpu::registers::DAIF; use abi::error::Error; use bitflags::bitflags; +use tock_registers::interfaces::Readable; use crate::mem::{ phys::{self, PageUsage}, @@ -77,6 +79,8 @@ bitflags! { const AP_BOTH_READWRITE = 1 << 6; /// For page/block mappings, only allows read access for EL0/EL1 const AP_BOTH_READONLY = 3 << 6; + + const NON_GLOBAL = 1 << 11; } } @@ -301,8 +305,6 @@ impl FixedTables { } self.device_l3i += count; - tlb_flush_vaae1(virt); - Ok(virt) } } @@ -315,6 +317,9 @@ impl VirtualMemoryManager for AddressSpace { len: usize, attrs: PageAttributes, ) -> Result { + assert_eq!(DAIF.read(DAIF::I), 1); + + assert_eq!(len, 1); if hint.is_some() { todo!(); } @@ -341,12 +346,17 @@ impl VirtualMemoryManager for AddressSpace { } fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> { + assert_eq!(DAIF.read(DAIF::I), 1); + for page in (addr..addr + len).step_by(0x1000) { - let Some(_phys) = self.translate(page) else { + let Some(phys) = self.translate(page) else { todo!(); }; self.write_entry(page, PageEntry::INVALID, true)?; + unsafe { + phys::free_page(phys); + } } Ok(()) @@ -402,6 +412,8 @@ impl AddressSpace { } l3[l3i] = entry; + tlb_flush_vaae1(virt); + Ok(()) } @@ -416,11 +428,11 @@ impl AddressSpace { } } -/// Flushes the virtual address from TLB -pub fn tlb_flush_vaae1(page: usize) { - assert_eq!(page & 0xFFF, 0); +/// Flush a virtual address from EL1/EL0 TLB for all ASIDs +pub fn tlb_flush_vaae1(mut page: usize) { + page >>= 12; unsafe { - core::arch::asm!("tlbi vaae1, {addr}", addr = in(reg) page); + core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); } } diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index d38a6d16..34651016 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -78,6 +78,14 @@ impl PhysicalMemoryManager { Err(Error::OutOfMemory) } + pub unsafe fn free_page(&mut self, addr: usize) { + assert!(addr > self.offset); + let index = (addr - self.offset) / 0x1000; + let page = &mut self.pages[index]; + assert_eq!(page.usage, PageUsage::Used); + page.usage = PageUsage::Available; + } + /// Marks a previously reserved page as available. /// /// # Panics diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 1a2dc4ab..c5b6272b 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -10,6 +10,7 @@ use crate::{ phys::reserved::{is_reserved, reserve_region}, ConvertAddress, KERNEL_PHYS_BASE, }, + sync::IrqSafeSpinlock, util::OneTimeInit, }; @@ -64,7 +65,8 @@ impl PhysicalMemoryRegion { } /// Global physical memory manager -pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); +pub static PHYSICAL_MEMORY: OneTimeInit> = + OneTimeInit::new(); /// Allocates a single physical page from the global manager pub fn alloc_page(usage: PageUsage) -> Result { @@ -79,6 +81,10 @@ pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result>( it: I, ) -> Option<(usize, usize)> { @@ -182,7 +188,7 @@ pub unsafe fn init_from_iter + Clone>( infoln!("{} available pages", page_count); - PHYSICAL_MEMORY.init(Spinlock::new(manager)); + PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager)); Ok(()) } diff --git a/src/proc/elf.rs b/src/proc/elf.rs index c74b6d26..4eac3817 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -73,7 +73,7 @@ where (_, 0) => PageAttributes::AP_BOTH_READWRITE, (0, _) => PageAttributes::AP_BOTH_READONLY, (_, _) => PageAttributes::AP_BOTH_READWRITE, - }; + } | PageAttributes::NON_GLOBAL; let dst_page_off = addr & 0xFFF; let dst_page_aligned = addr & !0xFFF; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 7e038341..7b506315 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -32,7 +32,11 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er let phys_page = phys::alloc_page(PageUsage::Used)?; // TODO check if this doesn't overwrite anything - space.map_page(virt, phys_page, PageAttributes::AP_BOTH_READWRITE)?; + space.map_page( + virt, + phys_page, + PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + )?; let write = unsafe { phys_page.virtualize() }; @@ -70,7 +74,7 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er fn setup_binary(space: AddressSpace, entry: usize, args: &[&str]) -> Result, Error> { const USER_STACK_PAGES: usize = 8; - let virt_stack_base = 0x10000000; + let virt_stack_base = 0x3000000; // 0x1000 of guard page let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; @@ -79,7 +83,7 @@ fn setup_binary(space: AddressSpace, entry: usize, args: &[&str]) -> Result(base: usize, len: usize) -> Result<&'a [u8], Error> { @@ -18,6 +19,9 @@ pub(super) fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8] pub(super) fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error> { let slice = arg_buffer_ref(base, len)?; + if slice.contains(&0) { + todo!("Incorrect ptr: {:#x}", base); + } Ok(core::str::from_utf8(slice).unwrap()) } @@ -30,3 +34,29 @@ pub(super) fn arg_user_ref<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { let value = unsafe { core::mem::transmute::<_, &'a T>(addr) }; Ok(value) } + +pub(super) fn arg_user_slice_mut<'a, T: Sized>( + base: usize, + count: usize, +) -> Result<&'a mut [T], Error> { + let layout = Layout::array::(count).unwrap(); + + if base % layout.align() != 0 { + todo!("Misaligned buffer"); + } + + if base + layout.size() > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + + let slice = unsafe { core::slice::from_raw_parts_mut(base as *mut _, count) }; + Ok(slice) +} + +pub(super) fn arg_option_fd(raw: u32) -> Option { + if raw == u32::MAX { + None + } else { + Some(RawFd(raw)) + } +} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 6c39c17c..966f369f 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -1,12 +1,12 @@ //! System function call handlers -use core::time::Duration; +use core::{mem::MaybeUninit, sync::atomic::AtomicUsize, time::Duration}; use abi::{ error::Error, - io::{FileMode, OpenOptions, RawFd}, + io::{DirectoryEntry, FileMode, OpenOptions, RawFd}, syscall::SyscallFunction, }; -use vfs::{Read, Write}; +use vfs::{Read, ReadDirectory, Write}; use yggdrasil_abi::{ error::SyscallResult, io::{MountOptions, UnmountOptions}, @@ -58,10 +58,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - let addr = space.allocate(None, len / 0x1000, PageAttributes::AP_BOTH_READWRITE); - debugln!("mmap({:#x}) = {:x?}", len, addr); - - addr + space.allocate( + None, + len / 0x1000, + PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + ) } SyscallFunction::UnmapMemory => { let addr = args[0] as usize; @@ -74,7 +75,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - debugln!("munmap({:#x}, {:#x})", addr, len); space.deallocate(addr, len)?; Ok(0) @@ -154,6 +154,37 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } + SyscallFunction::OpenDirectory => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + + let proc = Process::current(); + let mut io = proc.io.lock(); + + // TODO handle at + assert!(at.is_none()); + + let node = io.ioctx().find(None, path, true, true)?; + let file = node.open_directory()?; + let fd = io.place_file(file)?; + + Ok(fd.0 as usize) + } + SyscallFunction::ReadDirectory => { + let fd = RawFd(args[0] as u32); + let buffer = arg_user_slice_mut::>( + args[1] as usize, + args[2] as usize, + )?; + + let proc = Process::current(); + let mut io = proc.io.lock(); + + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); + + file_borrow.read_dir_entries(buffer) + } } } From 8642556b336ca5ff34ae283efc8b147a637299c4 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 22 Jul 2023 00:45:14 +0300 Subject: [PATCH 014/211] ***: fix warnings in check/clippy --- lib/vfs/src/file.rs | 4 +++- src/arch/aarch64/table.rs | 2 ++ src/mem/phys/manager.rs | 5 +++++ src/mem/phys/mod.rs | 6 +++++- src/syscall/mod.rs | 4 ++-- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs index c12442cb..a1b68b15 100644 --- a/lib/vfs/src/file.rs +++ b/lib/vfs/src/file.rs @@ -22,11 +22,13 @@ bitflags! { pub type FileRef = Rc>; enum DirectoryPosition { - DiskPosition(u64), // TODO not the best implementation, but at least somewhat safe? CachePosition(usize), Dot, DotDot, + #[allow(dead_code)] + DiskPosition(u64), + #[allow(dead_code)] End, } diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index c28e0b89..d5514a07 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -80,6 +80,8 @@ bitflags! { /// For page/block mappings, only allows read access for EL0/EL1 const AP_BOTH_READONLY = 3 << 6; + /// Indicates the mapping is unique to a specific ASID (important for proper TLB + /// maintenance) const NON_GLOBAL = 1 << 11; } } diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 34651016..7739cf8b 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -78,6 +78,11 @@ impl PhysicalMemoryManager { Err(Error::OutOfMemory) } + /// Deallocates a physical memory page. + /// + /// # Safety + /// + /// `addr` must be a page-aligned physical address previously allocated by this implementation. pub unsafe fn free_page(&mut self, addr: usize) { assert!(addr > self.offset); let index = (addr - self.offset) / 0x1000; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index c5b6272b..3d664187 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -2,7 +2,6 @@ use core::{iter::StepBy, mem::size_of, ops::Range}; use abi::error::Error; -use spinning_top::Spinlock; use crate::{ absolute_address, @@ -81,6 +80,11 @@ pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result Result )?; let proc = Process::current(); - let mut io = proc.io.lock(); + let io = proc.io.lock(); let file = io.file(fd)?; let mut file_borrow = file.borrow_mut(); From ba3819ee8eacc7e3f6c526c4cc8136fc02d5fbfc Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 22 Jul 2023 16:21:31 +0300 Subject: [PATCH 015/211] fs: implement GetMetadata, CreateDirectory, Remove --- lib/memfs/src/dir.rs | 21 ++++++++++- lib/memfs/src/file.rs | 10 ++++- lib/vfs/src/char.rs | 10 ++++- lib/vfs/src/node.rs | 88 ++++++++++++++++++++++++++++++++++++------- src/fs/devfs.rs | 31 ++++++++++++++- src/syscall/arg.rs | 10 +++++ src/syscall/mod.rs | 63 ++++++++++++++++++++++++++++++- 7 files changed, 212 insertions(+), 21 deletions(-) diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs index 995c3f33..bb7571ba 100644 --- a/lib/memfs/src/dir.rs +++ b/lib/memfs/src/dir.rs @@ -5,7 +5,7 @@ use alloc::boxed::Box; use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; use yggdrasil_abi::{ error::Error, - io::{FileMode, OpenOptions}, + io::{FileAttr, FileMode, FileType, OpenOptions}, }; use crate::{block::BlockAllocator, bvec::BVec, file::FileNode}; @@ -15,7 +15,7 @@ pub(crate) struct DirectoryNode { } impl VnodeImpl for DirectoryNode { - fn create(&mut self, _at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { let child = Vnode::new(name, kind); match kind { VnodeKind::Directory => child.set_data(Box::new(Self { _pd: PhantomData })), @@ -24,6 +24,7 @@ impl VnodeImpl for DirectoryNode { })), _ => todo!(), } + child.set_fs(at.fs().unwrap()); Ok(child) } @@ -35,6 +36,22 @@ impl VnodeImpl for DirectoryNode { ) -> Result { Ok(DIR_POSITION_FROM_CACHE) } + + fn remove(&mut self, _at: &VnodeRef, _name: &str) -> Result<(), Error> { + Ok(()) + } + + fn size(&mut self, node: &VnodeRef) -> Result { + Ok(node.children().len() as u64) + } + + fn metadata(&mut self, _node: &VnodeRef) -> Result { + Ok(FileAttr { + size: 0, + mode: FileMode::default_dir(), + ty: FileType::Directory, + }) + } } impl DirectoryNode { diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs index 57fcf54c..2ac22277 100644 --- a/lib/memfs/src/file.rs +++ b/lib/memfs/src/file.rs @@ -1,7 +1,7 @@ use vfs::{VnodeImpl, VnodeRef}; use yggdrasil_abi::{ error::Error, - io::{FileMode, OpenOptions}, + io::{FileAttr, FileMode, FileType, OpenOptions}, }; use crate::{block::BlockAllocator, bvec::BVec}; @@ -34,4 +34,12 @@ impl VnodeImpl for FileNode { fn size(&mut self, _node: &VnodeRef) -> Result { Ok(self.data.size() as u64) } + + fn metadata(&mut self, _node: &VnodeRef) -> Result { + Ok(FileAttr { + size: self.data.size() as u64, + mode: FileMode::default_file(), + ty: FileType::File, + }) + } } diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index 6da845f9..adbd946a 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -1,6 +1,6 @@ use yggdrasil_abi::{ error::Error, - io::{FileMode, OpenOptions}, + io::{FileAttr, FileMode, FileType, OpenOptions}, }; use crate::{node::VnodeImpl, VnodeRef}; @@ -41,4 +41,12 @@ impl VnodeImpl for CharDeviceWrapper { fn write(&mut self, _node: &VnodeRef, _pos: u64, data: &[u8]) -> Result { self.device.write(true, data) } + + fn metadata(&mut self, _node: &VnodeRef) -> Result { + Ok(FileAttr { + size: 0, + ty: FileType::Char, + mode: FileMode::default_file(), + }) + } } diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index 0a37b2ec..e76328c3 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -1,5 +1,5 @@ use core::{ - cell::{RefCell, RefMut}, + cell::{Ref, RefCell, RefMut}, fmt, }; @@ -11,7 +11,7 @@ use alloc::{ }; use yggdrasil_abi::{ error::Error, - io::{FileMode, FileType, OpenOptions}, + io::{FileAttr, FileMode, FileType, OpenOptions}, }; use crate::{ @@ -49,33 +49,37 @@ pub struct Vnode { target: RefCell>, } +#[allow(unused_variables)] pub trait VnodeImpl { - fn create(&mut self, _at: &VnodeRef, _name: &str, _kind: VnodeKind) -> Result { + fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { Err(Error::NotImplemented) } - fn open( - &mut self, - _node: &VnodeRef, - _opts: OpenOptions, - _mode: FileMode, - ) -> Result { + fn remove(&mut self, at: &VnodeRef, name: &str) -> Result<(), Error> { Err(Error::NotImplemented) } - fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + fn open(&mut self, node: &VnodeRef, opts: OpenOptions, mode: FileMode) -> Result { Err(Error::NotImplemented) } - fn read(&mut self, _node: &VnodeRef, _pos: u64, _data: &mut [u8]) -> Result { + fn close(&mut self, node: &VnodeRef) -> Result<(), Error> { Err(Error::NotImplemented) } - fn write(&mut self, _node: &VnodeRef, _pos: u64, _data: &[u8]) -> Result { + fn read(&mut self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { Err(Error::NotImplemented) } - fn size(&mut self, _node: &VnodeRef) -> Result { + fn write(&mut self, node: &VnodeRef, pos: u64, data: &[u8]) -> Result { + Err(Error::NotImplemented) + } + + fn size(&mut self, node: &VnodeRef) -> Result { + Err(Error::NotImplemented) + } + + fn metadata(&mut self, node: &VnodeRef) -> Result { Err(Error::NotImplemented) } } @@ -132,6 +136,11 @@ impl Vnode { } } + #[inline] + pub fn is_root(&self) -> bool { + self.tree.borrow().parent.is_none() + } + pub fn set_data(&self, data: Box) { self.data.borrow_mut().replace(data); } @@ -164,11 +173,25 @@ impl Vnode { parent_borrow.children.push(child); } + pub fn remove_child(self: &VnodeRef, name: &str) { + self.children_mut().retain(|node| node.name() != name); + } + pub fn child_at(self: &VnodeRef, index: usize) -> Option { let tree = self.tree.borrow(); tree.children.get(index).cloned() } + pub fn children(&self) -> Ref> { + let tree = self.tree.borrow(); + Ref::map(tree, |t| &t.children) + } + + pub fn children_mut(&self) -> RefMut> { + let tree = self.tree.borrow_mut(); + RefMut::map(tree, |t| &mut t.children) + } + pub fn dump(&self, f: &mut fmt::Formatter<'_>, depth: usize, indent: bool) -> fmt::Result { if indent { for _ in 0..depth { @@ -298,6 +321,37 @@ impl Vnode { } } + pub fn remove(self: &VnodeRef, node: VnodeRef, recurse: bool) -> Result<(), Error> { + let name = node.name(); + + if self.kind != VnodeKind::Directory { + todo!(); + } + if name.contains('/') { + todo!(); + } + + if node.kind() == VnodeKind::Directory { + if recurse { + todo!(); + } + + // Check if remove target is not empty + if node.size()? != 0 { + return Err(Error::DirectoryNotEmpty); + } + } + + if let Some(ref mut data) = *self.data() { + data.remove(self, name)?; + } + + // Unlink node from cache + self.remove_child(name); + + Ok(()) + } + pub fn write(self: &VnodeRef, pos: u64, buf: &[u8]) -> Result { if self.kind == VnodeKind::Directory { todo!(); @@ -371,6 +425,14 @@ impl Vnode { Ok(()) } + + pub fn metadata(self: &VnodeRef) -> Result { + if let Some(ref mut data) = *self.data() { + data.metadata(self) + } else { + todo!() + } + } } impl fmt::Debug for Vnode { diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index e6631fa8..cd018a1b 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -1,9 +1,14 @@ //! Device virtual file system use core::sync::atomic::{AtomicUsize, Ordering}; -use abi::error::Error; +use abi::{ + error::Error, + io::{FileAttr, FileMode, FileType, OpenOptions}, +}; use alloc::{boxed::Box, format, string::String}; -use vfs::{CharDevice, CharDeviceWrapper, Vnode, VnodeKind, VnodeRef}; +use vfs::{ + CharDevice, CharDeviceWrapper, Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE, +}; use crate::util::OneTimeInit; @@ -14,11 +19,33 @@ pub enum CharDeviceType { TtySerial, } +struct DevfsDirectory; + +impl VnodeImpl for DevfsDirectory { + fn open( + &mut self, + _node: &VnodeRef, + _opts: OpenOptions, + _mode: FileMode, + ) -> Result { + Ok(DIR_POSITION_FROM_CACHE) + } + + fn metadata(&mut self, _node: &VnodeRef) -> Result { + Ok(FileAttr { + size: 0, + mode: FileMode::default_dir(), + ty: FileType::Directory, + }) + } +} + static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); /// Sets up the device filesystem pub fn init() { let node = Vnode::new("", VnodeKind::Directory); + node.set_data(Box::new(DevfsDirectory)); DEVFS_ROOT.init(node); } diff --git a/src/syscall/arg.rs b/src/syscall/arg.rs index 9128ebcc..7bede0f1 100644 --- a/src/syscall/arg.rs +++ b/src/syscall/arg.rs @@ -35,6 +35,16 @@ pub(super) fn arg_user_ref<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { Ok(value) } +pub(super) fn arg_user_mut<'a, T: Sized>(addr: usize) -> Result<&'a mut T, Error> { + let layout = Layout::new::(); + if addr % layout.align() != 0 { + todo!("Misaligned argument"); + } + // TODO check that addr actually points to mapped (and user-accessible) memory + let value = unsafe { core::mem::transmute::<_, &'a mut T>(addr) }; + Ok(value) +} + pub(super) fn arg_user_slice_mut<'a, T: Sized>( base: usize, count: usize, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index dc647c44..aa10d081 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -3,10 +3,11 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, - io::{DirectoryEntry, FileMode, OpenOptions, RawFd}, + io::{DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd}, syscall::SyscallFunction, }; -use vfs::{Read, ReadDirectory, Write}; +use alloc::rc::Rc; +use vfs::{Read, ReadDirectory, VnodeKind, Write}; use yggdrasil_abi::{ error::SyscallResult, io::{MountOptions, UnmountOptions}, @@ -185,6 +186,64 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result file_borrow.read_dir_entries(buffer) } + SyscallFunction::CreateDirectory => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let mode = FileMode::from(args[3] as u32); + + let proc = Process::current(); + let mut io = proc.io.lock(); + + // TODO handle at + assert!(at.is_none()); + + let (parent, name) = abi::path::split_right(path); + let parent_node = io.ioctx().find(None, parent, true, true)?; + parent_node.create(name, VnodeKind::Directory)?; + + Ok(0) + } + SyscallFunction::Remove => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let recurse = args[3] != 0; + + let proc = Process::current(); + let mut io = proc.io.lock(); + + assert!(at.is_none()); + + let node = io.ioctx().find(None, path, false, false)?; + + if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { + todo!(); + } + + let (_, filename) = abi::path::split_right(path); + let parent = node.parent(); + + parent.remove(node, recurse)?; + + Ok(0) + } + SyscallFunction::GetMetadata => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let buffer = arg_user_mut::>(args[3] as usize)?; + let follow = args[4] != 0; + + let proc = Process::current(); + let mut io = proc.io.lock(); + + assert!(at.is_none()); + + let node = io.ioctx().find(None, path, follow, true)?; + + let metadata = node.metadata()?; + buffer.write(metadata); + + Ok(0) + } } } From 461bfb2791318c61565d5e64e0b589f97a0ed81e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 22 Jul 2023 18:27:40 +0300 Subject: [PATCH 016/211] fs: implement Seek syscall --- lib/memfs/src/lib.rs | 4 +- lib/vfs/src/file.rs | 7 ++ lib/vfs/src/lib.rs | 9 +- src/proc/elf.rs | 4 +- src/syscall/mod.rs | 203 ++++++++++++++++++++++++------------------- 5 files changed, 125 insertions(+), 102 deletions(-) diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index db5ef084..1215211e 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -198,8 +198,8 @@ impl MemoryFilesystem { #[cfg(test)] mod tests { use std::rc::Rc; - use vfs::{Filesystem, IoContext, Read, Seek, SeekFrom, VnodeKind, Write}; - use yggdrasil_abi::io::{FileMode, OpenOptions}; + use vfs::{Filesystem, IoContext, Read, Seek, VnodeKind, Write}; + use yggdrasil_abi::io::{FileMode, OpenOptions, SeekFrom}; use crate::MemoryFilesystem; diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs index a1b68b15..2c8c4cc2 100644 --- a/lib/vfs/src/file.rs +++ b/lib/vfs/src/file.rs @@ -73,6 +73,13 @@ impl File { flags: FileFlags::READ, })) } + + pub fn node(&self) -> Result { + match &self.inner { + FileInner::Normal(inner) => Ok(inner.vnode.clone()), + FileInner::Directory(inner) => Ok(inner.vnode.clone()), + } + } } impl Write for File { diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index e382838f..7c91c256 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -3,7 +3,7 @@ use core::mem::MaybeUninit; use yggdrasil_abi::error::Error; -use yggdrasil_abi::io::DirectoryEntry; +use yggdrasil_abi::io::{DirectoryEntry, SeekFrom}; extern crate alloc; @@ -29,13 +29,6 @@ pub use node::{Vnode, VnodeDump, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; pub const DIR_POSITION_FROM_CACHE: u64 = u64::MAX; -#[derive(Debug)] -pub enum SeekFrom { - Start(u64), - End(i64), - Current(i64), -} - pub trait Write { fn write(&mut self, data: &[u8]) -> Result; } diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 4eac3817..01f2b3fa 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -4,8 +4,8 @@ use elf::{ endian::AnyEndian, ElfStream, ParseError, }; -use vfs::{FileRef, Read, Seek, SeekFrom}; -use yggdrasil_abi::error::Error; +use vfs::{FileRef, Read, Seek}; +use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::mem::{ phys::{self, PageUsage}, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index aa10d081..1af0f4d5 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -3,11 +3,11 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, - io::{DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd}, + io::{DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, syscall::SyscallFunction, }; use alloc::rc::Rc; -use vfs::{Read, ReadDirectory, VnodeKind, Write}; +use vfs::{Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; use yggdrasil_abi::{ error::SyscallResult, io::{MountOptions, UnmountOptions}, @@ -16,13 +16,37 @@ use yggdrasil_abi::{ use crate::{ fs, mem::table::{PageAttributes, VirtualMemoryManager}, - proc::wait, + proc::{io::ProcessIo, wait}, + sync::IrqSafeSpinlockGuard, task::process::Process, }; mod arg; use arg::*; +fn run_with_io) -> T>(f: F) -> T { + let proc = Process::current(); + let io = proc.io.lock(); + + f(io) +} + +fn run_with_io_at< + T, + F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, +>( + at: Option, + f: F, +) -> Result { + let proc = Process::current(); + let io = proc.io.lock(); + let at = at + .map(|fd| io.file(fd).and_then(|f| f.borrow().node())) + .transpose()?; + + f(at, io) +} + fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { match func { SyscallFunction::DebugTrace => { @@ -84,92 +108,87 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - let proc = Process::current(); - let io = proc.io.lock(); - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); + run_with_io(|io| { + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); - file_borrow.write(data) + file_borrow.write(data) + }) } SyscallFunction::Read => { let fd = RawFd(args[0] as u32); let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - let proc = Process::current(); - let io = proc.io.lock(); - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); + run_with_io(|io| { + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); - file_borrow.read(data) + file_borrow.read(data) + }) } SyscallFunction::Open => { - let path = arg_user_str(args[0] as usize, args[1] as usize)?; - let opts = OpenOptions::from(args[2] as u32); - let mode = FileMode::from(args[3] as u32); + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let opts = OpenOptions::from(args[3] as u32); + let mode = FileMode::from(args[4] as u32); - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io_at(at, |at, mut io| { + debugln!("run_with_io_at {:?}", at); + let file = io.ioctx().open(at, path, opts, mode)?; + let fd = io.place_file(file)?; - let file = io.ioctx().open(None, path, opts, mode)?; - let fd = io.place_file(file)?; - - Ok(fd.0 as usize) + Ok(fd.0 as usize) + }) } SyscallFunction::Close => { let fd = RawFd(args[0] as u32); - let proc = Process::current(); - let mut io = proc.io.lock(); - io.close_file(fd)?; - Ok(0) + run_with_io(|mut io| { + io.close_file(fd)?; + Ok(0) + }) } SyscallFunction::Mount => { let options = arg_user_ref::(args[0] as usize)?; - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io(|mut io| { + let target_node = io.ioctx().find(None, options.target, true, false)?; + if !target_node.is_directory() { + return Err(Error::NotADirectory); + } - let target_node = io.ioctx().find(None, options.target, true, false)?; - if !target_node.is_directory() { - return Err(Error::NotADirectory); - } + let fs_root = fs::create_filesystem(options)?; - let fs_root = fs::create_filesystem(options)?; + target_node.mount(fs_root)?; - target_node.mount(fs_root)?; + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - - Ok(0) + Ok(0) + }) } SyscallFunction::Unmount => { let options = arg_user_ref::(args[0] as usize)?; - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io(|mut io| { + let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; + mountpoint.unmount_target()?; - let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; - mountpoint.unmount_target()?; + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - - Ok(0) + Ok(0) + }) } SyscallFunction::OpenDirectory => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io_at(at, |at, mut io| { + let node = io.ioctx().find(at, path, true, true)?; + let file = node.open_directory()?; + let fd = io.place_file(file)?; - // TODO handle at - assert!(at.is_none()); - - let node = io.ioctx().find(None, path, true, true)?; - let file = node.open_directory()?; - let fd = io.place_file(file)?; - - Ok(fd.0 as usize) + Ok(fd.0 as usize) + }) } SyscallFunction::ReadDirectory => { let fd = RawFd(args[0] as u32); @@ -178,53 +197,45 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result args[2] as usize, )?; - let proc = Process::current(); - let io = proc.io.lock(); + run_with_io(|io| { + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); - - file_borrow.read_dir_entries(buffer) + file_borrow.read_dir_entries(buffer) + }) } SyscallFunction::CreateDirectory => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; let mode = FileMode::from(args[3] as u32); - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io_at(at, |at, mut io| { + let (parent, name) = abi::path::split_right(path); + let parent_node = io.ioctx().find(at, parent, true, true)?; + parent_node.create(name, VnodeKind::Directory)?; - // TODO handle at - assert!(at.is_none()); - - let (parent, name) = abi::path::split_right(path); - let parent_node = io.ioctx().find(None, parent, true, true)?; - parent_node.create(name, VnodeKind::Directory)?; - - Ok(0) + Ok(0) + }) } SyscallFunction::Remove => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; let recurse = args[3] != 0; - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io_at(at, |at, mut io| { + let node = io.ioctx().find(at, path, false, false)?; - assert!(at.is_none()); + if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { + todo!(); + } - let node = io.ioctx().find(None, path, false, false)?; + let (_, filename) = abi::path::split_right(path); + let parent = node.parent(); - if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { - todo!(); - } + parent.remove(node, recurse)?; - let (_, filename) = abi::path::split_right(path); - let parent = node.parent(); - - parent.remove(node, recurse)?; - - Ok(0) + Ok(0) + }) } SyscallFunction::GetMetadata => { let at = arg_option_fd(args[0] as u32); @@ -232,17 +243,29 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let buffer = arg_user_mut::>(args[3] as usize)?; let follow = args[4] != 0; - let proc = Process::current(); - let mut io = proc.io.lock(); + run_with_io_at(at, |at, mut io| { + let node = if path.is_empty() { + at.ok_or(Error::InvalidArgument)? + } else { + io.ioctx().find(None, path, follow, true)? + }; - assert!(at.is_none()); + let metadata = node.metadata()?; + buffer.write(metadata); - let node = io.ioctx().find(None, path, follow, true)?; + Ok(0) + }) + } + SyscallFunction::Seek => { + let fd = RawFd(args[0] as u32); + let pos = SeekFrom::from(args[1]); - let metadata = node.metadata()?; - buffer.write(metadata); + run_with_io(|io| { + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); - Ok(0) + file_borrow.seek(pos).map(|v| v as usize) + }) } } } From b31e066590c94c1d05569421536130b21444da4e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 25 Jul 2023 10:49:11 +0300 Subject: [PATCH 017/211] proc: WaitProcess + Spawn --- src/main.rs | 2 +- src/proc/io.rs | 4 +++ src/proc/wait.rs | 2 ++ src/syscall/mod.rs | 63 ++++++++++++++++++++++++++++++++++++++++++--- src/task/process.rs | 60 +++++++++++++++++++++++++++++++++++++----- 5 files changed, 121 insertions(+), 10 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9ed9318e..653a2198 100644 --- a/src/main.rs +++ b/src/main.rs @@ -67,7 +67,7 @@ pub fn kernel_main() { let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); { - let user_init = proc::exec::load_elf(file, &["/init"]).unwrap(); + let user_init = proc::exec::load_elf(file, &["/init", "xxx"]).unwrap(); let mut io = user_init.io.lock(); io.set_ioctx(ioctx); drop(io); diff --git a/src/proc/io.rs b/src/proc/io.rs index 3dda86b5..eba36cbe 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -66,4 +66,8 @@ impl ProcessIo { pub fn ioctx(&mut self) -> &mut IoContext { self.ioctx.as_mut().unwrap() } + + pub fn handle_exit(&mut self) { + self.files.clear(); + } } diff --git a/src/proc/wait.rs b/src/proc/wait.rs index 70c2de0f..b08ed1f4 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -32,6 +32,8 @@ struct Timeout { deadline: Duration, } +pub static PROCESS_EXIT_WAIT: Wait = Wait::new("process-exit"); + impl Wait { /// Constructs a new wait notification channel pub const fn new(name: &'static str) -> Self { diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 1af0f4d5..070bdedd 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,10 +4,11 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, + process::{SpawnOption, SpawnOptions}, syscall::SyscallFunction, }; use alloc::rc::Rc; -use vfs::{Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; +use vfs::{IoContext, Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; use yggdrasil_abi::{ error::SyscallResult, io::{MountOptions, UnmountOptions}, @@ -16,9 +17,13 @@ use yggdrasil_abi::{ use crate::{ fs, mem::table::{PageAttributes, VirtualMemoryManager}, - proc::{io::ProcessIo, wait}, + proc::{ + self, + io::ProcessIo, + wait::{self, PROCESS_EXIT_WAIT}, + }, sync::IrqSafeSpinlockGuard, - task::process::Process, + task::{process::Process, ProcessId}, }; mod arg; @@ -267,6 +272,58 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result file_borrow.seek(pos).map(|v| v as usize) }) } + SyscallFunction::Spawn => { + let options = arg_user_ref::(args[0] as usize)?; + + debugln!("Spawn {:#?}", options); + + run_with_io(|mut io| { + let node = io.ioctx().find(None, options.program, true, true)?; + + // Setup a new process from the file + let file = node.open(OpenOptions::READ, FileMode::empty())?; + let child = proc::exec::load_elf(file, options.arguments)?; + let pid = child.id() as u32; + + // Inherit root from the creator + let child_ioctx = IoContext::new(io.ioctx().root().clone()); + + { + let mut child_io = child.io.lock(); + + for opt in options.optional { + if let SpawnOption::InheritFile { source, child } = opt { + let src_file = io.file(*source)?; + child_io.set_file(*child, src_file)?; + } + } + } + + child.enqueue_somewhere(); + + Ok(pid as _) + }) + } + SyscallFunction::WaitProcess => { + let pid = args[0] as ProcessId; + let status = arg_user_mut::(args[1] as _)?; + debugln!("WaitProcess #{}", pid); + + let target = Process::get(pid).ok_or(Error::DoesNotExist)?; + + loop { + if let Some(exit_status) = target.get_exit_status() { + *status = exit_status; + break Ok(0); + } + + // Suspend and wait for signal + match PROCESS_EXIT_WAIT.wait(Some(Duration::from_secs(3))) { + Ok(()) | Err(Error::TimedOut) => (), + Err(_) => todo!(), + } + } + } } } diff --git a/src/task/process.rs b/src/task/process.rs index e0c25def..dc5ef608 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -9,7 +9,7 @@ use crate::{ mem::table::AddressSpace, proc::{ io::ProcessIo, - wait::{Wait, WaitStatus}, + wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, }, sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, @@ -34,6 +34,7 @@ pub enum ProcessState { struct ProcessInner { pending_wait: Option<&'static Wait>, wait_status: WaitStatus, + exit_status: i32, } /// Process data and state structure @@ -65,6 +66,7 @@ impl Process { inner: IrqSafeSpinlock::new(ProcessInner { pending_wait: None, wait_status: WaitStatus::Done, + exit_status: 0, }), space, io: IrqSafeSpinlock::new(ProcessIo::new()), @@ -115,6 +117,11 @@ impl Process { self.space.as_ref().unwrap() } + /// Returns the address space of the task, if one is set + pub fn get_address_space(&self) -> Option<&AddressSpace> { + self.space.as_ref() + } + /// Selects a suitable CPU queue and submits the process for execution. /// /// # Panics @@ -212,28 +219,69 @@ impl Process { self.inner.lock().wait_status = status; } + pub fn get_exit_status(&self) -> Option { + if self.state() == ProcessState::Terminated { + Some(self.inner.lock().exit_status) + } else { + None + } + } + /// Returns the [Process] currently executing on local CPU, None if idling. pub fn get_current() -> Option> { let queue = Cpu::local().queue(); queue.current_process() } + pub fn get(pid: ProcessId) -> Option> { + PROCESSES.lock().get(pid).cloned() + } + /// Wraps [Process::get_current()] for cases when the caller is absolutely sure there is a /// running process (e.g. the call itself comes from a process). pub fn current() -> Rc { Self::get_current().unwrap() } - /// Terminate a process - pub fn exit(&self, status: usize) { - let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); + pub fn handle_exit(&self) { + // Queue lock is still held + assert_eq!(self.state(), ProcessState::Terminated); + // TODO cancel Wait if a process was killed while suspended? + { + let inner = self.inner.lock(); + debugln!( + "Handling exit of #{} with status {}", + self.id(), + inner.exit_status + ); + + // TODO cleanup address space + // if let Some(space) = self.get_address_space() { + // } + + self.io.lock().handle_exit(); + } + + // Notify any waiters we're done + PROCESS_EXIT_WAIT.wakeup_all(); + } + + /// Terminate a process + pub fn exit(&self, status: i32) { + self.inner.lock().exit_status = status; + let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); debugln!("Process {} exited with code {}", self.id(), status); match current_state { - ProcessState::Suspended => (), + ProcessState::Suspended => { + todo!(); + } ProcessState::Ready => todo!(), - ProcessState::Running => unsafe { Cpu::local().queue().yield_cpu() }, + ProcessState::Running => { + self.handle_exit(); + unsafe { Cpu::local().queue().yield_cpu() } + } ProcessState::Terminated => todo!(), } } From 3b324df4032af9be968a5c0ef4c094b91988eea7 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 25 Jul 2023 16:47:00 +0300 Subject: [PATCH 018/211] proc: implement exc/async signals --- src/arch/aarch64/context.rs | 40 +++++++ src/arch/aarch64/exception.rs | 20 +++- src/main.rs | 4 +- src/mem/mod.rs | 43 +++++++ src/proc/io.rs | 1 + src/proc/wait.rs | 1 + src/syscall/mod.rs | 35 +++++- src/task/process.rs | 205 +++++++++++++++++++++++++++++++--- src/task/sched.rs | 18 ++- 9 files changed, 339 insertions(+), 28 deletions(-) diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 8120a97f..f5eb63be 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -23,6 +23,8 @@ struct TaskContextInner { /// AArch64 implementation of a task context pub struct TaskContext { inner: UnsafeCell, + stack_base: usize, + stack_size: usize, } const COMMON_CONTEXT_SIZE: usize = 8 * 14; @@ -100,6 +102,8 @@ impl TaskContext { Ok(Self { inner: UnsafeCell::new(TaskContextInner { sp }), + stack_base, + stack_size: KERNEL_TASK_PAGES * 0x1000, }) } @@ -140,9 +144,45 @@ impl TaskContext { Ok(Self { inner: UnsafeCell::new(TaskContextInner { sp }), + stack_base, + stack_size: USER_TASK_PAGES * 0x1000, }) } + /// Constructs an empty context. One usage is to create an uninitialized context and have a + /// process set it up later (e.g. for a signal handler). + pub fn empty() -> Result { + const USER_TASK_PAGES: usize = 8; + let stack_base = + unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; + let sp = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000).build(); + + Ok(Self { + inner: UnsafeCell::new(TaskContextInner { sp }), + stack_base, + stack_size: USER_TASK_PAGES * 0x1000, + }) + } + + /// Initializes the context with a signal entry information. + /// + /// # Safety + /// + /// The function accepts arbitrary addresses and is unsafe to call. + pub unsafe fn setup_signal_entry(&self, entry: usize, arg: usize, ttbr0: usize, ustack: usize) { + let mut stack = StackBuilder::new(self.stack_base, self.stack_size); + + stack.push(entry); + stack.push(arg); + stack.push(0); + stack.push(ustack); + + stack.init_common(__aarch64_task_enter_user as _, ttbr0); + + let sp = stack.build(); + (*self.inner.get()).sp = sp; + } + /// Starts execution of `self` task on local CPU. /// /// # Safety diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index e9fd41a1..21d52c2d 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -2,6 +2,7 @@ use core::{arch::global_asm, fmt}; use aarch64_cpu::registers::{ELR_EL1, ESR_EL1, FAR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; +use abi::process::Signal; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ @@ -129,11 +130,26 @@ extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { } // BRK in AArch64 0b111100 => { - Process::current().exit(1); - panic!("Cannot return here"); + let proc = Process::current(); + warnln!("Process #{} hit a breakpoint", proc.id()); + proc.raise_exception(Signal::Aborted, None); } _ => { let iss = esr_el1 & 0x1FFFFFF; + if ec == 0b100100 { + // Data abort from lower level + let proc = Process::current(); + warnln!( + "Data abort in #{} at {:#x} with address {:#x}", + proc.id(), + ELR_EL1.get(), + FAR_EL1.get() + ); + + proc.raise_exception(Signal::MemoryAccessViolation, Some(FAR_EL1.get() as _)); + return; + } + dump_irrecoverable_exception(frame, ec, iss); panic!("Irrecoverable exception"); diff --git a/src/main.rs b/src/main.rs index 653a2198..f686a790 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ optimize_attribute, const_trait_impl, maybe_uninit_slice, + arbitrary_self_types, linked_list_cursors )] #![allow(clippy::new_without_default)] @@ -18,6 +19,7 @@ extern crate yggdrasil_abi as abi; use abi::{ error::Error, io::{FileMode, OpenOptions}, + process::ExitCode, }; use fs::{FileBlockAllocator, INITRD_DATA}; use memfs::MemoryFilesystem; @@ -74,5 +76,5 @@ pub fn kernel_main() { user_init.enqueue_somewhere(); } - Process::current().exit(0); + Process::current().exit(ExitCode::SUCCESS); } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index a84e6da3..1ab54a72 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,9 +1,13 @@ //! Memory management utilities and types +use core::mem::size_of; + use crate::{ arch::{Architecture, ArchitectureImpl, PlatformImpl}, device::platform::Platform, }; +use self::table::AddressSpace; + pub mod device; pub mod heap; pub mod phys; @@ -43,6 +47,24 @@ pub unsafe trait ConvertAddress { unsafe fn physicalize(self) -> Self; } +/// Helper trait to allow cross-address space access to pointers +pub trait ForeignPointer { + /// Perform a volatile pointer write without dropping the old value. + /// + /// # Panics + /// + /// The function panics if any of the following conditions is met: + /// + /// * The address of the pointer is not mapped in the `space`. + /// * The pointer is not writable. + /// * The pointer is misaligned. + /// + /// # Safety + /// + /// As this function allows direct memory writes, it is inherently unsafe. + unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); +} + unsafe impl ConvertAddress for usize { #[inline(always)] unsafe fn virtualize(self) -> Self { @@ -88,3 +110,24 @@ unsafe impl ConvertAddress for *const T { (self as usize).physicalize() as Self } } + +impl ForeignPointer for T { + unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { + // TODO check align + let addr = self as usize; + let start_page = addr & !0xFFF; + let end_page = (addr + size_of::()) & !0xFFF; + let page_offset = addr & 0xFFF; + + if start_page != end_page { + todo!("Foreign pointer write crossed a page boundary"); + } + + let phys_page = space + .translate(start_page) + .expect("Address is not mapped in the target address space"); + + let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; + virt_ptr.write_volatile(value); + } +} diff --git a/src/proc/io.rs b/src/proc/io.rs index eba36cbe..7efc9f29 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -67,6 +67,7 @@ impl ProcessIo { self.ioctx.as_mut().unwrap() } + /// Cleans up I/O context after the process exits pub fn handle_exit(&mut self) { self.files.clear(); } diff --git a/src/proc/wait.rs b/src/proc/wait.rs index b08ed1f4..297260df 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -32,6 +32,7 @@ struct Timeout { deadline: Duration, } +/// Common notification channel for tasks waiting on process exits pub static PROCESS_EXIT_WAIT: Wait = Wait::new("process-exit"); impl Wait { diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 070bdedd..5634f90b 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,7 +4,7 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, - process::{SpawnOption, SpawnOptions}, + process::{ExitCode, Signal, SpawnOption, SpawnOptions}, syscall::SyscallFunction, }; use alloc::rc::Rc; @@ -75,7 +75,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } SyscallFunction::Exit => { - Process::current().exit(args[0] as _); + let code = ExitCode::from(args[0] as i32); + Process::current().exit(code); panic!(); } SyscallFunction::MapMemory => { @@ -212,7 +213,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::CreateDirectory => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let mode = FileMode::from(args[3] as u32); + let _mode = FileMode::from(args[3] as u32); run_with_io_at(at, |at, mut io| { let (parent, name) = abi::path::split_right(path); @@ -234,7 +235,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - let (_, filename) = abi::path::split_right(path); let parent = node.parent(); parent.remove(node, recurse)?; @@ -291,7 +291,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { let mut child_io = child.io.lock(); + child_io.set_ioctx(child_ioctx); + for opt in options.optional { + #[allow(irrefutable_let_patterns)] if let SpawnOption::InheritFile { source, child } = opt { let src_file = io.file(*source)?; child_io.set_file(*child, src_file)?; @@ -306,7 +309,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } SyscallFunction::WaitProcess => { let pid = args[0] as ProcessId; - let status = arg_user_mut::(args[1] as _)?; + let status = arg_user_mut::(args[1] as _)?; debugln!("WaitProcess #{}", pid); let target = Process::get(pid).ok_or(Error::DoesNotExist)?; @@ -324,6 +327,27 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } } } + SyscallFunction::SendSignal => { + let pid = args[0] as u32; + let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + + let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; + target.try_set_signal(signal)?; + + Ok(0) + } + SyscallFunction::SetSignalEntry => { + let entry = args[0] as usize; + let sp = args[1] as usize; + + Process::current().set_signal_entry(entry, sp); + + Ok(0) + } + SyscallFunction::ExitSignal => { + Process::current().exit_signal(); + } + SyscallFunction::GetPid => Ok(Process::current().id()), } } @@ -333,5 +357,6 @@ pub fn raw_syscall_handler(func: u64, args: &[u64]) -> u64 { todo!("Undefined syscall: {}", func); }; + // debugln!("raw_syscall_handler {:?}, {:x?}", func, args); syscall_handler(func, args).into_syscall_result() as u64 } diff --git a/src/task/process.rs b/src/task/process.rs index dc5ef608..4e9e956f 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,12 +1,21 @@ //! Process data structures -use core::sync::atomic::{AtomicU32, Ordering}; +use core::{ + mem::size_of, + sync::atomic::{AtomicU32, Ordering}, +}; +use aarch64_cpu::registers::DAIF; +use abi::{ + error::Error, + process::{ExitCode, Signal, SignalEntryData}, +}; use alloc::rc::Rc; use atomic_enum::atomic_enum; +use tock_registers::interfaces::Readable; use crate::{ arch::aarch64::{context::TaskContext, cpu::Cpu}, - mem::table::AddressSpace, + mem::{table::AddressSpace, ForeignPointer}, proc::{ io::ProcessIo, wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, @@ -31,22 +40,36 @@ pub enum ProcessState { Terminated, } +/// Describes signal entry information +#[derive(Debug)] +pub struct SignalEntry { + entry: usize, + stack: usize, +} + struct ProcessInner { pending_wait: Option<&'static Wait>, wait_status: WaitStatus, exit_status: i32, + + signal_entry: Option, } /// Process data and state structure pub struct Process { - context: TaskContext, + normal_context: TaskContext, + signal_context: TaskContext, // Process state info id: OneTimeInit, state: AtomicProcessState, cpu_id: AtomicU32, - inner: IrqSafeSpinlock, space: Option, + pending_signal: AtomicU32, + async_signal: AtomicU32, + + inner: IrqSafeSpinlock, + /// I/O state of the task pub io: IrqSafeSpinlock, } @@ -57,18 +80,26 @@ impl Process { /// # Note /// /// Has side-effect of allocating a new PID for itself. - pub fn new_with_context(space: Option, context: TaskContext) -> Rc { + pub fn new_with_context(space: Option, normal_context: TaskContext) -> Rc { let this = Rc::new(Self { - context, + normal_context, + signal_context: TaskContext::empty().unwrap(), + id: OneTimeInit::new(), state: AtomicProcessState::new(ProcessState::Suspended), cpu_id: AtomicU32::new(0), + space, + async_signal: AtomicU32::new(0), + pending_signal: AtomicU32::new(0), + inner: IrqSafeSpinlock::new(ProcessInner { pending_wait: None, wait_status: WaitStatus::Done, exit_status: 0, + + signal_entry: None, }), - space, + io: IrqSafeSpinlock::new(ProcessIo::new()), }); @@ -79,8 +110,22 @@ impl Process { } /// Returns a reference to the inner architecture-specific [TaskContext]. - pub fn context(&self) -> &TaskContext { - &self.context + pub fn current_context(&self) -> &TaskContext { + if self.pending_signal.load(Ordering::Acquire) == 0 { + &self.normal_context + } else { + &self.signal_context + } + } + + /// Returns a signal currently handled by the process, if any + pub fn pending_signal(&self) -> Option { + let value = self.pending_signal.load(Ordering::Acquire); + if value == 0 { + None + } else { + Some(Signal::try_from(value).unwrap()) + } } /// Returns this process' ID @@ -219,9 +264,10 @@ impl Process { self.inner.lock().wait_status = status; } - pub fn get_exit_status(&self) -> Option { + /// Returns an exit code if the process exited, [None] if it didn't + pub fn get_exit_status(&self) -> Option { if self.state() == ProcessState::Terminated { - Some(self.inner.lock().exit_status) + Some(ExitCode::from(self.inner.lock().exit_status)) } else { None } @@ -233,6 +279,7 @@ impl Process { queue.current_process() } + /// Returns a process by its ID pub fn get(pid: ProcessId) -> Option> { PROCESSES.lock().get(pid).cloned() } @@ -243,6 +290,7 @@ impl Process { Self::get_current().unwrap() } + /// Handles the cleanup of an exited process pub fn handle_exit(&self) { // Queue lock is still held assert_eq!(self.state(), ProcessState::Terminated); @@ -250,10 +298,11 @@ impl Process { // TODO cancel Wait if a process was killed while suspended? { let inner = self.inner.lock(); + let exit_status = ExitCode::from(inner.exit_status); debugln!( - "Handling exit of #{} with status {}", + "Handling exit of #{} with status {:?}", self.id(), - inner.exit_status + exit_status ); // TODO cleanup address space @@ -268,10 +317,10 @@ impl Process { } /// Terminate a process - pub fn exit(&self, status: i32) { - self.inner.lock().exit_status = status; + pub fn exit(&self, status: ExitCode) { + self.inner.lock().exit_status = status.into(); let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); - debugln!("Process {} exited with code {}", self.id(), status); + infoln!("Process {} exited with code {:?}", self.id(), status); match current_state { ProcessState::Suspended => { @@ -285,6 +334,130 @@ impl Process { ProcessState::Terminated => todo!(), } } + + unsafe fn setup_signal_context(&self, signal: Signal, data: Option) { + let inner = self.inner.lock(); + let space = self.address_space(); + let signum = u32::from(signal); + assert_eq!(DAIF.read(DAIF::I), 1); + + let old_pending_signal = self.pending_signal.swap(signum, Ordering::AcqRel); + + if old_pending_signal != 0 { + todo!(); + } + // TODO: interrupt wait, if any + + let Some(entry) = inner.signal_entry.as_ref() else { + drop(inner); + warnln!( + "Process #{} has no signal entry set up, terminating it", + self.id() + ); + // Treat this as memory access violation + self.exit(ExitCode::BySignal(Signal::MemoryAccessViolation)); + panic!(); + }; + + let ttbr0 = self.address_space().physical_address(); + + // Place signal data on the provided stack + // TODO check the validity of the stack + let ustack_sp = unsafe { + let size_aligned = (size_of::() + 0x1F) & !0x1F; + let ptr = (entry.stack - size_aligned) as *mut SignalEntryData; + + // This function can be called from non-current context, so assume it is in a foreign + // address space + ptr.write_foreign_volatile(space, SignalEntryData { signal, data }); + + ptr as _ + }; + + unsafe { + self.signal_context + .setup_signal_entry(entry.entry, ustack_sp, ttbr0, ustack_sp); + } + + debugln!( + "Set up signal context for #{}: entry={:#x}, sp={:#x}", + self.id(), + entry.entry, + ustack_sp + ); + } + + /// Raises a synchronous exception signal for the current process. + /// + /// Must be called on the current process. + pub fn try_raise_exception(&self, signal: Signal, data: Option) -> Result<(), Error> { + assert_eq!(self.state(), ProcessState::Running); + assert_eq!(DAIF.read(DAIF::I), 1); + + if !signal.is_synchronous() { + return Err(Error::InvalidArgument); + } + + unsafe { + self.setup_signal_context(signal, data); + self.signal_context.switch(&self.normal_context); + } + + Ok(()) + } + + /// Raises a synchronous exception signal and terminates the process if it cannot currently + /// handle it. + /// + /// Must be called on the current process. + pub fn raise_exception(&self, signal: Signal, data: Option) { + if let Err(err) = self.try_raise_exception(signal, data) { + warnln!( + "Could not raise {:?} for #{} ({:?}), terminating it", + signal, + self.id(), + err + ); + + self.exit(ExitCode::BySignal(signal)); + } + } + + /// Raises an asynchronous signal for the target process + pub fn try_set_signal(&self, signal: Signal) -> Result<(), Error> { + assert_eq!(DAIF.read(DAIF::I), 1); + assert_ne!(self.state(), ProcessState::Running); + + if signal.is_synchronous() { + return Err(Error::InvalidArgument); + } + + unsafe { + self.setup_signal_context(signal, None); + } + + Ok(()) + } + + /// Sets up signal entry information so the kernel can initialize a signal context for the + /// process + pub fn set_signal_entry(&self, entry: usize, stack: usize) { + let mut inner = self.inner.lock(); + inner.signal_entry.replace(SignalEntry { entry, stack }); + } + + /// Leaves the current signal handler. + /// + /// Must be called for a current process. + pub fn exit_signal(&self) -> ! { + assert_eq!(self.state(), ProcessState::Running); + let old_pending_signal = self.pending_signal.swap(0, Ordering::AcqRel); + assert_ne!(old_pending_signal, 0); + self.async_signal.store(0, Ordering::Release); + + unsafe { self.normal_context.switch(&self.signal_context) } + panic!("Cannot return here"); + } } impl Drop for Process { diff --git a/src/task/sched.rs b/src/task/sched.rs index cd6e5f91..5dd24545 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -152,21 +152,26 @@ impl CpuQueue { drop(inner); let (from, _from_rc) = if let Some(current) = current.as_ref() { - (current.context(), Rc::strong_count(current)) + (current.current_context(), Rc::strong_count(current)) } else { (&self.idle, 0) }; let (to, _to_rc) = if let Some(next) = next.as_ref() { next.set_running(Cpu::local_id()); - (next.context(), Rc::strong_count(next)) + (next.current_context(), Rc::strong_count(next)) } else { (&self.idle, 0) }; // log_print_raw!(crate::debug::LogLevel::Info, "{}: ", Cpu::local_id()); // if let Some(from) = current.as_ref() { - // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id()); + // log_print_raw!( + // crate::debug::LogLevel::Info, + // "{} ({:?})", + // from.id(), + // from.pending_signal() + // ); // } else { // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); // } @@ -174,7 +179,12 @@ impl CpuQueue { // log_print_raw!(crate::debug::LogLevel::Info, " -> "); // if let Some(to) = next.as_ref() { - // log_print_raw!(crate::debug::LogLevel::Info, "{}", to.id()); + // log_print_raw!( + // crate::debug::LogLevel::Info, + // "{} ({:?})", + // to.id(), + // to.pending_signal() + // ); // } else { // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); // } From 40e00d1a09aa899494b31730e69e3992a016bee3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 25 Jul 2023 17:28:02 +0300 Subject: [PATCH 019/211] proc: More extensive user pointer validation --- src/mem/mod.rs | 133 ++++++++++++++++++++++++++++++++++++++++++++- src/syscall/arg.rs | 55 +++++++------------ 2 files changed, 151 insertions(+), 37 deletions(-) diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 1ab54a72..da8d09ab 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,5 +1,7 @@ //! Memory management utilities and types -use core::mem::size_of; +use core::{alloc::Layout, mem::size_of}; + +use abi::error::Error; use crate::{ arch::{Architecture, ArchitectureImpl, PlatformImpl}, @@ -48,7 +50,7 @@ pub unsafe trait ConvertAddress { } /// Helper trait to allow cross-address space access to pointers -pub trait ForeignPointer { +pub trait ForeignPointer: Sized { /// Perform a volatile pointer write without dropping the old value. /// /// # Panics @@ -63,6 +65,36 @@ pub trait ForeignPointer { /// /// As this function allows direct memory writes, it is inherently unsafe. unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); + + /// Performs pointer validation for given address space: + /// + /// * Checks if the pointer has proper alignment for the type. + /// * Checks if the pointer is mapped in the address space. + /// * Checks if the pointer is above the userspace memory boundary. + unsafe fn validate_user_ptr<'a>( + self: *const Self, + space: &AddressSpace, + ) -> Result<&'a Self, Error>; + + /// [ForeignPointer::validate_user_ptr], with extra "writability" check. + unsafe fn validate_user_mut<'a>( + self: *mut Self, + space: &AddressSpace, + ) -> Result<&'a mut Self, Error>; + + /// [ForeignPointer::validate_user_ptr], but for slices + unsafe fn validate_user_slice<'a>( + self: *const Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a [Self], Error>; + + /// [ForeignPointer::validate_user_slice], but for mutable slices + unsafe fn validate_user_slice_mut<'a>( + self: *mut Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a mut [Self], Error>; } unsafe impl ConvertAddress for usize { @@ -130,4 +162,101 @@ impl ForeignPointer for T { let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; virt_ptr.write_volatile(value); } + + unsafe fn validate_user_slice_mut<'a>( + self: *mut Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a mut [Self], Error> { + let base = self as usize; + let layout = Layout::array::(len).unwrap(); + + validate_user_align_size(base, &layout)?; + validate_user_region(space, base, layout.size(), true)?; + + Ok(core::slice::from_raw_parts_mut(self, len)) + } + + unsafe fn validate_user_slice<'a>( + self: *const Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a [Self], Error> { + let base = self as usize; + let layout = Layout::array::(len).unwrap(); + + validate_user_align_size(base, &layout)?; + validate_user_region(space, base, layout.size(), false)?; + + Ok(core::slice::from_raw_parts(self, len)) + } + + unsafe fn validate_user_mut<'a>( + self: *mut Self, + space: &AddressSpace, + ) -> Result<&'a mut Self, Error> { + let addr = self as usize; + let layout = Layout::new::(); + + // Common validation + validate_user_align_size(addr, &layout)?; + + // Validate that the pages covered by this address are mapped as writable by the process + // TODO for CoW this may differ + validate_user_region(space, addr, layout.size(), true)?; + + Ok(&mut *self) + } + + unsafe fn validate_user_ptr<'a>( + self: *const Self, + space: &AddressSpace, + ) -> Result<&'a Self, Error> { + let addr = self as usize; + let layout = Layout::new::(); + + // Common validation + validate_user_align_size(addr, &layout)?; + validate_user_region(space, addr, layout.size(), false)?; + + Ok(&*self) + } +} + +fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { + // Explicitly disallow NULL + if addr == 0 { + return Err(Error::InvalidArgument); + } + // Validate alignment + if addr % layout.align() != 0 { + return Err(Error::InvalidArgument); + } + if addr + layout.size() > KERNEL_VIRT_OFFSET { + todo!(); + } + + Ok(()) +} + +/// Validates access to given userspace memory region with given constraints +pub fn validate_user_region( + space: &AddressSpace, + base: usize, + len: usize, + _need_write: bool, +) -> Result<(), Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + + let aligned_start = base & !0xFFF; + let aligned_end = (base + len + 0xFFF) & !0xFFF; + + for page in (aligned_start..aligned_end).step_by(0x1000) { + // TODO check writability + space.translate(page).ok_or(Error::InvalidArgument)?; + } + + Ok(()) } diff --git a/src/syscall/arg.rs b/src/syscall/arg.rs index 7bede0f1..dede36c3 100644 --- a/src/syscall/arg.rs +++ b/src/syscall/arg.rs @@ -1,66 +1,51 @@ -use core::alloc::Layout; - use abi::io::RawFd; use yggdrasil_abi::error::Error; +use crate::{ + mem::{validate_user_region, ForeignPointer}, + task::process::Process, +}; + pub(super) fn arg_buffer_ref<'a>(base: usize, len: usize) -> Result<&'a [u8], Error> { - if base + len > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } + let proc = Process::current(); + validate_user_region(proc.address_space(), base, len, false)?; Ok(unsafe { core::slice::from_raw_parts(base as *const u8, len) }) } pub(super) fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8], Error> { - if base + len > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } + let proc = Process::current(); + validate_user_region(proc.address_space(), base, len, true)?; Ok(unsafe { core::slice::from_raw_parts_mut(base as *mut u8, len) }) } pub(super) fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error> { let slice = arg_buffer_ref(base, len)?; if slice.contains(&0) { - todo!("Incorrect ptr: {:#x}", base); + warnln!("User-supplied string contains NUL characters"); + return Err(Error::InvalidArgument); } Ok(core::str::from_utf8(slice).unwrap()) } pub(super) fn arg_user_ref<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { - let layout = Layout::new::(); - if addr % layout.align() != 0 { - todo!("Misaligned argument"); - } - // TODO check that addr actually points to mapped (and user-accessible) memory - let value = unsafe { core::mem::transmute::<_, &'a T>(addr) }; - Ok(value) + let proc = Process::current(); + let ptr = addr as *const T; + unsafe { ptr.validate_user_ptr(proc.address_space()) } } pub(super) fn arg_user_mut<'a, T: Sized>(addr: usize) -> Result<&'a mut T, Error> { - let layout = Layout::new::(); - if addr % layout.align() != 0 { - todo!("Misaligned argument"); - } - // TODO check that addr actually points to mapped (and user-accessible) memory - let value = unsafe { core::mem::transmute::<_, &'a mut T>(addr) }; - Ok(value) + let proc = Process::current(); + let ptr = addr as *mut T; + unsafe { ptr.validate_user_mut(proc.address_space()) } } pub(super) fn arg_user_slice_mut<'a, T: Sized>( base: usize, count: usize, ) -> Result<&'a mut [T], Error> { - let layout = Layout::array::(count).unwrap(); - - if base % layout.align() != 0 { - todo!("Misaligned buffer"); - } - - if base + layout.size() > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } - - let slice = unsafe { core::slice::from_raw_parts_mut(base as *mut _, count) }; - Ok(slice) + let proc = Process::current(); + let ptr = base as *mut T; + unsafe { ptr.validate_user_slice_mut(count, proc.address_space()) } } pub(super) fn arg_option_fd(raw: u32) -> Option { From 1a197462e3aaf076abf6e88ad7414b10953165c9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 25 Jul 2023 17:48:26 +0300 Subject: [PATCH 020/211] proc: add CurrentProcess safety wrapper --- src/debug.rs | 2 +- src/syscall/mod.rs | 5 +- src/task/process.rs | 157 ++++++++++++++++++++++++-------------------- src/task/sched.rs | 10 ++- 4 files changed, 95 insertions(+), 79 deletions(-) diff --git a/src/debug.rs b/src/debug.rs index f2e6e051..876b908e 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -136,7 +136,7 @@ pub fn init() { } } -#[doc = "hide"] +#[doc(hidden)] pub fn debug_internal(args: Arguments, level: LogLevel) { use fmt::Write; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 5634f90b..e1c520a1 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -55,10 +55,7 @@ fn run_with_io_at< fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { match func { SyscallFunction::DebugTrace => { - let pid = Process::get_current() - .as_deref() - .map(Process::id) - .unwrap_or(0); + let pid = Process::current().id(); let arg = arg_user_str(args[0] as usize, args[1] as usize)?; debugln!("[{}] TRACE: {:?}", pid, arg); diff --git a/src/task/process.rs b/src/task/process.rs index 4e9e956f..113ace79 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,6 +1,7 @@ //! Process data structures use core::{ mem::size_of, + ops::Deref, sync::atomic::{AtomicU32, Ordering}, }; @@ -74,6 +75,9 @@ pub struct Process { pub io: IrqSafeSpinlock, } +/// Guard type that provides [Process] operations only available for current processes +pub struct CurrentProcess(Rc); + impl Process { /// Creates a process from raw architecture-specific [TaskContext]. /// @@ -205,11 +209,6 @@ impl Process { /// /// The process may not halt its execution immediately when this function is called, only when /// this function is called targeting the *current process* running on *local* CPU. - /// - /// # TODO - /// - /// The code currently does not allow suspension of active processes on either local or other - /// CPUs. pub fn suspend(&self) { let _irq = IrqGuard::acquire(); let current_state = self.state.swap(ProcessState::Suspended, Ordering::SeqCst); @@ -237,18 +236,6 @@ impl Process { } } - /// Sets up a pending wait for the process. - /// - /// # Safety - /// - /// This function is only meant to be called in no-IRQ context and when caller can guarantee - /// the task won't get scheduled to a CPU in such state. - pub unsafe fn setup_wait(&self, wait: &'static Wait) { - let mut inner = self.inner.lock(); - inner.pending_wait.replace(wait); - inner.wait_status = WaitStatus::Pending; - } - /// Returns current wait status of the task pub fn wait_status(&self) -> WaitStatus { self.inner.lock().wait_status @@ -274,7 +261,7 @@ impl Process { } /// Returns the [Process] currently executing on local CPU, None if idling. - pub fn get_current() -> Option> { + pub fn get_current() -> Option { let queue = Cpu::local().queue(); queue.current_process() } @@ -286,7 +273,7 @@ impl Process { /// Wraps [Process::get_current()] for cases when the caller is absolutely sure there is a /// running process (e.g. the call itself comes from a process). - pub fn current() -> Rc { + pub fn current() -> CurrentProcess { Self::get_current().unwrap() } @@ -316,30 +303,10 @@ impl Process { PROCESS_EXIT_WAIT.wakeup_all(); } - /// Terminate a process - pub fn exit(&self, status: ExitCode) { - self.inner.lock().exit_status = status.into(); - let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); - infoln!("Process {} exited with code {:?}", self.id(), status); - - match current_state { - ProcessState::Suspended => { - todo!(); - } - ProcessState::Ready => todo!(), - ProcessState::Running => { - self.handle_exit(); - unsafe { Cpu::local().queue().yield_cpu() } - } - ProcessState::Terminated => todo!(), - } - } - unsafe fn setup_signal_context(&self, signal: Signal, data: Option) { let inner = self.inner.lock(); let space = self.address_space(); let signum = u32::from(signal); - assert_eq!(DAIF.read(DAIF::I), 1); let old_pending_signal = self.pending_signal.swap(signum, Ordering::AcqRel); @@ -354,9 +321,12 @@ impl Process { "Process #{} has no signal entry set up, terminating it", self.id() ); + // Treat this as memory access violation - self.exit(ExitCode::BySignal(Signal::MemoryAccessViolation)); - panic!(); + todo!(); + + // self.exit(ExitCode::BySignal(Signal::MemoryAccessViolation)); + // panic!(); }; let ttbr0 = self.address_space().physical_address(); @@ -387,9 +357,53 @@ impl Process { ); } - /// Raises a synchronous exception signal for the current process. + /// Raises an asynchronous signal for the target process + pub fn try_set_signal(&self, signal: Signal) -> Result<(), Error> { + assert_eq!(DAIF.read(DAIF::I), 1); + assert_ne!(self.state(), ProcessState::Running); + + if signal.is_synchronous() { + return Err(Error::InvalidArgument); + } + + unsafe { + self.setup_signal_context(signal, None); + } + + Ok(()) + } +} + +impl Drop for Process { + fn drop(&mut self) { + infoln!("Drop process!"); + } +} + +impl CurrentProcess { + /// Wraps a process in this structure. /// - /// Must be called on the current process. + /// # Safety + /// + /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. + pub unsafe fn new(inner: Rc) -> Self { + assert_eq!(DAIF.read(DAIF::I), 1); + Self(inner) + } + + /// Sets up a pending wait for the process. + /// + /// # Safety + /// + /// This function is only meant to be called in no-IRQ context and when caller can guarantee + /// the task won't get scheduled to a CPU in such state. + pub unsafe fn setup_wait(&self, wait: &'static Wait) { + let mut inner = self.inner.lock(); + inner.pending_wait.replace(wait); + inner.wait_status = WaitStatus::Pending; + } + + /// Raises a synchronous exception signal for the current process pub fn try_raise_exception(&self, signal: Signal, data: Option) -> Result<(), Error> { assert_eq!(self.state(), ProcessState::Running); assert_eq!(DAIF.read(DAIF::I), 1); @@ -407,9 +421,7 @@ impl Process { } /// Raises a synchronous exception signal and terminates the process if it cannot currently - /// handle it. - /// - /// Must be called on the current process. + /// handle it pub fn raise_exception(&self, signal: Signal, data: Option) { if let Err(err) = self.try_raise_exception(signal, data) { warnln!( @@ -423,32 +435,13 @@ impl Process { } } - /// Raises an asynchronous signal for the target process - pub fn try_set_signal(&self, signal: Signal) -> Result<(), Error> { - assert_eq!(DAIF.read(DAIF::I), 1); - assert_ne!(self.state(), ProcessState::Running); - - if signal.is_synchronous() { - return Err(Error::InvalidArgument); - } - - unsafe { - self.setup_signal_context(signal, None); - } - - Ok(()) - } - - /// Sets up signal entry information so the kernel can initialize a signal context for the - /// process + /// Configures signal entry information for the process pub fn set_signal_entry(&self, entry: usize, stack: usize) { let mut inner = self.inner.lock(); inner.signal_entry.replace(SignalEntry { entry, stack }); } - /// Leaves the current signal handler. - /// - /// Must be called for a current process. + /// Returns back to normal context from a signal handler pub fn exit_signal(&self) -> ! { assert_eq!(self.state(), ProcessState::Running); let old_pending_signal = self.pending_signal.swap(0, Ordering::AcqRel); @@ -458,10 +451,32 @@ impl Process { unsafe { self.normal_context.switch(&self.signal_context) } panic!("Cannot return here"); } -} -impl Drop for Process { - fn drop(&mut self) { - infoln!("Drop process!"); + /// Terminate the current process + pub fn exit(&self, status: ExitCode) { + self.inner.lock().exit_status = status.into(); + let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); + assert_eq!(current_state, ProcessState::Running); + infoln!("Process {} exited with code {:?}", self.id(), status); + + match current_state { + ProcessState::Suspended => { + todo!(); + } + ProcessState::Ready => todo!(), + ProcessState::Running => { + self.handle_exit(); + unsafe { Cpu::local().queue().yield_cpu() } + } + ProcessState::Terminated => todo!(), + } + } +} + +impl Deref for CurrentProcess { + type Target = Rc; + + fn deref(&self) -> &Self::Target { + &self.0 } } diff --git a/src/task/sched.rs b/src/task/sched.rs index 5dd24545..dbdd26f9 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -11,7 +11,7 @@ use crate::{ }; use super::{ - process::{Process, ProcessState}, + process::{CurrentProcess, Process, ProcessState}, ProcessId, }; @@ -242,8 +242,12 @@ impl CpuQueue { /// * (in irq) the code cannot be interrupted and other CPUs shouldn't change this queue, so it /// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu] /// is called. - pub fn current_process(&self) -> Option> { - self.inner.lock().current.clone() + pub fn current_process(&self) -> Option { + self.inner + .lock() + .current + .clone() + .map(|p| unsafe { CurrentProcess::new(p) }) } /// Returns a queue for given CPU index From 7fa347aa207d73e5e6444a3adb6f67ee10487fb3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 27 Jul 2023 16:24:52 +0300 Subject: [PATCH 021/211] proc: better signal handling --- lib/vfs/src/char.rs | 8 +- lib/vfs/src/node.rs | 15 +- src/arch/aarch64/context.rs | 36 +---- src/arch/aarch64/exception.rs | 116 +++++++++++++-- src/arch/aarch64/table.rs | 1 - src/arch/aarch64/timer.rs | 6 +- src/arch/aarch64/vectors.S | 2 +- src/device/serial/pl011.rs | 12 +- src/device/tty.rs | 25 +++- src/main.rs | 32 ++++- src/mem/mod.rs | 22 ++- src/mem/phys/manager.rs | 23 ++- src/mem/phys/mod.rs | 49 +++++++ src/proc/io.rs | 5 + src/proc/wait.rs | 22 +-- src/syscall/mod.rs | 71 ++++++++- src/task/mod.rs | 1 + src/task/process.rs | 261 +++++++++++++++++----------------- src/task/tasklet.rs | 63 ++++++++ 19 files changed, 565 insertions(+), 205 deletions(-) create mode 100644 src/task/tasklet.rs diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index adbd946a..70774490 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -1,6 +1,6 @@ use yggdrasil_abi::{ error::Error, - io::{FileAttr, FileMode, FileType, OpenOptions}, + io::{DeviceRequest, FileAttr, FileMode, FileType, OpenOptions}, }; use crate::{node::VnodeImpl, VnodeRef}; @@ -8,6 +8,8 @@ use crate::{node::VnodeImpl, VnodeRef}; pub trait CharDevice { fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result; fn write(&self, blocking: bool, data: &[u8]) -> Result; + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error>; } pub struct CharDeviceWrapper { @@ -49,4 +51,8 @@ impl VnodeImpl for CharDeviceWrapper { mode: FileMode::default_file(), }) } + + fn device_request(&mut self, _node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { + self.device.device_request(req) + } } diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index e76328c3..7ef5a652 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -11,7 +11,7 @@ use alloc::{ }; use yggdrasil_abi::{ error::Error, - io::{FileAttr, FileMode, FileType, OpenOptions}, + io::{DeviceRequest, FileAttr, FileMode, FileType, OpenOptions}, }; use crate::{ @@ -82,6 +82,10 @@ pub trait VnodeImpl { fn metadata(&mut self, node: &VnodeRef) -> Result { Err(Error::NotImplemented) } + + fn device_request(&mut self, node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { + Err(Error::NotImplemented) + } } impl Vnode { @@ -206,7 +210,6 @@ impl Vnode { let target = self.target(); if let Some(target) = target { - assert_eq!(tree.children.len(), 0); f.write_str(" -> ")?; return target.dump(f, depth, false); } @@ -433,6 +436,14 @@ impl Vnode { todo!() } } + + pub fn device_request(self: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { + if let Some(ref mut data) = *self.data() { + data.device_request(self, req) + } else { + todo!() + } + } } impl fmt::Debug for Vnode { diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index f5eb63be..1bcc0e55 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -21,8 +21,10 @@ struct TaskContextInner { } /// AArch64 implementation of a task context +#[allow(unused)] pub struct TaskContext { inner: UnsafeCell, + // TODO will be used when I write proper task cleanup stack_base: usize, stack_size: usize, } @@ -149,40 +151,6 @@ impl TaskContext { }) } - /// Constructs an empty context. One usage is to create an uninitialized context and have a - /// process set it up later (e.g. for a signal handler). - pub fn empty() -> Result { - const USER_TASK_PAGES: usize = 8; - let stack_base = - unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; - let sp = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000).build(); - - Ok(Self { - inner: UnsafeCell::new(TaskContextInner { sp }), - stack_base, - stack_size: USER_TASK_PAGES * 0x1000, - }) - } - - /// Initializes the context with a signal entry information. - /// - /// # Safety - /// - /// The function accepts arbitrary addresses and is unsafe to call. - pub unsafe fn setup_signal_entry(&self, entry: usize, arg: usize, ttbr0: usize, ustack: usize) { - let mut stack = StackBuilder::new(self.stack_base, self.stack_size); - - stack.push(entry); - stack.push(arg); - stack.push(0); - stack.push(ustack); - - stack.init_common(__aarch64_task_enter_user as _, ttbr0); - - let sp = stack.build(); - (*self.inner.get()).sp = sp; - } - /// Starts execution of `self` task on local CPU. /// /// # Safety diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 21d52c2d..546e4145 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -2,7 +2,10 @@ use core::{arch::global_asm, fmt}; use aarch64_cpu::registers::{ELR_EL1, ESR_EL1, FAR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; -use abi::process::Signal; +use abi::{ + process::{SavedFrame, Signal, SignalEntryData}, + syscall::SyscallFunction, +}; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ @@ -17,11 +20,38 @@ use crate::{ /// Struct for register values saved when taking an exception #[repr(C)] pub struct ExceptionFrame { - r: [u64; 32], - c: [u64; 4], + /// General-purpose registers + pub r: [u64; 32], + /// SPSR_EL1, userspace flags register + pub spsr_el1: u64, + /// ELR_EL1, userspace program counter + pub elr_el1: u64, + /// SP_EL0, userspace stack pointer + pub sp_el0: u64, + _x: u64, // ... } +impl ExceptionFrame { + /// Saves an userspace execution context into a frame "snapshot" + pub fn to_saved_frame(&self) -> SavedFrame { + SavedFrame { + gp_regs: self.r, + spsr_el1: self.spsr_el1, + elr_el1: self.elr_el1, + sp_el0: self.sp_el0, + } + } + + /// Restores an userspace execution context from its frame "snapshot" + pub fn restore_from_signal(&mut self, frame: &SavedFrame) { + self.r = frame.gp_regs; + self.spsr_el1 = frame.spsr_el1; + self.elr_el1 = frame.elr_el1; + self.sp_el0 = frame.sp_el0; + } +} + impl fmt::Debug for ExceptionFrame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for i in (0..32).step_by(2) { @@ -114,9 +144,57 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { } #[no_mangle] -extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { +extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) { let frame = unsafe { &mut *frame }; + let process = Process::current(); + el0_sync_inner(frame); + unsafe { + process.handle_signal(frame); + } +} +#[no_mangle] +extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) { + let frame = unsafe { &mut *frame }; + let process = Process::current(); + irq_common(); + unsafe { + process.handle_signal(frame); + } +} + +#[no_mangle] +extern "C" fn __aa64_el0_fiq_handler() { + todo!(); +} + +#[no_mangle] +extern "C" fn __aa64_el0_serror_handler() { + todo!(); +} + +// EL1 +#[no_mangle] +extern "C" fn __aa64_el1_sync_handler(_frame: *mut ExceptionFrame) { + todo!(); +} + +#[no_mangle] +extern "C" fn __aa64_el1_irq_handler(_frame: *mut ExceptionFrame) { + irq_common(); +} + +#[no_mangle] +extern "C" fn __aa64_el1_fiq_handler() { + todo!(); +} + +#[no_mangle] +extern "C" fn __aa64_el1_serror_handler() { + todo!(); +} + +fn el0_sync_inner(frame: &mut ExceptionFrame) { let esr_el1 = ESR_EL1.get(); let ec = (esr_el1 >> 26) & 0x3F; @@ -124,6 +202,13 @@ extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { // SVC in AArch64 0b010101 => { let func = frame.r[8]; + if func == usize::from(SyscallFunction::ExitSignal) as u64 { + unsafe { + handle_signal_exit(frame); + } + return; + } + let args = &frame.r[0..6]; let result = raw_syscall_handler(func, args); frame.r[0] = result; @@ -132,7 +217,7 @@ extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { 0b111100 => { let proc = Process::current(); warnln!("Process #{} hit a breakpoint", proc.id()); - proc.raise_exception(Signal::Aborted, None); + proc.try_set_signal(Signal::Aborted).ok(); } _ => { let iss = esr_el1 & 0x1FFFFFF; @@ -146,7 +231,7 @@ extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { FAR_EL1.get() ); - proc.raise_exception(Signal::MemoryAccessViolation, Some(FAR_EL1.get() as _)); + proc.try_set_signal(Signal::MemoryAccessViolation).ok(); return; } @@ -157,22 +242,23 @@ extern "C" fn __aa64_exc_sync_handler(frame: *mut ExceptionFrame) { } } -#[no_mangle] -extern "C" fn __aa64_exc_irq_handler(_frame: *mut ExceptionFrame) { +fn irq_common() { unsafe { let ic = IrqContext::new(); PLATFORM.interrupt_controller().handle_pending_irqs(&ic); } } -#[no_mangle] -extern "C" fn __aa64_exc_fiq_handler() { - todo!(); -} +unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) { + // TODO validate the argument + let saved_data = &*(frame.r[0] as *const SignalEntryData); + debugln!( + "Handling signal exit to pc={:#x}, sp={:#x}", + saved_data.frame.elr_el1, + saved_data.frame.sp_el0 + ); -#[no_mangle] -extern "C" fn __aa64_exc_serror_handler() { - todo!(); + frame.restore_from_signal(&saved_data.frame); } pub(super) fn ipi_handler(msg: Option) { diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index d5514a07..66ffd73e 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -321,7 +321,6 @@ impl VirtualMemoryManager for AddressSpace { ) -> Result { assert_eq!(DAIF.read(DAIF::I), 1); - assert_eq!(len, 1); if hint.is_some() { todo!(); } diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 7f8e37a1..2d8622fd 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -10,6 +10,7 @@ use crate::{ arch::PLATFORM, device::{interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device}, proc::wait, + task::tasklet, }; use super::{cpu::Cpu, gic::IrqNumber}; @@ -45,7 +46,10 @@ impl TimestampSource for ArmTimer { impl InterruptSource for ArmTimer { fn handle_irq(&self) -> Result<(), Error> { CNTP_TVAL_EL0.set(TICK_INTERVAL); - wait::tick(); + let t = self.timestamp()?; + + wait::tick(t); + tasklet::tick(t); unsafe { Cpu::local().queue().yield_cpu(); diff --git a/src/arch/aarch64/vectors.S b/src/arch/aarch64/vectors.S index d16c12fa..586d76dc 100644 --- a/src/arch/aarch64/vectors.S +++ b/src/arch/aarch64/vectors.S @@ -15,7 +15,7 @@ __aa\bits\()_el\el\ht\()_\kind: EXC_SAVE_STATE mov x0, sp mov lr, xzr - bl __aa64_exc_\kind\()_handler + bl __aa64_el\el\()_\kind\()_handler EXC_RESTORE_STATE eret .endm diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index be767314..051ae0e9 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -1,5 +1,5 @@ //! ARM PL011 driver -use abi::error::Error; +use abi::{error::Error, io::DeviceRequest}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -118,6 +118,16 @@ impl CharDevice for Pl011 { assert!(blocking); self.line_read(data) } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(id as _); + Ok(()) + } + _ => Err(Error::InvalidArgument), + } + } } impl SerialDevice for Pl011 { diff --git a/src/device/tty.rs b/src/device/tty.rs index 63a11baf..b4f526ce 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -2,9 +2,14 @@ use abi::{ error::Error, io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, + process::Signal, }; -use crate::{proc::wait::Wait, sync::IrqSafeSpinlock}; +use crate::{ + proc::wait::Wait, + sync::IrqSafeSpinlock, + task::{process::Process, ProcessId}, +}; use super::serial::SerialDevice; @@ -13,6 +18,7 @@ struct CharRingInner { wr: usize, data: [u8; N], flags: u8, + process_group: Option, } /// Ring buffer for a character device. Handles reads, writes and channel notifications for a @@ -39,6 +45,11 @@ pub trait TtyDevice: SerialDevice { } } + /// Sets the process group to which signals from this terminal should be delivered + fn set_signal_group(&self, id: ProcessId) { + self.ring().inner.lock().process_group = Some(id); + } + /// Sends a single byte to the terminal fn line_send(&self, byte: u8) -> Result<(), Error> { let config = self.ring().config.lock(); @@ -80,7 +91,16 @@ pub trait TtyDevice: SerialDevice { } } - // TODO handle signals + if byte == config.chars.interrupt && config.line.contains(TerminalLineOptions::SIGNAL) { + drop(config); + let pgrp = ring.inner.lock().process_group; + if let Some(pgrp) = pgrp { + Process::signal_group(pgrp, Signal::Interrupted); + return; + } else { + debugln!("Terminal has no process group attached"); + } + } ring.putc(byte, false).ok(); } @@ -174,6 +194,7 @@ impl CharRing { wr: 0, data: [0; N], flags: 0, + process_group: None, }), wait_read: Wait::new("char_ring_read"), wait_write: Wait::new("char_ring_write"), diff --git a/src/main.rs b/src/main.rs index f686a790..17092cd2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ const_trait_impl, maybe_uninit_slice, arbitrary_self_types, + let_chains, linked_list_cursors )] #![allow(clippy::new_without_default)] @@ -18,10 +19,10 @@ extern crate yggdrasil_abi as abi; use abi::{ error::Error, - io::{FileMode, OpenOptions}, + io::{FileMode, OpenOptions, RawFd}, process::ExitCode, }; -use fs::{FileBlockAllocator, INITRD_DATA}; +use fs::{devfs, FileBlockAllocator, INITRD_DATA}; use memfs::MemoryFilesystem; use task::process::Process; use vfs::{Filesystem, IoContext, VnodeRef}; @@ -56,6 +57,19 @@ fn setup_root() -> Result { /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kernel_main() { + // use crate::{debug::LogLevel, mem::phys}; + // use crate::task::tasklet::{self, TaskFlow}; + // use core::time::Duration; + + // Schedule a tasklet to print memory usage stats every 10 seconds + // tasklet::add_periodic("mem-stats", Duration::from_secs(3), || { + // let phys_mem = phys::PHYSICAL_MEMORY.get().lock(); + // let stats = phys_mem.stats(); + // stats.dump(LogLevel::Debug); + + // TaskFlow::Continue + // }); + let root = match setup_root() { Ok(root) => root, Err(err) => { @@ -68,11 +82,25 @@ pub fn kernel_main() { let node = ioctx.find(None, "/init", true, true).unwrap(); let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let devfs = devfs::root(); + let console = ioctx + .find(Some(devfs.clone()), "ttyS0", true, true) + .unwrap(); + let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); + let stderr = stdout.clone(); + { let user_init = proc::exec::load_elf(file, &["/init", "xxx"]).unwrap(); let mut io = user_init.io.lock(); io.set_ioctx(ioctx); + io.set_file(RawFd::STDIN, stdin).unwrap(); + io.set_file(RawFd::STDOUT, stdout).unwrap(); + io.set_file(RawFd::STDERR, stderr).unwrap(); drop(io); + + user_init.set_session_terminal(console); + user_init.enqueue_somewhere(); } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index da8d09ab..725d7d37 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -71,18 +71,33 @@ pub trait ForeignPointer: Sized { /// * Checks if the pointer has proper alignment for the type. /// * Checks if the pointer is mapped in the address space. /// * Checks if the pointer is above the userspace memory boundary. + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. unsafe fn validate_user_ptr<'a>( self: *const Self, space: &AddressSpace, ) -> Result<&'a Self, Error>; /// [ForeignPointer::validate_user_ptr], with extra "writability" check. + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. unsafe fn validate_user_mut<'a>( self: *mut Self, space: &AddressSpace, ) -> Result<&'a mut Self, Error>; /// [ForeignPointer::validate_user_ptr], but for slices + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. unsafe fn validate_user_slice<'a>( self: *const Self, len: usize, @@ -90,6 +105,11 @@ pub trait ForeignPointer: Sized { ) -> Result<&'a [Self], Error>; /// [ForeignPointer::validate_user_slice], but for mutable slices + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. unsafe fn validate_user_slice_mut<'a>( self: *mut Self, len: usize, @@ -148,7 +168,7 @@ impl ForeignPointer for T { // TODO check align let addr = self as usize; let start_page = addr & !0xFFF; - let end_page = (addr + size_of::()) & !0xFFF; + let end_page = (addr + size_of::() - 1) & !0xFFF; let page_offset = addr & 0xFFF; if start_page != end_page { diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 7739cf8b..c08e8f3f 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -3,12 +3,13 @@ use core::mem::size_of; use abi::error::Error; -use super::{Page, PageUsage}; +use super::{Page, PageUsage, PhysicalMemoryStats}; /// Physical memory management interface pub struct PhysicalMemoryManager { pages: &'static mut [Page], offset: usize, + stats: PhysicalMemoryStats, } impl PhysicalMemoryManager { @@ -32,7 +33,14 @@ impl PhysicalMemoryManager { }; } - PhysicalMemoryManager { pages, offset } + PhysicalMemoryManager { + pages, + offset, + stats: PhysicalMemoryStats { + available_pages: 0, + used_pages: 0, + }, + } } /// Allocates a single page, marking it as used with `usage` @@ -43,6 +51,7 @@ impl PhysicalMemoryManager { for index in 0..self.pages.len() { if self.pages[index].usage == PageUsage::Available { self.pages[index].usage = PageUsage::Used; + self.stats.add_allocated_pages(1, usage); return Ok(index * 4096 + self.offset); } } @@ -72,6 +81,7 @@ impl PhysicalMemoryManager { page.usage = usage; page.refcount = 1; } + self.stats.add_allocated_pages(count, usage); return Ok(self.offset + i * 0x1000); } @@ -88,6 +98,9 @@ impl PhysicalMemoryManager { let index = (addr - self.offset) / 0x1000; let page = &mut self.pages[index]; assert_eq!(page.usage, PageUsage::Used); + + self.stats.add_freed_pages(1, page.usage); + page.usage = PageUsage::Available; } @@ -103,6 +116,12 @@ impl PhysicalMemoryManager { assert_eq!(self.pages[index].usage, PageUsage::Reserved); assert_eq!(self.pages[index].refcount, 0); + self.stats.add_available_pages(1); self.pages[index].usage = PageUsage::Available; } + + /// Returns a reference to physical memory stats info + pub fn stats(&self) -> &PhysicalMemoryStats { + &self.stats + } } diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 3d664187..a19f435f 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -5,6 +5,7 @@ use abi::error::Error; use crate::{ absolute_address, + debug::LogLevel, mem::{ phys::reserved::{is_reserved, reserve_region}, ConvertAddress, KERNEL_PHYS_BASE, @@ -18,6 +19,15 @@ use self::manager::PhysicalMemoryManager; pub mod manager; pub mod reserved; +/// Contains information about the physical memory usage +#[derive(Clone, Copy, Debug)] +pub struct PhysicalMemoryStats { + /// Number of pages available for allocation + pub available_pages: usize, + /// Number of pages being used + pub used_pages: usize, +} + /// Represents the way in which the page is used (or not) #[derive(PartialEq, Clone, Copy, Debug)] #[repr(u32)] @@ -63,6 +73,45 @@ impl PhysicalMemoryRegion { } } +impl PhysicalMemoryStats { + /// Handles "alloc" cases of the memory manager + pub fn add_allocated_pages(&mut self, count: usize, _usage: PageUsage) { + assert!(self.available_pages >= count); + self.available_pages -= count; + self.used_pages += count; + } + + /// Handles "free" cases of the memory manager + pub fn add_freed_pages(&mut self, count: usize, _usage: PageUsage) { + assert!(self.used_pages >= count); + self.used_pages -= count; + self.available_pages += count; + } + + /// Increases the available pages counter + pub fn add_available_pages(&mut self, count: usize) { + self.available_pages += count; + } + + /// Prints out the statistics into specified log level + pub fn dump(&self, level: LogLevel) { + log_print_raw!(level, "+++ Physical memory stats +++\n"); + log_print_raw!( + level, + "Available: {}K ({} pages)\n", + self.available_pages * 4, + self.available_pages + ); + log_print_raw!( + level, + "Used: {}K ({} pages)\n", + self.used_pages * 4, + self.used_pages + ); + log_print_raw!(level, "-----------------------------\n"); + } +} + /// Global physical memory manager pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); diff --git a/src/proc/io.rs b/src/proc/io.rs index 7efc9f29..e91a5915 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -24,6 +24,11 @@ impl ProcessIo { self.files.get(&fd).cloned().ok_or(Error::InvalidFile) } + /// Iterates over the file descriptors in [ProcessIo] and removes them if predicate is `false` + pub fn retain bool>(&mut self, f: F) { + self.files.retain(f) + } + /// Sets the inner I/O context pub fn set_ioctx(&mut self, ioctx: IoContext) { self.ioctx.replace(ioctx); diff --git a/src/proc/wait.rs b/src/proc/wait.rs index 297260df..ad6cf382 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -5,11 +5,14 @@ use abi::error::Error; use alloc::{collections::LinkedList, rc::Rc}; use crate::{ - arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, task::process::Process, + arch::PLATFORM, + device::platform::Platform, + sync::IrqSafeSpinlock, + task::process::{Process, ProcessState}, }; /// Defines whether the wait channel is available for a specific task -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum WaitStatus { /// Wait on the channel was interrupted Interrupted, @@ -66,10 +69,12 @@ impl Wait { drop(tick_lock); - unsafe { - proc.set_wait_status(WaitStatus::Done); + if proc.state() != ProcessState::Terminated { + unsafe { + proc.set_wait_status(WaitStatus::Done); + } + proc.enqueue_somewhere(); } - proc.enqueue_somewhere(); } limit -= 1; @@ -109,7 +114,7 @@ impl Wait { match process.wait_status() { WaitStatus::Pending => (), WaitStatus::Done => return Ok(()), - WaitStatus::Interrupted => todo!(), + WaitStatus::Interrupted => return Err(Error::InvalidArgument), } drop(queue_lock); @@ -132,7 +137,7 @@ impl Wait { } } - panic!(); + // Most likely the process was killed by a signal } } } @@ -160,8 +165,7 @@ pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { } /// Updates all pending timeouts and wakes up the tasks that have reached theirs -pub fn tick() { - let now = PLATFORM.timestamp_source().timestamp().unwrap(); +pub fn tick(now: Duration) { let mut list = TICK_LIST.lock(); let mut cursor = list.cursor_front_mut(); diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index e1c520a1..49b6c939 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -3,7 +3,7 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, - io::{DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, + io::{DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, process::{ExitCode, Signal, SpawnOption, SpawnOptions}, syscall::SyscallFunction, }; @@ -134,12 +134,19 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; let opts = OpenOptions::from(args[3] as u32); let mode = FileMode::from(args[4] as u32); + let proc = Process::current(); run_with_io_at(at, |at, mut io| { debugln!("run_with_io_at {:?}", at); let file = io.ioctx().open(at, path, opts, mode)?; - let fd = io.place_file(file)?; + if proc.session_terminal().is_none() && + let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { + debugln!("Session terminal set for #{}: {}", proc.id(), path); + proc.set_session_terminal(node); + } + + let fd = io.place_file(file)?; Ok(fd.0 as usize) }) } @@ -272,7 +279,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::Spawn => { let options = arg_user_ref::(args[0] as usize)?; - debugln!("Spawn {:#?}", options); + // debugln!("Spawn {:#?}", options); + let proc = Process::current(); run_with_io(|mut io| { let node = io.ioctx().find(None, options.program, true, true)?; @@ -282,6 +290,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let child = proc::exec::load_elf(file, options.arguments)?; let pid = child.id() as u32; + // Inherit group and session from the creator + child.inherit(&proc)?; + // Inherit root from the creator let child_ioctx = IoContext::new(io.ioctx().root().clone()); @@ -298,7 +309,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } } } - child.enqueue_somewhere(); Ok(pid as _) @@ -342,9 +352,60 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } SyscallFunction::ExitSignal => { - Process::current().exit_signal(); + panic!("Handled elsewhere"); + // Process::current().exit_signal(); } SyscallFunction::GetPid => Ok(Process::current().id()), + SyscallFunction::GetSessionId => todo!(), + SyscallFunction::GetProcessGroupId => todo!(), + SyscallFunction::SetProcessGroupId => { + let pid = args[0] as ProcessId; + let group_id = args[1] as ProcessId; + + let proc = Process::current(); + // Other syscall variants are not currently supported + assert_eq!(pid, group_id); + assert_eq!(proc.id(), pid); + + proc.set_group_id(group_id); + Ok(0) + } + SyscallFunction::StartSession => { + let proc = Process::current(); + + let session_terminal = proc.clear_session_terminal(); + + if let Some(ctty) = session_terminal { + // Drop all FDs referring to the old session terminal + run_with_io(|mut io| { + io.retain(|_, f| { + f.borrow() + .node() + .map(|node| !Rc::ptr_eq(&node, &ctty)) + .unwrap_or(true) + }); + }); + } + + proc.set_session_id(proc.id()); + proc.set_group_id(proc.id()); + + Ok(0) + } + + SyscallFunction::DeviceRequest => { + let fd = RawFd(args[0] as u32); + let req = arg_user_mut::(args[1] as usize)?; + + run_with_io(|io| { + let file = io.file(fd)?; + let node = file.borrow().node()?; + + node.device_request(req)?; + + Ok(0) + }) + } } } diff --git a/src/task/mod.rs b/src/task/mod.rs index 29d86f3d..3b7c0985 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -17,6 +17,7 @@ use self::process::Process; pub mod process; pub mod sched; +pub mod tasklet; /// Process identifier alias for clarity pub type ProcessId = usize; diff --git a/src/task/process.rs b/src/task/process.rs index 113ace79..f6417b33 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -10,12 +10,13 @@ use abi::{ error::Error, process::{ExitCode, Signal, SignalEntryData}, }; -use alloc::rc::Rc; +use alloc::{collections::VecDeque, rc::Rc}; use atomic_enum::atomic_enum; use tock_registers::interfaces::Readable; +use vfs::VnodeRef; use crate::{ - arch::aarch64::{context::TaskContext, cpu::Cpu}, + arch::aarch64::{context::TaskContext, cpu::Cpu, exception::ExceptionFrame}, mem::{table::AddressSpace, ForeignPointer}, proc::{ io::ProcessIo, @@ -42,7 +43,7 @@ pub enum ProcessState { } /// Describes signal entry information -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SignalEntry { entry: usize, stack: usize, @@ -53,21 +54,23 @@ struct ProcessInner { wait_status: WaitStatus, exit_status: i32, + session_id: ProcessId, + group_id: ProcessId, + session_terminal: Option, + signal_entry: Option, + signal_stack: VecDeque, } /// Process data and state structure pub struct Process { normal_context: TaskContext, - signal_context: TaskContext, // Process state info id: OneTimeInit, state: AtomicProcessState, cpu_id: AtomicU32, space: Option, - pending_signal: AtomicU32, - async_signal: AtomicU32, inner: IrqSafeSpinlock, @@ -87,21 +90,23 @@ impl Process { pub fn new_with_context(space: Option, normal_context: TaskContext) -> Rc { let this = Rc::new(Self { normal_context, - signal_context: TaskContext::empty().unwrap(), id: OneTimeInit::new(), state: AtomicProcessState::new(ProcessState::Suspended), cpu_id: AtomicU32::new(0), space, - async_signal: AtomicU32::new(0), - pending_signal: AtomicU32::new(0), inner: IrqSafeSpinlock::new(ProcessInner { pending_wait: None, wait_status: WaitStatus::Done, exit_status: 0, + session_id: 0, + group_id: 0, + session_terminal: None, + signal_entry: None, + signal_stack: VecDeque::new(), }), io: IrqSafeSpinlock::new(ProcessIo::new()), @@ -110,26 +115,18 @@ impl Process { let id = unsafe { PROCESSES.lock().push(this.clone()) }; this.id.init(id); + { + let mut inner = this.inner.lock(); + inner.session_id = id; + inner.group_id = id; + } + this } /// Returns a reference to the inner architecture-specific [TaskContext]. pub fn current_context(&self) -> &TaskContext { - if self.pending_signal.load(Ordering::Acquire) == 0 { - &self.normal_context - } else { - &self.signal_context - } - } - - /// Returns a signal currently handled by the process, if any - pub fn pending_signal(&self) -> Option { - let value = self.pending_signal.load(Ordering::Acquire); - if value == 0 { - None - } else { - Some(Signal::try_from(value).unwrap()) - } + &self.normal_context } /// Returns this process' ID @@ -171,6 +168,36 @@ impl Process { self.space.as_ref() } + /// Replaces the task's session terminal device with another one + pub fn set_session_terminal(&self, terminal: VnodeRef) { + self.inner.lock().session_terminal.replace(terminal); + } + + /// Removes the task's current terminal + pub fn clear_session_terminal(&self) -> Option { + self.inner.lock().session_terminal.take() + } + + /// Returns the current terminal of the task + pub fn session_terminal(&self) -> Option { + self.inner.lock().session_terminal.clone() + } + + /// Sets the session ID of the task + pub fn set_session_id(&self, sid: ProcessId) { + self.inner.lock().session_id = sid; + } + + /// Sets the process group ID of the task + pub fn set_group_id(&self, gid: ProcessId) { + self.inner.lock().group_id = gid; + } + + /// Returns the process group ID of the task + pub fn group_id(&self) -> ProcessId { + self.inner.lock().group_id + } + /// Selects a suitable CPU queue and submits the process for execution. /// /// # Panics @@ -194,12 +221,12 @@ impl Process { pub fn enqueue_to(self: Rc, queue: &CpuQueue) { let current_state = self.state.swap(ProcessState::Ready, Ordering::SeqCst); - if current_state != ProcessState::Suspended { + if current_state == ProcessState::Terminated { todo!("Handle attempt to enqueue an already queued/running/terminated process"); - } - - unsafe { - queue.enqueue(self); + } else if current_state == ProcessState::Suspended { + unsafe { + queue.enqueue(self); + } } } @@ -303,75 +330,45 @@ impl Process { PROCESS_EXIT_WAIT.wakeup_all(); } - unsafe fn setup_signal_context(&self, signal: Signal, data: Option) { - let inner = self.inner.lock(); - let space = self.address_space(); - let signum = u32::from(signal); - - let old_pending_signal = self.pending_signal.swap(signum, Ordering::AcqRel); - - if old_pending_signal != 0 { - todo!(); - } - // TODO: interrupt wait, if any - - let Some(entry) = inner.signal_entry.as_ref() else { - drop(inner); - warnln!( - "Process #{} has no signal entry set up, terminating it", - self.id() - ); - - // Treat this as memory access violation - todo!(); - - // self.exit(ExitCode::BySignal(Signal::MemoryAccessViolation)); - // panic!(); - }; - - let ttbr0 = self.address_space().physical_address(); - - // Place signal data on the provided stack - // TODO check the validity of the stack - let ustack_sp = unsafe { - let size_aligned = (size_of::() + 0x1F) & !0x1F; - let ptr = (entry.stack - size_aligned) as *mut SignalEntryData; - - // This function can be called from non-current context, so assume it is in a foreign - // address space - ptr.write_foreign_volatile(space, SignalEntryData { signal, data }); - - ptr as _ - }; - - unsafe { - self.signal_context - .setup_signal_entry(entry.entry, ustack_sp, ttbr0, ustack_sp); - } - - debugln!( - "Set up signal context for #{}: entry={:#x}, sp={:#x}", - self.id(), - entry.entry, - ustack_sp - ); - } - /// Raises an asynchronous signal for the target process - pub fn try_set_signal(&self, signal: Signal) -> Result<(), Error> { - assert_eq!(DAIF.read(DAIF::I), 1); - assert_ne!(self.state(), ProcessState::Running); - - if signal.is_synchronous() { - return Err(Error::InvalidArgument); + pub fn try_set_signal(self: &Rc, signal: Signal) -> Result<(), Error> { + { + let mut inner = self.inner.lock(); + inner.wait_status = WaitStatus::Interrupted; + inner.signal_stack.push_back(signal); } - unsafe { - self.setup_signal_context(signal, None); + if self.state() == ProcessState::Suspended { + self.clone().enqueue_somewhere(); } Ok(()) } + + /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. + pub fn inherit(&self, parent: &Rc) -> Result<(), Error> { + let mut our_inner = self.inner.lock(); + let their_inner = parent.inner.lock(); + + our_inner.session_id = their_inner.session_id; + our_inner.group_id = their_inner.group_id; + our_inner.session_terminal = their_inner.session_terminal.clone(); + + Ok(()) + } + + /// Raises a signal for the specified process group + pub fn signal_group(group_id: ProcessId, signal: Signal) { + let processes = PROCESSES.lock(); + for (_, proc) in processes.data.iter() { + let inner = proc.inner.lock(); + if proc.state() != ProcessState::Terminated && inner.group_id == group_id { + debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); + drop(inner); + proc.try_set_signal(signal).unwrap(); + } + } + } } impl Drop for Process { @@ -403,35 +400,54 @@ impl CurrentProcess { inner.wait_status = WaitStatus::Pending; } - /// Raises a synchronous exception signal for the current process - pub fn try_raise_exception(&self, signal: Signal, data: Option) -> Result<(), Error> { - assert_eq!(self.state(), ProcessState::Running); - assert_eq!(DAIF.read(DAIF::I), 1); + /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. + /// + /// # Safety + /// + /// This function is only meant to be called right before returning from an userspace + /// exception handler. + pub unsafe fn handle_signal(&self, frame: &mut ExceptionFrame) { + let mut inner = self.inner.lock(); + if let Some(signal) = inner.signal_stack.pop_front() { + let Some(entry) = inner.signal_entry.clone() else { + todo!(); + }; - if !signal.is_synchronous() { - return Err(Error::InvalidArgument); - } - - unsafe { - self.setup_signal_context(signal, data); - self.signal_context.switch(&self.normal_context); - } - - Ok(()) - } - - /// Raises a synchronous exception signal and terminates the process if it cannot currently - /// handle it - pub fn raise_exception(&self, signal: Signal, data: Option) { - if let Err(err) = self.try_raise_exception(signal, data) { - warnln!( - "Could not raise {:?} for #{} ({:?}), terminating it", - signal, - self.id(), - err + debugln!( + "Enter signal handler from: pc={:#x}, sp={:#x}", + frame.elr_el1, + frame.sp_el0 ); - self.exit(ExitCode::BySignal(signal)); + // TODO check if really in a syscall, lol + let syscall_return = -(u32::from(Error::Interrupted) as isize); + frame.r[0] = syscall_return as u64; + + // Setup signal frame + let usp = (entry.stack - size_of::()) & !0xF; + let frame_ptr = usp as *mut SignalEntryData; + + let saved_frame = frame.to_saved_frame(); + frame_ptr.write_foreign_volatile( + self.address_space(), + SignalEntryData { + signal, + frame: saved_frame, + }, + ); + + // Setup return to signal handler + debugln!( + "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", + entry.entry, + usp, + entry.stack + ); + frame.sp_el0 = usp as _; + frame.elr_el1 = entry.entry as _; + + // Pass the frame pointer as an argument to signal handler entry + frame.r[0] = usp as _; } } @@ -441,17 +457,6 @@ impl CurrentProcess { inner.signal_entry.replace(SignalEntry { entry, stack }); } - /// Returns back to normal context from a signal handler - pub fn exit_signal(&self) -> ! { - assert_eq!(self.state(), ProcessState::Running); - let old_pending_signal = self.pending_signal.swap(0, Ordering::AcqRel); - assert_ne!(old_pending_signal, 0); - self.async_signal.store(0, Ordering::Release); - - unsafe { self.normal_context.switch(&self.signal_context) } - panic!("Cannot return here"); - } - /// Terminate the current process pub fn exit(&self, status: ExitCode) { self.inner.lock().exit_status = status.into(); diff --git a/src/task/tasklet.rs b/src/task/tasklet.rs new file mode 100644 index 00000000..bcecea04 --- /dev/null +++ b/src/task/tasklet.rs @@ -0,0 +1,63 @@ +//! Small deferred tasks which run in interrupt context +use alloc::{boxed::Box, vec::Vec}; +use core::time::Duration; + +use crate::sync::IrqSafeSpinlock; + +/// Specifies whether a periodic task should be cancelled after its current iteration +#[derive(Debug, PartialEq)] +pub enum TaskFlow { + /// The task will be executed again later + Continue, + /// The task is finished and will be removed from the list + Cancel, +} + +struct PeriodicTaskState { + last_run: Duration, + next_run: Duration, + interval: Duration, + f: Box TaskFlow>, +} + +struct OneTimeTask { + run_time: Duration, + f: Box, +} + +static PERIODIC_TASKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); +static ONE_TIME_TASKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); + +/// Setup a periodic task to run at a specified interval +pub fn add_periodic TaskFlow + 'static>(name: &str, interval: Duration, f: F) { + debugln!("Schedule {:?} every {:?}", name, interval); + let f = Box::new(f); + PERIODIC_TASKS.lock().push(PeriodicTaskState { + interval, + last_run: Duration::ZERO, + next_run: Duration::ZERO, + f, + }); +} + +/// Updates the state of the tasklets, running the ones which meet their deadline conditions +pub fn tick(now: Duration) { + PERIODIC_TASKS.lock().retain_mut(|task| { + if now >= task.next_run { + task.last_run = now; + task.next_run = now + task.interval; + (task.f)() == TaskFlow::Continue + } else { + true + } + }); + + ONE_TIME_TASKS.lock().retain(|task| { + if now >= task.run_time { + (task.f)(); + false + } else { + true + } + }) +} From e2381ee25aa10a5d76407d0e83d9f45faf51dfcd Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 28 Jul 2023 14:26:39 +0300 Subject: [PATCH 022/211] x86_64: initial booting capability --- Cargo.toml | 4 + src/arch/mod.rs | 100 ++++++++++++--------- src/arch/x86_64/entry.S | 0 src/arch/x86_64/mod.rs | 73 +++++++++++++++ src/arch/x86_64/tables.rs | 0 src/main.rs | 183 ++++++++++++++++++++------------------ 6 files changed, 227 insertions(+), 133 deletions(-) create mode 100644 src/arch/x86_64/entry.S create mode 100644 src/arch/x86_64/mod.rs create mode 100644 src/arch/x86_64/tables.rs diff --git a/Cargo.toml b/Cargo.toml index 7ae70ef8..7c4e8789 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,13 @@ linked_list_allocator = "0.10.5" spinning_top = "0.2.5" static_assertions = "1.1.0" tock-registers = "0.8.1" +cfg-if = "1.0.0" [dependencies.elf] version = "0.7.2" git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" default-features = false features = ["no_std_stream"] + +[target.'cfg(target_arch = "x86_64")'.dependencies] +yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 06a59214..6496b936 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,48 +1,60 @@ //! Provides architecture/platform-specific implementation details -pub mod aarch64; -pub use aarch64::plat_qemu::{QemuPlatform as PlatformImpl, PLATFORM}; -pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; -use abi::error::Error; +use cfg_if::cfg_if; -/// Describes messages sent from some CPU to others -#[derive(Clone, Copy, PartialEq, Debug)] -#[repr(u64)] -pub enum CpuMessage { - /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow - Panic, -} - -/// Interface for an architecture-specific facilities -pub trait Architecture { - /// Address, to which "zero" address is mapped in the virtual address space - const KERNEL_VIRT_OFFSET: usize; - - /// Initializes the memory management unit and sets up virtual memory management. - /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP - /// system. - /// - /// # Safety - /// - /// Unsafe to call if the MMU has already been initialized. - unsafe fn init_mmu(&self, bsp: bool); - - /// Allocates a virtual mapping for the specified physical memory region - fn map_device_pages(&self, phys: usize, count: usize) -> Result; - - // Architecture intrinsics - - /// Suspends CPU until an interrupt is received - fn wait_for_interrupt(); - - /// Sets the local CPU's interrupt mask. - /// - /// # Safety - /// - /// Enabling interrupts may lead to unexpected behavior unless the context explicitly expects - /// them. - unsafe fn set_interrupt_mask(mask: bool); - - /// Returns the local CPU's interrupt mask - fn interrupt_mask() -> bool; +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub mod aarch64; + + pub use aarch64::plat_qemu::{QemuPlatform as PlatformImpl, PLATFORM}; + pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; + use abi::error::Error; + use cfg_if::cfg_if; + + /// Describes messages sent from some CPU to others + #[derive(Clone, Copy, PartialEq, Debug)] + #[repr(u64)] + pub enum CpuMessage { + /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow + Panic, + } + + /// Interface for an architecture-specific facilities + pub trait Architecture { + /// Address, to which "zero" address is mapped in the virtual address space + const KERNEL_VIRT_OFFSET: usize; + + /// Initializes the memory management unit and sets up virtual memory management. + /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP + /// system. + /// + /// # Safety + /// + /// Unsafe to call if the MMU has already been initialized. + unsafe fn init_mmu(&self, bsp: bool); + + /// Allocates a virtual mapping for the specified physical memory region + fn map_device_pages(&self, phys: usize, count: usize) -> Result; + + // Architecture intrinsics + + /// Suspends CPU until an interrupt is received + fn wait_for_interrupt(); + + /// Sets the local CPU's interrupt mask. + /// + /// # Safety + /// + /// Enabling interrupts may lead to unexpected behavior unless the context explicitly expects + /// them. + unsafe fn set_interrupt_mask(mask: bool); + + /// Returns the local CPU's interrupt mask + fn interrupt_mask() -> bool; + } + } else if #[cfg(target_arch = "x86_64")] { + pub mod x86_64; + } else { + compile_error!("Architecture is not supported"); + } } diff --git a/src/arch/x86_64/entry.S b/src/arch/x86_64/entry.S new file mode 100644 index 00000000..e69de29b diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs new file mode 100644 index 00000000..b0a34b30 --- /dev/null +++ b/src/arch/x86_64/mod.rs @@ -0,0 +1,73 @@ +use core::arch::global_asm; + +use yboot_proto::{ + v1::FramebufferOption, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, + PROTOCOL_VERSION_1, +}; + +pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; +const BOOT_STACK_SIZE: usize = 65536; + +#[repr(C, align(0x20))] +struct BootStack { + data: [u8; BOOT_STACK_SIZE], +} + +#[link_section = ".bss"] +static mut BSP_STACK: BootStack = BootStack { + data: [0; BOOT_STACK_SIZE], +}; + +#[used] +#[link_section = ".data.yboot"] +static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { + header: LoadProtocolHeader { + kernel_magic: KERNEL_MAGIC, + version: PROTOCOL_VERSION_1, + }, + kernel_virt_offset: KERNEL_VIRT_OFFSET as _, + + opt_framebuffer: FramebufferOption { + req_width: 1024, + req_height: 768, + + res_width: 0, + res_height: 0, + res_stride: 0, + res_address: 0, + }, +}; + +extern "C" fn __x86_64_upper_entry() -> ! { + loop {} +} + +global_asm!( + r#" +.global __x86_64_entry + +.section .text.entry +__x86_64_entry: + mov ${yboot_loader_magic}, %edi + cmp %edi, %eax + je 2f + + // (Currently) unsupported bootloader +1: + cli + hlt + jmp 1b + +2: + // yboot entry method + movabsq ${stack_bottom} + {stack_size}, %rax + movabsq ${entry}, %rcx + mov %rax, %rsp + jmp *%rcx +"#, + yboot_loader_magic = const LOADER_MAGIC, + stack_size = const BOOT_STACK_SIZE, + stack_bottom = sym BSP_STACK, + entry = sym __x86_64_upper_entry, + options(att_syntax) +); diff --git a/src/arch/x86_64/tables.rs b/src/arch/x86_64/tables.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/main.rs b/src/main.rs index 17092cd2..ab5246ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,102 +7,107 @@ const_trait_impl, maybe_uninit_slice, arbitrary_self_types, - let_chains, - linked_list_cursors + const_mut_refs, + let_chains )] #![allow(clippy::new_without_default)] #![warn(missing_docs)] #![no_std] #![no_main] -extern crate yggdrasil_abi as abi; - -use abi::{ - error::Error, - io::{FileMode, OpenOptions, RawFd}, - process::ExitCode, -}; -use fs::{devfs, FileBlockAllocator, INITRD_DATA}; -use memfs::MemoryFilesystem; -use task::process::Process; -use vfs::{Filesystem, IoContext, VnodeRef}; - -extern crate alloc; - -#[macro_use] -pub mod debug; +// extern crate yggdrasil_abi as abi; +// +// use abi::{ +// error::Error, +// io::{FileMode, OpenOptions, RawFd}, +// process::ExitCode, +// }; +// use fs::{devfs, FileBlockAllocator, INITRD_DATA}; +// use memfs::MemoryFilesystem; +// use task::process::Process; +// use vfs::{Filesystem, IoContext, VnodeRef}; +// +// extern crate alloc; +// +// #[macro_use] +// pub mod debug; #[macro_use] pub mod arch; -pub mod device; -pub mod fs; -pub mod mem; -pub mod panic; -pub mod proc; -pub mod sync; -pub mod syscall; -pub mod task; -pub mod util; - -fn setup_root() -> Result { - let initrd_data = INITRD_DATA.get(); - let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); - fs.root() -} - -/// Entry point for common kernel code. -/// -/// # Note -/// -/// This function is meant to be used as a kernel-space process after all the platform-specific -/// initialization has finished. -pub fn kernel_main() { - // use crate::{debug::LogLevel, mem::phys}; - // use crate::task::tasklet::{self, TaskFlow}; - // use core::time::Duration; - - // Schedule a tasklet to print memory usage stats every 10 seconds - // tasklet::add_periodic("mem-stats", Duration::from_secs(3), || { - // let phys_mem = phys::PHYSICAL_MEMORY.get().lock(); - // let stats = phys_mem.stats(); - // stats.dump(LogLevel::Debug); - - // TaskFlow::Continue - // }); - - let root = match setup_root() { - Ok(root) => root, - Err(err) => { - warnln!("Could not setup root from initrd: {:?}", err); - return; - } - }; - - let ioctx = IoContext::new(root); - let node = ioctx.find(None, "/init", true, true).unwrap(); - let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); - - let devfs = devfs::root(); - let console = ioctx - .find(Some(devfs.clone()), "ttyS0", true, true) - .unwrap(); - let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); - let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); - let stderr = stdout.clone(); - - { - let user_init = proc::exec::load_elf(file, &["/init", "xxx"]).unwrap(); - let mut io = user_init.io.lock(); - io.set_ioctx(ioctx); - io.set_file(RawFd::STDIN, stdin).unwrap(); - io.set_file(RawFd::STDOUT, stdout).unwrap(); - io.set_file(RawFd::STDERR, stderr).unwrap(); - drop(io); - - user_init.set_session_terminal(console); - - user_init.enqueue_somewhere(); - } - - Process::current().exit(ExitCode::SUCCESS); +#[panic_handler] +fn panic_handler(_pi: &core::panic::PanicInfo) -> ! { + loop {} } +// +// pub mod device; +// pub mod fs; +// pub mod mem; +// pub mod panic; +// pub mod proc; +// pub mod sync; +// pub mod syscall; +// pub mod task; +// pub mod util; +// +// fn setup_root() -> Result { +// let initrd_data = INITRD_DATA.get(); +// let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); +// fs.root() +// } +// +// /// Entry point for common kernel code. +// /// +// /// # Note +// /// +// /// This function is meant to be used as a kernel-space process after all the platform-specific +// /// initialization has finished. +// pub fn kernel_main() { +// // use crate::{debug::LogLevel, mem::phys}; +// // use crate::task::tasklet::{self, TaskFlow}; +// // use core::time::Duration; +// +// // Schedule a tasklet to print memory usage stats every 10 seconds +// // tasklet::add_periodic("mem-stats", Duration::from_secs(3), || { +// // let phys_mem = phys::PHYSICAL_MEMORY.get().lock(); +// // let stats = phys_mem.stats(); +// // stats.dump(LogLevel::Debug); +// +// // TaskFlow::Continue +// // }); +// +// let root = match setup_root() { +// Ok(root) => root, +// Err(err) => { +// warnln!("Could not setup root from initrd: {:?}", err); +// return; +// } +// }; +// +// let ioctx = IoContext::new(root); +// let node = ioctx.find(None, "/init", true, true).unwrap(); +// let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); +// +// let devfs = devfs::root(); +// let console = ioctx +// .find(Some(devfs.clone()), "ttyS0", true, true) +// .unwrap(); +// let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); +// let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); +// let stderr = stdout.clone(); +// +// { +// let user_init = proc::exec::load_elf(file, &["/init", "xxx"]).unwrap(); +// let mut io = user_init.io.lock(); +// io.set_ioctx(ioctx); +// io.set_file(RawFd::STDIN, stdin).unwrap(); +// io.set_file(RawFd::STDOUT, stdout).unwrap(); +// io.set_file(RawFd::STDERR, stderr).unwrap(); +// drop(io); +// +// user_init.set_session_terminal(console); +// +// user_init.enqueue_somewhere(); +// } +// +// Process::current().exit(ExitCode::SUCCESS); +// } From 48d12c9e7787a41250d18e85ca88f549fb58ccd7 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 29 Jul 2023 19:31:56 +0300 Subject: [PATCH 023/211] x86_64: get some text on the screen --- Cargo.toml | 22 +- src/arch/aarch64/table.rs | 6 - src/arch/mod.rs | 86 +++---- src/arch/x86_64/boot/mod.rs | 292 ++++++++++++++++++++++++ src/arch/x86_64/mod.rs | 104 +++------ src/arch/x86_64/table/fixed.rs | 109 +++++++++ src/arch/x86_64/table/mod.rs | 158 +++++++++++++ src/arch/x86_64/tables.rs | 0 src/debug.rs | 54 +++-- src/main.rs | 12 +- src/mem/device.rs | 14 +- src/mem/mod.rs | 396 +++++++++++++++++---------------- src/mem/table.rs | 15 +- src/sync.rs | 12 +- 14 files changed, 922 insertions(+), 358 deletions(-) create mode 100644 src/arch/x86_64/boot/mod.rs create mode 100644 src/arch/x86_64/table/fixed.rs create mode 100644 src/arch/x86_64/table/mod.rs delete mode 100644 src/arch/x86_64/tables.rs diff --git a/Cargo.toml b/Cargo.toml index 7c4e8789..e1067719 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,18 +7,17 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -vfs = { path = "lib/vfs" } -memfs = { path = "lib/memfs" } +# vfs = { path = "lib/vfs" } +# memfs = { path = "lib/memfs" } -aarch64-cpu = "9.3.1" -atomic_enum = "0.2.0" +# atomic_enum = "0.2.0" bitflags = "2.3.3" -fdt-rs = { version = "0.4.3", default-features = false } -linked_list_allocator = "0.10.5" -spinning_top = "0.2.5" -static_assertions = "1.1.0" -tock-registers = "0.8.1" +# linked_list_allocator = "0.10.5" +# spinning_top = "0.2.5" +# static_assertions = "1.1.0" +# tock-registers = "0.8.1" cfg-if = "1.0.0" +embedded-graphics = "0.8.0" [dependencies.elf] version = "0.7.2" @@ -26,5 +25,10 @@ git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" default-features = false features = ["no_std_stream"] +[target.'cfg(target_arch = "aarch64")'.dependencies] +fdt-rs = { version = "0.4.3", default-features = false } +aarch64-cpu = "9.3.1" + [target.'cfg(target_arch = "x86_64")'.dependencies] +bitmap-font = { version = "0.3.0" } yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index 66ffd73e..57dcc78a 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -41,12 +41,6 @@ pub struct L2; #[derive(Clone, Copy)] pub struct L3; -/// Tag trait to mark that the page table level may point to a next-level table -pub trait NonTerminalEntryLevel: EntryLevel { - /// Tag type of the level this entry level may point to - type NextLevel: EntryLevel; -} - impl NonTerminalEntryLevel for L1 { type NextLevel = L2; } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 6496b936..3775b552 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,5 +1,6 @@ //! Provides architecture/platform-specific implementation details +use abi::error::Error; use cfg_if::cfg_if; cfg_if! { @@ -11,50 +12,53 @@ cfg_if! { use abi::error::Error; use cfg_if::cfg_if; - /// Describes messages sent from some CPU to others - #[derive(Clone, Copy, PartialEq, Debug)] - #[repr(u64)] - pub enum CpuMessage { - /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow - Panic, - } - - /// Interface for an architecture-specific facilities - pub trait Architecture { - /// Address, to which "zero" address is mapped in the virtual address space - const KERNEL_VIRT_OFFSET: usize; - - /// Initializes the memory management unit and sets up virtual memory management. - /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP - /// system. - /// - /// # Safety - /// - /// Unsafe to call if the MMU has already been initialized. - unsafe fn init_mmu(&self, bsp: bool); - - /// Allocates a virtual mapping for the specified physical memory region - fn map_device_pages(&self, phys: usize, count: usize) -> Result; - - // Architecture intrinsics - - /// Suspends CPU until an interrupt is received - fn wait_for_interrupt(); - - /// Sets the local CPU's interrupt mask. - /// - /// # Safety - /// - /// Enabling interrupts may lead to unexpected behavior unless the context explicitly expects - /// them. - unsafe fn set_interrupt_mask(mask: bool); - - /// Returns the local CPU's interrupt mask - fn interrupt_mask() -> bool; - } } else if #[cfg(target_arch = "x86_64")] { pub mod x86_64; + + pub use x86_64::{X86_64 as ArchitectureImpl, ARCHITECTURE}; } else { compile_error!("Architecture is not supported"); } } + +/// Describes messages sent from some CPU to others +#[derive(Clone, Copy, PartialEq, Debug)] +#[repr(u64)] +pub enum CpuMessage { + /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow + Panic, +} + +/// Interface for an architecture-specific facilities +pub trait Architecture { + /// Address, to which "zero" address is mapped in the virtual address space + const KERNEL_VIRT_OFFSET: usize; + + /// Initializes the memory management unit and sets up virtual memory management. + /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP + /// system. + /// + /// # Safety + /// + /// Unsafe to call if the MMU has already been initialized. + unsafe fn init_mmu(&self, bsp: bool); + + /// Allocates a virtual mapping for the specified physical memory region + fn map_device_pages(&self, phys: usize, count: usize) -> Result; + + // Architecture intrinsics + + /// Suspends CPU until an interrupt is received + fn wait_for_interrupt(); + + /// Sets the local CPU's interrupt mask. + /// + /// # Safety + /// + /// Enabling interrupts may lead to unexpected behavior unless the context explicitly expects + /// them. + unsafe fn set_interrupt_mask(mask: bool); + + /// Returns the local CPU's interrupt mask + fn interrupt_mask() -> bool; +} diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs new file mode 100644 index 00000000..520924dc --- /dev/null +++ b/src/arch/x86_64/boot/mod.rs @@ -0,0 +1,292 @@ +use core::arch::global_asm; + +use abi::error::Error; +use bitmap_font::TextStyle; +use embedded_graphics::{ + pixelcolor::BinaryColor, + prelude::{DrawTarget, OriginDimensions, Point}, + text::Text, + Drawable, Pixel, +}; +use yboot_proto::{ + v1::FramebufferOption, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, + PROTOCOL_VERSION_1, +}; + +use crate::{ + arch::Architecture, + debug::{self, DebugSink}, + mem::device::DeviceMemory, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +use super::ARCHITECTURE; + +pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; +const BOOT_STACK_SIZE: usize = 65536; + +#[repr(C, align(0x20))] +struct BootStack { + data: [u8; BOOT_STACK_SIZE], +} + +#[link_section = ".bss"] +static mut BSP_STACK: BootStack = BootStack { + data: [0; BOOT_STACK_SIZE], +}; + +#[used] +#[link_section = ".data.yboot"] +static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { + header: LoadProtocolHeader { + kernel_magic: KERNEL_MAGIC, + version: PROTOCOL_VERSION_1, + }, + kernel_virt_offset: KERNEL_VIRT_OFFSET as _, + + opt_framebuffer: FramebufferOption { + req_width: 640, + req_height: 480, + + res_width: 0, + res_height: 0, + res_stride: 0, + res_address: 0, + res_size: 0, + }, +}; + +struct LinearFramebufferInner { + mmio: DeviceMemory, + base: usize, + stride: usize, + width: usize, + height: usize, +} + +pub struct LinearFramebuffer { + inner: IrqSafeSpinlock, +} + +struct Position { + row: u32, + col: u32, +} + +pub struct FramebufferConsole { + framebuffer: &'static LinearFramebuffer, + position: IrqSafeSpinlock, + char_height: usize, + char_width: usize, + width_chars: usize, + height_chars: usize, +} + +impl OriginDimensions for LinearFramebufferInner { + fn size(&self) -> embedded_graphics::prelude::Size { + embedded_graphics::prelude::Size::new(self.width as _, self.height as _) + } +} + +impl DrawTarget for LinearFramebufferInner { + type Color = BinaryColor; + type Error = (); + + fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> + where + I: IntoIterator>, + { + for Pixel(coord, color) in pixels { + let x = coord.x as usize; + let y = coord.y as usize; + let addr = self.base + y * self.stride + x * 4; + let ptr = addr as *mut u32; + unsafe { + if color.is_on() { + ptr.write_volatile(0xFFFFFFFF); + } else { + ptr.write_volatile(0); + } + } + } + + Ok(()) + } +} + +impl DebugSink for FramebufferConsole { + fn putc(&self, c: u8) -> Result<(), Error> { + let mut pos = self.position.lock(); + + self.framebuffer.draw_glyph( + self.char_width * pos.col as usize, + self.char_height * pos.row as usize, + c, + ); + + if c == b'\n' { + pos.row += 1; + pos.col = 0; + } else { + pos.col += 1; + } + + if pos.col == self.width_chars as u32 { + pos.row += 1; + pos.col = 0; + } + if pos.row == self.height_chars as u32 { + pos.row = self.height_chars as u32 - 1; + } + + Ok(()) + } +} + +impl LinearFramebuffer { + pub fn from_yboot(fb: &FramebufferOption) -> Result { + let mmio = + unsafe { DeviceMemory::map("framebuffer", fb.res_address as _, fb.res_size as _) }?; + let inner = LinearFramebufferInner { + base: mmio.base(), + mmio, + stride: fb.res_stride as _, + width: fb.res_width as _, + height: fb.res_height as _, + }; + + Ok(Self { + inner: IrqSafeSpinlock::new(inner), + }) + } + + pub fn draw_glyph(&self, x: usize, y: usize, c: u8) { + let mut inner = self.inner.lock(); + let font = &bitmap_font::tamzen::FONT_6x12; + + let text_data = [c]; + let text_str = unsafe { core::str::from_utf8_unchecked(&text_data) }; + let text = Text::new( + text_str, + Point::new(x as _, y as _), + TextStyle::new(font, BinaryColor::On), + ); + + text.draw(&mut *inner).ok(); + } +} + +impl FramebufferConsole { + pub fn new(framebuffer: &'static LinearFramebuffer) -> Self { + let char_width = 6; + let char_height = 12; + let (w, h) = { + let inner = framebuffer.inner.lock(); + (inner.width, inner.height) + }; + + Self { + framebuffer, + position: IrqSafeSpinlock::new(Position { row: 0, col: 0 }), + width_chars: w / char_width, + height_chars: h / char_height, + char_width, + char_height, + } + } +} + +static DISPLAY: OneTimeInit = OneTimeInit::new(); +static CONSOLE: OneTimeInit = OneTimeInit::new(); + +extern "C" fn __x86_64_upper_entry() -> ! { + unsafe { + ARCHITECTURE.init_mmu(true); + core::arch::asm!("wbinvd"); + } + + let fb = unsafe { &YBOOT_DATA.opt_framebuffer }; + + DISPLAY.init(LinearFramebuffer::from_yboot(fb).unwrap()); + CONSOLE.init(FramebufferConsole::new(DISPLAY.get())); + + debug::init_with_sink(CONSOLE.get()); + + for i in 0..10 { + debugln!("Test {}", i); + } + + loop { + unsafe { + core::arch::asm!("cli; hlt"); + } + } + + // if let Ok(fb_mmio) = unsafe { DeviceMemory::map("framebuffer", fb.res_address as _, 0x1000) } { + // unsafe { + // core::arch::asm!("mov %cr3, %rax; mov %rax, %cr3", options(att_syntax)); + // } + // let addr = 0xffffff8140000000usize; + // let slice = unsafe { core::slice::from_raw_parts_mut(addr as *mut u32, 1024) }; + // slice.fill(0xFFFF0000); + // loop {} + // // for y in 0..2 { + // // let y_val = (y * 255) / fb.res_height; + + // // let addr = fb_mmio.base() + y as usize * fb.res_stride as usize; + // // let row = + // // unsafe { core::slice::from_raw_parts_mut(addr as *mut u32, fb.res_width as _) }; + + // // let v = 0xFF000000 | (y_val << 16) | (y_val << 8) | y_val; + + // // row.fill(v); + // // } + // } + + // unsafe { + // core::arch::asm!( + // r#" + // mov $0x3F8, %dx + // mov $'@', %al + + // out %al, %dx + // "#, + // options(att_syntax) + // ); + // } + + loop {} +} + +global_asm!( + r#" +.global __x86_64_entry + +.section .text.entry +__x86_64_entry: + mov ${yboot_loader_magic}, %edi + cmp %edi, %eax + je 2f + + // (Currently) unsupported bootloader +1: + cli + hlt + jmp 1b + +2: + // yboot entry method + movabsq ${stack_bottom} + {stack_size}, %rax + movabsq ${entry}, %rcx + mov %rax, %rsp + jmp *%rcx + +.section .text +"#, + yboot_loader_magic = const LOADER_MAGIC, + stack_size = const BOOT_STACK_SIZE, + stack_bottom = sym BSP_STACK, + entry = sym __x86_64_upper_entry, + options(att_syntax) +); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index b0a34b30..ce756c1a 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,73 +1,41 @@ -use core::arch::global_asm; +use abi::error::Error; -use yboot_proto::{ - v1::FramebufferOption, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, - PROTOCOL_VERSION_1, -}; +use crate::arch::x86_64::table::{init_fixed_tables, KERNEL_TABLES}; -pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; -const BOOT_STACK_SIZE: usize = 65536; +use super::Architecture; -#[repr(C, align(0x20))] -struct BootStack { - data: [u8; BOOT_STACK_SIZE], +pub mod boot; +pub mod table; + +pub struct X86_64; + +impl Architecture for X86_64 { + const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; + + unsafe fn init_mmu(&self, bsp: bool) { + if bsp { + init_fixed_tables(); + } + + let cr3 = KERNEL_TABLES.physical_address(); + core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); + } + + fn map_device_pages(&self, phys: usize, count: usize) -> Result { + unsafe { KERNEL_TABLES.map_device_pages(phys, count) } + } + + fn wait_for_interrupt() { + todo!() + } + + unsafe fn set_interrupt_mask(mask: bool) { + todo!() + } + + fn interrupt_mask() -> bool { + todo!() + } } -#[link_section = ".bss"] -static mut BSP_STACK: BootStack = BootStack { - data: [0; BOOT_STACK_SIZE], -}; - -#[used] -#[link_section = ".data.yboot"] -static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { - header: LoadProtocolHeader { - kernel_magic: KERNEL_MAGIC, - version: PROTOCOL_VERSION_1, - }, - kernel_virt_offset: KERNEL_VIRT_OFFSET as _, - - opt_framebuffer: FramebufferOption { - req_width: 1024, - req_height: 768, - - res_width: 0, - res_height: 0, - res_stride: 0, - res_address: 0, - }, -}; - -extern "C" fn __x86_64_upper_entry() -> ! { - loop {} -} - -global_asm!( - r#" -.global __x86_64_entry - -.section .text.entry -__x86_64_entry: - mov ${yboot_loader_magic}, %edi - cmp %edi, %eax - je 2f - - // (Currently) unsupported bootloader -1: - cli - hlt - jmp 1b - -2: - // yboot entry method - movabsq ${stack_bottom} + {stack_size}, %rax - movabsq ${entry}, %rcx - mov %rax, %rsp - jmp *%rcx -"#, - yboot_loader_magic = const LOADER_MAGIC, - stack_size = const BOOT_STACK_SIZE, - stack_bottom = sym BSP_STACK, - entry = sym __x86_64_upper_entry, - options(att_syntax) -); +pub static ARCHITECTURE: X86_64 = X86_64; diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs new file mode 100644 index 00000000..cdae45f4 --- /dev/null +++ b/src/arch/x86_64/table/fixed.rs @@ -0,0 +1,109 @@ +use abi::error::Error; + +use crate::{ + arch::x86_64::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3}, + mem::KERNEL_VIRT_OFFSET, +}; + +// Means 4 lower GiB are mapped +const KERNEL_PD_COUNT: usize = 4; +// Leave 1GiB gap just for fool safety +const DEVICE_MAPPING_L1I: usize = KERNEL_PD_COUNT + 1; +const DEVICE_VIRT_OFFSET: usize = (DEVICE_MAPPING_L1I << 30) + KERNEL_VIRT_OFFSET; + +pub struct FixedTables { + // Common + l0: PageTable, + l1: PageTable, + + // Kernel mapping + kernel_l2: [PageTable; KERNEL_PD_COUNT], + // Device mapping + // 511 entries + device_l2: PageTable, + // 512 entries + device_l3: PageTable, + + device_l3i: usize, +} + +impl FixedTables { + pub const fn zeroed() -> Self { + Self { + // Global + l0: PageTable::zeroed(), + + // Higher-half common + l1: PageTable::zeroed(), + + // Kernel + kernel_l2: [PageTable::zeroed(); KERNEL_PD_COUNT], + + // Device + device_l2: PageTable::zeroed(), + device_l3: PageTable::zeroed(), + + device_l3i: 0, + } + } + + pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result { + if count > 512 * 512 { + panic!("Unsupported device memory mapping size"); + } else if count > 512 { + // 2MiB mappings + todo!(); + } else { + // 4KiB mappings + if self.device_l3i + count > 512 { + return Err(Error::OutOfMemory); + } + + let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12); + for i in 0..count { + self.device_l3[self.device_l3i + i] = + PageEntry::page(phys + i * 0x1000, PageAttributes::WRITABLE); + } + self.device_l3i += count; + + Ok(virt) + } + } + + pub fn physical_address(&self) -> usize { + self.l0.physical_address() + } +} + +pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed(); + +pub unsafe fn init_fixed_tables() { + // Kernel L2 + for i in 0..512 * KERNEL_PD_COUNT { + let table_index = i / 512; + let table_offset = i % 512; + + KERNEL_TABLES.kernel_l2[table_index][table_offset] = + PageEntry::block(i << 21, PageAttributes::WRITABLE); + } + + // Device L2 + let addr = KERNEL_TABLES.device_l3.physical_address(); + KERNEL_TABLES.device_l2[0] = PageEntry::table(addr, PageAttributes::empty()); + + // Higher-half L1 + // Map kernel nGiB + for i in 0..KERNEL_PD_COUNT { + let addr = KERNEL_TABLES.kernel_l2[i].physical_address(); + KERNEL_TABLES.l1[i] = PageEntry::table(addr, PageAttributes::empty()); + } + + // Map device tables + let addr = KERNEL_TABLES.device_l2.physical_address(); + KERNEL_TABLES.l1[DEVICE_MAPPING_L1I] = PageEntry::table(addr, PageAttributes::empty()); + + // Global L0 + let addr = KERNEL_TABLES.l1.physical_address(); + // No lower mapping anymore + KERNEL_TABLES.l0[511] = PageEntry::table(addr, PageAttributes::empty()); +} diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/table/mod.rs new file mode 100644 index 00000000..672d1289 --- /dev/null +++ b/src/arch/x86_64/table/mod.rs @@ -0,0 +1,158 @@ +use core::{ + marker::PhantomData, + ops::{Index, IndexMut}, +}; + +use bitflags::bitflags; + +mod fixed; + +pub use fixed::{init_fixed_tables, KERNEL_TABLES}; + +use crate::mem::{ + table::{EntryLevel, NonTerminalEntryLevel}, + ConvertAddress, +}; + +bitflags! { + pub struct PageAttributes: u64 { + const PRESENT = 1 << 0; + const WRITABLE = 1 << 1; + const BLOCK = 1 << 7; + } +} + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct PageEntry(u64, PhantomData); + +#[derive(Clone, Copy)] +#[repr(C, align(0x1000))] +pub struct PageTable { + data: [PageEntry; 512], +} + +// L0: PML4, 512GiB page +#[derive(Clone, Copy)] +pub struct L0; +// L1: PDPT, 1GiB page +#[derive(Clone, Copy)] +pub struct L1; +// L2: Page directory, 2MiB page +#[derive(Clone, Copy)] +pub struct L2; +// L3: Page table, 4KiB page +#[derive(Clone, Copy)] +pub struct L3; + +impl NonTerminalEntryLevel for L0 { + type NextLevel = L1; +} +impl NonTerminalEntryLevel for L1 { + type NextLevel = L2; +} +impl NonTerminalEntryLevel for L2 { + type NextLevel = L3; +} + +// #[repr(C)] +// pub struct AddressSpace { +// l0: *mut PageTable, +// } + +impl const EntryLevel for L0 { + fn index(addr: usize) -> usize { + todo!() + } + + fn page_offset(addr: usize) -> usize { + todo!() + } +} + +impl const EntryLevel for L1 { + fn index(addr: usize) -> usize { + (addr >> 30) & 0x1FF + } + + fn page_offset(addr: usize) -> usize { + addr & 0x3FFFFFFF + } +} + +impl const EntryLevel for L2 { + fn index(addr: usize) -> usize { + (addr >> 21) & 0x1FF + } + + fn page_offset(addr: usize) -> usize { + addr & 0x1FFFFF + } +} + +impl const EntryLevel for L3 { + fn index(addr: usize) -> usize { + (addr >> 12) & 0x1FF + } + + fn page_offset(addr: usize) -> usize { + addr & 0xFFF + } +} + +impl PageEntry { + pub fn page(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) | (attrs | PageAttributes::PRESENT).bits(), + PhantomData, + ) + } +} + +impl PageEntry { + pub fn block(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), + PhantomData, + ) + } +} + +impl PageEntry { + pub fn table(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::WRITABLE).bits(), + PhantomData, + ) + } +} + +impl PageEntry { + pub const INVALID: Self = Self(0, PhantomData); +} + +impl PageTable { + pub const fn zeroed() -> Self { + Self { + data: [PageEntry::INVALID; 512], + } + } + + pub fn physical_address(&self) -> usize { + unsafe { (self.data.as_ptr() as usize).physicalize() } + } +} + +impl Index for PageTable { + type Output = PageEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.data[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.data[index] + } +} diff --git a/src/arch/x86_64/tables.rs b/src/arch/x86_64/tables.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/src/debug.rs b/src/debug.rs index 876b908e..df707292 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -1,12 +1,16 @@ //! Utilities for debug information logging use core::fmt::{self, Arguments}; -use crate::{ - arch::PLATFORM, - device::{platform::Platform, serial::SerialDevice}, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use abi::error::Error; + +use crate::{sync::IrqSafeSpinlock, util::OneTimeInit}; + +// use crate::{ +// arch::PLATFORM, +// device::{platform::Platform, serial::SerialDevice}, +// sync::IrqSafeSpinlock, +// util::OneTimeInit, +// }; /// Defines the severity of the message #[derive(Clone, Copy)] @@ -23,8 +27,16 @@ pub enum LogLevel { Fatal, } +pub trait DebugSink { + fn putc(&self, c: u8) -> Result<(), Error>; + + fn supports_color(&self) -> bool { + return false; + } +} + struct DebugPrinter { - sink: &'static dyn SerialDevice, + sink: &'static dyn DebugSink, } macro_rules! log_print_raw { @@ -35,7 +47,7 @@ macro_rules! log_print_raw { macro_rules! log_print { ($level:expr, $($args:tt)+) => { - log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::arch::aarch64::cpu::Cpu::local_id(), file!(), line!(), format_args!($($args)+)) + log_print_raw!($level, "{}:{}: {}", file!(), line!(), format_args!($($args)+)) }; } @@ -92,7 +104,7 @@ impl LogLevel { impl fmt::Write for DebugPrinter { fn write_str(&mut self, s: &str) -> fmt::Result { for c in s.bytes() { - self.sink.send(c).ok(); + self.sink.putc(c).ok(); } Ok(()) @@ -126,14 +138,18 @@ pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { /// /// Will panic if called more than once. pub fn init() { - DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { - sink: PLATFORM.primary_serial().unwrap(), - })); - unsafe { - vfs::init_debug_hook(&move |args| { - debug_internal(args, LogLevel::Debug); - }); - } + // DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { + // sink: PLATFORM.primary_serial().unwrap(), + // })); + // unsafe { + // vfs::init_debug_hook(&move |args| { + // debug_internal(args, LogLevel::Debug); + // }); + // } +} + +pub fn init_with_sink(sink: &'static dyn DebugSink) { + DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { sink })); } #[doc(hidden)] @@ -143,8 +159,8 @@ pub fn debug_internal(args: Arguments, level: LogLevel) { if DEBUG_PRINTER.is_initialized() { let mut printer = DEBUG_PRINTER.get().lock(); - printer.write_str(level.log_prefix()).ok(); + // printer.write_str(level.log_prefix()).ok(); printer.write_fmt(args).ok(); - printer.write_str(level.log_suffix()).ok(); + // printer.write_str(level.log_suffix()).ok(); } } diff --git a/src/main.rs b/src/main.rs index ab5246ba..3d739565 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,7 @@ #![no_std] #![no_main] -// extern crate yggdrasil_abi as abi; +extern crate yggdrasil_abi as abi; // // use abi::{ // error::Error, @@ -29,8 +29,8 @@ // // extern crate alloc; // -// #[macro_use] -// pub mod debug; +#[macro_use] +pub mod debug; #[macro_use] pub mod arch; @@ -41,13 +41,13 @@ fn panic_handler(_pi: &core::panic::PanicInfo) -> ! { // // pub mod device; // pub mod fs; -// pub mod mem; +pub mod mem; // pub mod panic; // pub mod proc; -// pub mod sync; +pub mod sync; // pub mod syscall; // pub mod task; -// pub mod util; +pub mod util; // // fn setup_root() -> Result { // let initrd_data = INITRD_DATA.get(); diff --git a/src/mem/device.rs b/src/mem/device.rs index 898ddda8..d689a6eb 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -29,14 +29,20 @@ impl DeviceMemory { /// points to some device's MMIO. The caller must also make sure no aliasing for that range is /// possible. pub unsafe fn map(name: &'static str, phys: usize, size: usize) -> Result { - if size > 0x1000 { - todo!("Device memory mappings larger than 4K"); - } + let aligned_base = phys & !0xFFF; + let base_offset = phys & 0xFFF; + let aligned_size = (size + 0xFFF) & !0xFFF; - let base = ARCHITECTURE.map_device_pages(phys, 1)?; + let base = ARCHITECTURE.map_device_pages(aligned_base, aligned_size / 0x1000)?; + let base = base + base_offset; Ok(Self { name, base, size }) } + + #[inline] + pub fn base(&self) -> usize { + self.base + } } impl DeviceMemoryIo { diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 725d7d37..5c8f2c7a 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,25 +1,27 @@ //! Memory management utilities and types use core::{alloc::Layout, mem::size_of}; -use abi::error::Error; - -use crate::{ - arch::{Architecture, ArchitectureImpl, PlatformImpl}, - device::platform::Platform, -}; - -use self::table::AddressSpace; +// use abi::error::Error; +// +// use crate::{ +// arch::{Architecture, ArchitectureImpl, PlatformImpl}, +// device::platform::Platform, +// }; +// +// use self::table::AddressSpace; pub mod device; -pub mod heap; -pub mod phys; +// pub mod heap; +// pub mod phys; pub mod table; /// Kernel's physical load address -pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; +// pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; /// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + /// [KERNEL_VIRT_OFFSET]) -pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; +// pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; +// TODO fix this +pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; /// Interface for converting between address spaces. /// @@ -49,74 +51,74 @@ pub unsafe trait ConvertAddress { unsafe fn physicalize(self) -> Self; } -/// Helper trait to allow cross-address space access to pointers -pub trait ForeignPointer: Sized { - /// Perform a volatile pointer write without dropping the old value. - /// - /// # Panics - /// - /// The function panics if any of the following conditions is met: - /// - /// * The address of the pointer is not mapped in the `space`. - /// * The pointer is not writable. - /// * The pointer is misaligned. - /// - /// # Safety - /// - /// As this function allows direct memory writes, it is inherently unsafe. - unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); - - /// Performs pointer validation for given address space: - /// - /// * Checks if the pointer has proper alignment for the type. - /// * Checks if the pointer is mapped in the address space. - /// * Checks if the pointer is above the userspace memory boundary. - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_ptr<'a>( - self: *const Self, - space: &AddressSpace, - ) -> Result<&'a Self, Error>; - - /// [ForeignPointer::validate_user_ptr], with extra "writability" check. - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_mut<'a>( - self: *mut Self, - space: &AddressSpace, - ) -> Result<&'a mut Self, Error>; - - /// [ForeignPointer::validate_user_ptr], but for slices - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_slice<'a>( - self: *const Self, - len: usize, - space: &AddressSpace, - ) -> Result<&'a [Self], Error>; - - /// [ForeignPointer::validate_user_slice], but for mutable slices - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_slice_mut<'a>( - self: *mut Self, - len: usize, - space: &AddressSpace, - ) -> Result<&'a mut [Self], Error>; -} - +// /// Helper trait to allow cross-address space access to pointers +// pub trait ForeignPointer: Sized { +// /// Perform a volatile pointer write without dropping the old value. +// /// +// /// # Panics +// /// +// /// The function panics if any of the following conditions is met: +// /// +// /// * The address of the pointer is not mapped in the `space`. +// /// * The pointer is not writable. +// /// * The pointer is misaligned. +// /// +// /// # Safety +// /// +// /// As this function allows direct memory writes, it is inherently unsafe. +// unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); +// +// /// Performs pointer validation for given address space: +// /// +// /// * Checks if the pointer has proper alignment for the type. +// /// * Checks if the pointer is mapped in the address space. +// /// * Checks if the pointer is above the userspace memory boundary. +// /// +// /// # Safety +// /// +// /// Even though this function does the necessary checks, it is still a raw pointer to reference +// /// conversion, and thus is unsafe. +// unsafe fn validate_user_ptr<'a>( +// self: *const Self, +// space: &AddressSpace, +// ) -> Result<&'a Self, Error>; +// +// /// [ForeignPointer::validate_user_ptr], with extra "writability" check. +// /// +// /// # Safety +// /// +// /// Even though this function does the necessary checks, it is still a raw pointer to reference +// /// conversion, and thus is unsafe. +// unsafe fn validate_user_mut<'a>( +// self: *mut Self, +// space: &AddressSpace, +// ) -> Result<&'a mut Self, Error>; +// +// /// [ForeignPointer::validate_user_ptr], but for slices +// /// +// /// # Safety +// /// +// /// Even though this function does the necessary checks, it is still a raw pointer to reference +// /// conversion, and thus is unsafe. +// unsafe fn validate_user_slice<'a>( +// self: *const Self, +// len: usize, +// space: &AddressSpace, +// ) -> Result<&'a [Self], Error>; +// +// /// [ForeignPointer::validate_user_slice], but for mutable slices +// /// +// /// # Safety +// /// +// /// Even though this function does the necessary checks, it is still a raw pointer to reference +// /// conversion, and thus is unsafe. +// unsafe fn validate_user_slice_mut<'a>( +// self: *mut Self, +// len: usize, +// space: &AddressSpace, +// ) -> Result<&'a mut [Self], Error>; +// } +// unsafe impl ConvertAddress for usize { #[inline(always)] unsafe fn virtualize(self) -> Self { @@ -163,120 +165,120 @@ unsafe impl ConvertAddress for *const T { } } -impl ForeignPointer for T { - unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { - // TODO check align - let addr = self as usize; - let start_page = addr & !0xFFF; - let end_page = (addr + size_of::() - 1) & !0xFFF; - let page_offset = addr & 0xFFF; - - if start_page != end_page { - todo!("Foreign pointer write crossed a page boundary"); - } - - let phys_page = space - .translate(start_page) - .expect("Address is not mapped in the target address space"); - - let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; - virt_ptr.write_volatile(value); - } - - unsafe fn validate_user_slice_mut<'a>( - self: *mut Self, - len: usize, - space: &AddressSpace, - ) -> Result<&'a mut [Self], Error> { - let base = self as usize; - let layout = Layout::array::(len).unwrap(); - - validate_user_align_size(base, &layout)?; - validate_user_region(space, base, layout.size(), true)?; - - Ok(core::slice::from_raw_parts_mut(self, len)) - } - - unsafe fn validate_user_slice<'a>( - self: *const Self, - len: usize, - space: &AddressSpace, - ) -> Result<&'a [Self], Error> { - let base = self as usize; - let layout = Layout::array::(len).unwrap(); - - validate_user_align_size(base, &layout)?; - validate_user_region(space, base, layout.size(), false)?; - - Ok(core::slice::from_raw_parts(self, len)) - } - - unsafe fn validate_user_mut<'a>( - self: *mut Self, - space: &AddressSpace, - ) -> Result<&'a mut Self, Error> { - let addr = self as usize; - let layout = Layout::new::(); - - // Common validation - validate_user_align_size(addr, &layout)?; - - // Validate that the pages covered by this address are mapped as writable by the process - // TODO for CoW this may differ - validate_user_region(space, addr, layout.size(), true)?; - - Ok(&mut *self) - } - - unsafe fn validate_user_ptr<'a>( - self: *const Self, - space: &AddressSpace, - ) -> Result<&'a Self, Error> { - let addr = self as usize; - let layout = Layout::new::(); - - // Common validation - validate_user_align_size(addr, &layout)?; - validate_user_region(space, addr, layout.size(), false)?; - - Ok(&*self) - } -} - -fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { - // Explicitly disallow NULL - if addr == 0 { - return Err(Error::InvalidArgument); - } - // Validate alignment - if addr % layout.align() != 0 { - return Err(Error::InvalidArgument); - } - if addr + layout.size() > KERNEL_VIRT_OFFSET { - todo!(); - } - - Ok(()) -} - -/// Validates access to given userspace memory region with given constraints -pub fn validate_user_region( - space: &AddressSpace, - base: usize, - len: usize, - _need_write: bool, -) -> Result<(), Error> { - if base + len > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } - - let aligned_start = base & !0xFFF; - let aligned_end = (base + len + 0xFFF) & !0xFFF; - - for page in (aligned_start..aligned_end).step_by(0x1000) { - // TODO check writability - space.translate(page).ok_or(Error::InvalidArgument)?; - } - - Ok(()) -} +// impl ForeignPointer for T { +// unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { +// // TODO check align +// let addr = self as usize; +// let start_page = addr & !0xFFF; +// let end_page = (addr + size_of::() - 1) & !0xFFF; +// let page_offset = addr & 0xFFF; +// +// if start_page != end_page { +// todo!("Foreign pointer write crossed a page boundary"); +// } +// +// let phys_page = space +// .translate(start_page) +// .expect("Address is not mapped in the target address space"); +// +// let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; +// virt_ptr.write_volatile(value); +// } +// +// unsafe fn validate_user_slice_mut<'a>( +// self: *mut Self, +// len: usize, +// space: &AddressSpace, +// ) -> Result<&'a mut [Self], Error> { +// let base = self as usize; +// let layout = Layout::array::(len).unwrap(); +// +// validate_user_align_size(base, &layout)?; +// validate_user_region(space, base, layout.size(), true)?; +// +// Ok(core::slice::from_raw_parts_mut(self, len)) +// } +// +// unsafe fn validate_user_slice<'a>( +// self: *const Self, +// len: usize, +// space: &AddressSpace, +// ) -> Result<&'a [Self], Error> { +// let base = self as usize; +// let layout = Layout::array::(len).unwrap(); +// +// validate_user_align_size(base, &layout)?; +// validate_user_region(space, base, layout.size(), false)?; +// +// Ok(core::slice::from_raw_parts(self, len)) +// } +// +// unsafe fn validate_user_mut<'a>( +// self: *mut Self, +// space: &AddressSpace, +// ) -> Result<&'a mut Self, Error> { +// let addr = self as usize; +// let layout = Layout::new::(); +// +// // Common validation +// validate_user_align_size(addr, &layout)?; +// +// // Validate that the pages covered by this address are mapped as writable by the process +// // TODO for CoW this may differ +// validate_user_region(space, addr, layout.size(), true)?; +// +// Ok(&mut *self) +// } +// +// unsafe fn validate_user_ptr<'a>( +// self: *const Self, +// space: &AddressSpace, +// ) -> Result<&'a Self, Error> { +// let addr = self as usize; +// let layout = Layout::new::(); +// +// // Common validation +// validate_user_align_size(addr, &layout)?; +// validate_user_region(space, addr, layout.size(), false)?; +// +// Ok(&*self) +// } +// } +// +// fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { +// // Explicitly disallow NULL +// if addr == 0 { +// return Err(Error::InvalidArgument); +// } +// // Validate alignment +// if addr % layout.align() != 0 { +// return Err(Error::InvalidArgument); +// } +// if addr + layout.size() > KERNEL_VIRT_OFFSET { +// todo!(); +// } +// +// Ok(()) +// } +// +// /// Validates access to given userspace memory region with given constraints +// pub fn validate_user_region( +// space: &AddressSpace, +// base: usize, +// len: usize, +// _need_write: bool, +// ) -> Result<(), Error> { +// if base + len > crate::mem::KERNEL_VIRT_OFFSET { +// panic!("Invalid argument"); +// } +// +// let aligned_start = base & !0xFFF; +// let aligned_end = (base + len + 0xFFF) & !0xFFF; +// +// for page in (aligned_start..aligned_end).step_by(0x1000) { +// // TODO check writability +// space.translate(page).ok_or(Error::InvalidArgument)?; +// } +// +// Ok(()) +// } diff --git a/src/mem/table.rs b/src/mem/table.rs index 0456bca3..317b4262 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -1,7 +1,14 @@ //! Virtual memory table interface use abi::error::Error; +use cfg_if::cfg_if; -pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; + } else if #[cfg(target_arch = "x86_64")] { + pub use crate::arch::x86_64::table::{PageAttributes}; + } +} /// Interface for virtual memory address space management pub trait VirtualMemoryManager { @@ -38,3 +45,9 @@ pub trait EntryLevel: Copy { /// Returns the offset of an address from the page start at current level fn page_offset(addr: usize) -> usize; } + +/// Tag trait to mark that the page table level may point to a next-level table +pub trait NonTerminalEntryLevel: EntryLevel { + /// Tag type of the level this entry level may point to + type NextLevel: EntryLevel; +} diff --git a/src/sync.rs b/src/sync.rs index 042388f8..baf080c9 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -5,9 +5,6 @@ use core::{ sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; -use aarch64_cpu::registers::DAIF; -use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; - /// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is /// met. pub struct SpinFence { @@ -132,15 +129,16 @@ impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { impl IrqGuard { /// Saves the current IRQ state and masks them pub fn acquire() -> Self { - let this = Self(DAIF.get()); - DAIF.modify(DAIF::I::SET); - this + Self(0) + // let this = Self(DAIF.get()); + // DAIF.modify(DAIF::I::SET); + // this } } impl Drop for IrqGuard { fn drop(&mut self) { - DAIF.set(self.0); + // DAIF.set(self.0); } } From 47f920addfcde46e4a937050112d6a7dbe1d838b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 29 Jul 2023 21:11:15 +0300 Subject: [PATCH 024/211] x86_64: cleanup linear fb code a bit --- Cargo.toml | 2 +- src/arch/mod.rs | 7 +- src/arch/x86_64/boot/mod.rs | 224 +++---------------------------- src/arch/x86_64/mod.rs | 79 ++++++++++- src/debug.rs | 20 +-- src/device/display/fb_console.rs | 156 +++++++++++++++++++++ src/device/display/linear_fb.rs | 142 ++++++++++++++++++++ src/device/display/mod.rs | 14 ++ src/device/mod.rs | 9 +- src/device/platform.rs | 61 +++++---- src/main.rs | 2 +- 11 files changed, 462 insertions(+), 254 deletions(-) create mode 100644 src/device/display/fb_console.rs create mode 100644 src/device/display/linear_fb.rs create mode 100644 src/device/display/mod.rs diff --git a/Cargo.toml b/Cargo.toml index e1067719..06cfa7f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ bitflags = "2.3.3" # static_assertions = "1.1.0" # tock-registers = "0.8.1" cfg-if = "1.0.0" +bitmap-font = { version = "0.3.0" } embedded-graphics = "0.8.0" [dependencies.elf] @@ -30,5 +31,4 @@ fdt-rs = { version = "0.4.3", default-features = false } aarch64-cpu = "9.3.1" [target.'cfg(target_arch = "x86_64")'.dependencies] -bitmap-font = { version = "0.3.0" } yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 3775b552..c8231cd5 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -15,7 +15,12 @@ cfg_if! { } else if #[cfg(target_arch = "x86_64")] { pub mod x86_64; - pub use x86_64::{X86_64 as ArchitectureImpl, ARCHITECTURE}; + pub use x86_64::{ + X86_64 as ArchitectureImpl, + X86_64 as PlatformImpl, + ARCHITECTURE, + ARCHITECTURE as PLATFORM + }; } else { compile_error!("Architecture is not supported"); } diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 520924dc..45446adf 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,24 +1,14 @@ use core::arch::global_asm; -use abi::error::Error; -use bitmap_font::TextStyle; -use embedded_graphics::{ - pixelcolor::BinaryColor, - prelude::{DrawTarget, OriginDimensions, Point}, - text::Text, - Drawable, Pixel, -}; use yboot_proto::{ v1::FramebufferOption, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, }; use crate::{ - arch::Architecture, - debug::{self, DebugSink}, - mem::device::DeviceMemory, - sync::IrqSafeSpinlock, - util::OneTimeInit, + arch::{Architecture, ArchitectureImpl}, + debug, + device::platform::Platform, }; use super::ARCHITECTURE; @@ -57,206 +47,28 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { }, }; -struct LinearFramebufferInner { - mmio: DeviceMemory, - base: usize, - stride: usize, - width: usize, - height: usize, -} +unsafe extern "C" fn __x86_64_upper_entry() -> ! { + ArchitectureImpl::set_interrupt_mask(true); -pub struct LinearFramebuffer { - inner: IrqSafeSpinlock, -} + ARCHITECTURE.init_mmu(true); + core::arch::asm!("wbinvd"); -struct Position { - row: u32, - col: u32, -} + ARCHITECTURE + .yboot_framebuffer + .init(YBOOT_DATA.opt_framebuffer); + ARCHITECTURE.init_primary_debug_sink(); -pub struct FramebufferConsole { - framebuffer: &'static LinearFramebuffer, - position: IrqSafeSpinlock, - char_height: usize, - char_width: usize, - width_chars: usize, - height_chars: usize, -} - -impl OriginDimensions for LinearFramebufferInner { - fn size(&self) -> embedded_graphics::prelude::Size { - embedded_graphics::prelude::Size::new(self.width as _, self.height as _) - } -} - -impl DrawTarget for LinearFramebufferInner { - type Color = BinaryColor; - type Error = (); - - fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> - where - I: IntoIterator>, - { - for Pixel(coord, color) in pixels { - let x = coord.x as usize; - let y = coord.y as usize; - let addr = self.base + y * self.stride + x * 4; - let ptr = addr as *mut u32; - unsafe { - if color.is_on() { - ptr.write_volatile(0xFFFFFFFF); - } else { - ptr.write_volatile(0); - } - } - } - - Ok(()) - } -} - -impl DebugSink for FramebufferConsole { - fn putc(&self, c: u8) -> Result<(), Error> { - let mut pos = self.position.lock(); - - self.framebuffer.draw_glyph( - self.char_width * pos.col as usize, - self.char_height * pos.row as usize, - c, - ); - - if c == b'\n' { - pos.row += 1; - pos.col = 0; - } else { - pos.col += 1; - } - - if pos.col == self.width_chars as u32 { - pos.row += 1; - pos.col = 0; - } - if pos.row == self.height_chars as u32 { - pos.row = self.height_chars as u32 - 1; - } - - Ok(()) - } -} - -impl LinearFramebuffer { - pub fn from_yboot(fb: &FramebufferOption) -> Result { - let mmio = - unsafe { DeviceMemory::map("framebuffer", fb.res_address as _, fb.res_size as _) }?; - let inner = LinearFramebufferInner { - base: mmio.base(), - mmio, - stride: fb.res_stride as _, - width: fb.res_width as _, - height: fb.res_height as _, - }; - - Ok(Self { - inner: IrqSafeSpinlock::new(inner), - }) - } - - pub fn draw_glyph(&self, x: usize, y: usize, c: u8) { - let mut inner = self.inner.lock(); - let font = &bitmap_font::tamzen::FONT_6x12; - - let text_data = [c]; - let text_str = unsafe { core::str::from_utf8_unchecked(&text_data) }; - let text = Text::new( - text_str, - Point::new(x as _, y as _), - TextStyle::new(font, BinaryColor::On), - ); - - text.draw(&mut *inner).ok(); - } -} - -impl FramebufferConsole { - pub fn new(framebuffer: &'static LinearFramebuffer) -> Self { - let char_width = 6; - let char_height = 12; - let (w, h) = { - let inner = framebuffer.inner.lock(); - (inner.width, inner.height) - }; - - Self { - framebuffer, - position: IrqSafeSpinlock::new(Position { row: 0, col: 0 }), - width_chars: w / char_width, - height_chars: h / char_height, - char_width, - char_height, - } - } -} - -static DISPLAY: OneTimeInit = OneTimeInit::new(); -static CONSOLE: OneTimeInit = OneTimeInit::new(); - -extern "C" fn __x86_64_upper_entry() -> ! { - unsafe { - ARCHITECTURE.init_mmu(true); - core::arch::asm!("wbinvd"); - } - - let fb = unsafe { &YBOOT_DATA.opt_framebuffer }; - - DISPLAY.init(LinearFramebuffer::from_yboot(fb).unwrap()); - CONSOLE.init(FramebufferConsole::new(DISPLAY.get())); - - debug::init_with_sink(CONSOLE.get()); - - for i in 0..10 { - debugln!("Test {}", i); - } + debug::init(); + let mut i = 0; loop { - unsafe { - core::arch::asm!("cli; hlt"); - } + debugln!("{} {}", env!("PROFILE"), i); + i += 1; } - // if let Ok(fb_mmio) = unsafe { DeviceMemory::map("framebuffer", fb.res_address as _, 0x1000) } { - // unsafe { - // core::arch::asm!("mov %cr3, %rax; mov %rax, %cr3", options(att_syntax)); - // } - // let addr = 0xffffff8140000000usize; - // let slice = unsafe { core::slice::from_raw_parts_mut(addr as *mut u32, 1024) }; - // slice.fill(0xFFFF0000); - // loop {} - // // for y in 0..2 { - // // let y_val = (y * 255) / fb.res_height; - - // // let addr = fb_mmio.base() + y as usize * fb.res_stride as usize; - // // let row = - // // unsafe { core::slice::from_raw_parts_mut(addr as *mut u32, fb.res_width as _) }; - - // // let v = 0xFF000000 | (y_val << 16) | (y_val << 8) | y_val; - - // // row.fill(v); - // // } + // loop { + // ArchitectureImpl::wait_for_interrupt(); // } - - // unsafe { - // core::arch::asm!( - // r#" - // mov $0x3F8, %dx - // mov $'@', %al - - // out %al, %dx - // "#, - // options(att_syntax) - // ); - // } - - loop {} } global_asm!( @@ -280,7 +92,7 @@ __x86_64_entry: movabsq ${stack_bottom} + {stack_size}, %rax movabsq ${entry}, %rcx mov %rax, %rsp - jmp *%rcx + callq *%rcx .section .text "#, diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index ce756c1a..922cd604 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,13 +1,24 @@ use abi::error::Error; +use yboot_proto::v1::FramebufferOption; -use crate::arch::x86_64::table::{init_fixed_tables, KERNEL_TABLES}; +use crate::{ + arch::x86_64::table::{init_fixed_tables, KERNEL_TABLES}, + debug::DebugSink, + device::{ + display::{fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, + platform::Platform, + }, + util::OneTimeInit, +}; use super::Architecture; pub mod boot; pub mod table; -pub struct X86_64; +pub struct X86_64 { + yboot_framebuffer: OneTimeInit, +} impl Architecture for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; @@ -26,16 +37,72 @@ impl Architecture for X86_64 { } fn wait_for_interrupt() { - todo!() + unsafe { + core::arch::asm!("hlt"); + } } unsafe fn set_interrupt_mask(mask: bool) { - todo!() + if mask { + core::arch::asm!("cli"); + } else { + core::arch::asm!("sti"); + } } fn interrupt_mask() -> bool { - todo!() + let mut flags: u64; + unsafe { + core::arch::asm!("pushfd; pop {0}", out(reg) flags, options(att_syntax)); + } + // If IF is zero, interrupts are disabled (masked) + flags & (1 << 9) == 0 } } -pub static ARCHITECTURE: X86_64 = X86_64; +impl Platform for X86_64 { + unsafe fn init(&'static self, _is_bsp: bool) -> Result<(), Error> { + Ok(()) + } + + unsafe fn init_primary_debug_sink(&self) { + let Some(fb) = self.yboot_framebuffer.try_get() else { + // TODO fallback to serial as primary + return; + }; + + LINEAR_FB.init( + LinearFramebuffer::from_physical_bits( + fb.res_address as _, + fb.res_size as _, + fb.res_stride as _, + fb.res_width, + fb.res_height, + ) + .unwrap(), + ); + FB_CONSOLE.init(FramebufferConsole::from_framebuffer( + LINEAR_FB.get(), + &bitmap_font::tamzen::FONT_6x12, + )); + } + + fn name(&self) -> &'static str { + "x86-64" + } + + fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { + if let Some(console) = FB_CONSOLE.try_get() { + Some(console) + } else { + None + } + } +} + +pub static ARCHITECTURE: X86_64 = X86_64 { + yboot_framebuffer: OneTimeInit::new(), +}; + +static LINEAR_FB: OneTimeInit = OneTimeInit::new(); +static FB_CONSOLE: OneTimeInit = OneTimeInit::new(); diff --git a/src/debug.rs b/src/debug.rs index df707292..6db53a88 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -3,7 +3,7 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use crate::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use crate::{arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, util::OneTimeInit}; // use crate::{ // arch::PLATFORM, @@ -30,6 +30,13 @@ pub enum LogLevel { pub trait DebugSink { fn putc(&self, c: u8) -> Result<(), Error>; + fn puts(&self, s: &str) -> Result<(), Error> { + for &byte in s.as_bytes() { + self.putc(byte)?; + } + Ok(()) + } + fn supports_color(&self) -> bool { return false; } @@ -103,10 +110,7 @@ impl LogLevel { impl fmt::Write for DebugPrinter { fn write_str(&mut self, s: &str) -> fmt::Result { - for c in s.bytes() { - self.sink.putc(c).ok(); - } - + self.sink.puts(s).ok(); Ok(()) } } @@ -138,9 +142,9 @@ pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { /// /// Will panic if called more than once. pub fn init() { - // DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { - // sink: PLATFORM.primary_serial().unwrap(), - // })); + DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { + sink: PLATFORM.primary_debug_sink().unwrap(), + })); // unsafe { // vfs::init_debug_hook(&move |args| { // debug_internal(args, LogLevel::Debug); diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs new file mode 100644 index 00000000..32cdeea6 --- /dev/null +++ b/src/device/display/fb_console.rs @@ -0,0 +1,156 @@ +use abi::error::Error; +use bitmap_font::{BitmapFont, TextStyle}; +use embedded_graphics::{ + pixelcolor::BinaryColor, + prelude::{DrawTarget, OriginDimensions, Point, Size}, + text::Text, + Drawable, Pixel, +}; + +use crate::{debug::DebugSink, sync::IrqSafeSpinlock}; + +use super::{linear_fb::LinearFramebuffer, DisplayDevice, DisplayDimensions}; + +struct Inner { + framebuffer: &'static LinearFramebuffer, + font: &'static BitmapFont<'static>, + row: u32, + col: u32, + char_height: u32, + char_width: u32, + width_chars: u32, + height_chars: u32, +} + +pub struct FramebufferConsole { + inner: IrqSafeSpinlock, +} + +impl FramebufferConsole { + pub fn from_framebuffer( + framebuffer: &'static LinearFramebuffer, + font: &'static BitmapFont<'static>, + ) -> Self { + let char_width = 6; + let char_height = 12; + let dim = framebuffer.dimensions(); + + let inner = Inner { + framebuffer, + font, + row: 0, + col: 0, + width_chars: dim.width / char_width, + height_chars: dim.height / char_height, + char_width, + char_height, + }; + + Self { + inner: IrqSafeSpinlock::new(inner), + } + } +} + +impl Inner { + fn putc(&mut self, c: u8) { + self.draw_glyph( + self.font, + self.char_width * self.col, + self.char_height * self.row, + c, + ); + + if c == b'\n' { + self.col = 0; + self.row += 1; + } else { + self.col += 1; + } + + if self.col == self.width_chars { + self.col = 0; + self.row += 1; + } + + self.scroll(); + // if self.row == self.height_chars { + // self.row = self.height_chars - 1; + // } + } + + fn scroll(&mut self) { + // Should only happen once + while self.row >= self.height_chars { + let mut fb = unsafe { self.framebuffer.lock() }; + + fb.copy_rows(self.char_height, 0, self.char_height * self.height_chars); + fb.fill_rows( + (self.height_chars - 1) * self.char_height, + self.char_height, + 0x00000000, + ); + + self.row -= 1; + } + } + + fn draw_glyph(&mut self, font: &BitmapFont, x: u32, y: u32, c: u8) { + let text_data = [c]; + let text_str = unsafe { core::str::from_utf8_unchecked(&text_data) }; + let text = Text::new( + text_str, + Point::new(x as _, y as _), + TextStyle::new(font, BinaryColor::On), + ); + + text.draw(self).ok(); + } +} + +impl DebugSink for FramebufferConsole { + fn putc(&self, c: u8) -> Result<(), Error> { + self.inner.lock().putc(c); + Ok(()) + } + + fn supports_color(&self) -> bool { + false + } +} + +impl OriginDimensions for Inner { + fn size(&self) -> Size { + self.framebuffer.dimensions().into() + } +} + +impl DrawTarget for Inner { + type Color = BinaryColor; + type Error = (); + + fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> + where + I: IntoIterator>, + { + let mut fb = unsafe { self.framebuffer.lock() }; + for Pixel(coord, color) in pixels { + let row = &mut fb[coord.y as u32]; + let color = if color.is_on() { + 0xFFFFFFFF + } else { + 0xFF000000 + }; + + row[coord.x as usize] = color; + } + + Ok(()) + } +} + +impl From for Size { + fn from(value: DisplayDimensions) -> Self { + Self::new(value.width as _, value.height as _) + } +} diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs new file mode 100644 index 00000000..1abd302e --- /dev/null +++ b/src/device/display/linear_fb.rs @@ -0,0 +1,142 @@ +use core::ops::{Index, IndexMut}; + +use abi::error::Error; + +use crate::{device::Device, mem::device::DeviceMemory, sync::IrqSafeSpinlock}; + +use super::{DisplayDevice, DisplayDimensions}; + +struct Inner { + dimensions: DisplayDimensions, + base: usize, + stride: usize, +} + +pub struct FramebufferAccess { + dimensions: DisplayDimensions, + base: usize, + stride: usize, +} + +pub struct LinearFramebuffer { + inner: IrqSafeSpinlock, +} + +impl LinearFramebuffer { + pub unsafe fn from_physical_bits( + base: usize, + size: usize, + stride: usize, + width: u32, + height: u32, + ) -> Result { + // TODO this may get Dropped later + let mmio = unsafe { DeviceMemory::map("framebuffer", base, size) }?; + + let inner = Inner { + dimensions: DisplayDimensions { width, height }, + base: mmio.base(), + stride, + }; + + Ok(Self { + inner: IrqSafeSpinlock::new(inner), + }) + } + + // TODO doesn't actually lock + pub unsafe fn lock(&self) -> FramebufferAccess { + let inner = self.inner.lock(); + + FramebufferAccess { + dimensions: inner.dimensions, + base: inner.base, + stride: inner.stride, + } + } +} + +impl Device for LinearFramebuffer { + fn name(&self) -> &'static str { + "Linear Framebuffer" + } + + unsafe fn init(&self) -> Result<(), Error> { + Ok(()) + } +} + +impl DisplayDevice for LinearFramebuffer { + fn dimensions(&self) -> DisplayDimensions { + self.inner.lock().dimensions + } +} + +impl FramebufferAccess { + pub fn copy_rows(&mut self, src_row: u32, dst_row: u32, count: u32) { + use core::ffi::c_void; + extern "C" { + fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void; + } + + if src_row == dst_row { + return; + } + + let src_end_row = core::cmp::min(self.dimensions.height, src_row + count); + let dst_end_row = core::cmp::min(self.dimensions.height, dst_row + count); + + if dst_end_row <= dst_row || src_end_row <= dst_row { + return; + } + let count = core::cmp::min(src_end_row - src_row, dst_end_row - dst_row) as usize; + + let src_base_addr = self.base + self.stride * src_row as usize; + let dst_base_addr = self.base + self.stride * dst_row as usize; + + unsafe { + memmove( + dst_base_addr as *mut c_void, + src_base_addr as *mut c_void, + self.stride * count, + ); + } + } + + pub fn fill_rows(&mut self, start_row: u32, count: u32, value: u32) { + use core::ffi::c_void; + extern "C" { + fn memset(s: *mut c_void, c: u32, len: usize) -> *mut c_void; + } + + let end_row = core::cmp::min(self.dimensions.height, start_row + count); + if end_row <= start_row { + return; + } + + let count = (end_row - start_row) as usize; + let base_addr = self.base + self.stride * start_row as usize; + + unsafe { + memset(base_addr as *mut c_void, value, self.stride * count); + } + } +} + +impl Index for FramebufferAccess { + type Output = [u32]; + + fn index(&self, index: u32) -> &Self::Output { + assert!(index < self.dimensions.height); + let row_addr = self.base + self.stride * index as usize; + unsafe { core::slice::from_raw_parts(row_addr as *const u32, self.dimensions.width as _) } + } +} + +impl IndexMut for FramebufferAccess { + fn index_mut(&mut self, index: u32) -> &mut Self::Output { + assert!(index < self.dimensions.height); + let row_addr = self.base + self.stride * index as usize; + unsafe { core::slice::from_raw_parts_mut(row_addr as *mut u32, self.dimensions.width as _) } + } +} diff --git a/src/device/display/mod.rs b/src/device/display/mod.rs new file mode 100644 index 00000000..7f718c4f --- /dev/null +++ b/src/device/display/mod.rs @@ -0,0 +1,14 @@ +use super::Device; + +pub mod fb_console; +pub mod linear_fb; + +#[derive(Clone, Copy, Debug)] +pub struct DisplayDimensions { + pub width: u32, + pub height: u32, +} + +pub trait DisplayDevice: Device { + fn dimensions(&self) -> DisplayDimensions; +} diff --git a/src/device/mod.rs b/src/device/mod.rs index 456339b3..d00e96e1 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,11 +1,12 @@ //! Device management and interfaces use abi::error::Error; -pub mod interrupt; +// pub mod interrupt; +pub mod display; pub mod platform; -pub mod serial; -pub mod timer; -pub mod tty; +// pub mod serial; +// pub mod timer; +// pub mod tty; /// General device interface pub trait Device { diff --git a/src/device/platform.rs b/src/device/platform.rs index 674ef590..ee637776 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -2,15 +2,17 @@ use abi::error::Error; -use super::{interrupt::InterruptController, serial::SerialDevice, timer::TimestampSource}; +use crate::debug::DebugSink; + +// use super::{interrupt::InterruptController, serial::SerialDevice, timer::TimestampSource}; /// Platform interface for interacting with a general hardware set pub trait Platform { /// Interrupt number type for the platform - type IrqNumber; + // type IrqNumber; /// Address, to which the kernel is expected to be loaded for this platform - const KERNEL_PHYS_BASE: usize; + // const KERNEL_PHYS_BASE: usize; /// Initializes the platform devices to their usable state. /// @@ -18,34 +20,39 @@ pub trait Platform { /// /// Unsafe to call if the platform has already been initialized. unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error>; - /// Initializes the primary serial device to provide the debugging output as early as possible. - /// - /// # Safety - /// - /// Unsafe to call if the device has already been initialized. - unsafe fn init_primary_serial(&self); + + unsafe fn init_primary_debug_sink(&self); + + // /// Initializes the primary serial device to provide the debugging output as early as possible. + // /// + // /// # Safety + // /// + // /// Unsafe to call if the device has already been initialized. + // unsafe fn init_primary_serial(&self); /// Returns a display name for the platform fn name(&self) -> &'static str; - /// Returns a reference to the primary serial device. - /// - /// # Note - /// - /// May not be initialized at the moment of calling. - fn primary_serial(&self) -> Option<&dyn SerialDevice>; + fn primary_debug_sink(&self) -> Option<&dyn DebugSink>; - /// Returns a reference to the platform's interrupt controller. - /// - /// # Note - /// - /// May not be initialized at the moment of calling. - fn interrupt_controller(&self) -> &dyn InterruptController; + // /// Returns a reference to the primary serial device. + // /// + // /// # Note + // /// + // /// May not be initialized at the moment of calling. + // fn primary_serial(&self) -> Option<&dyn SerialDevice>; - /// Returns the platform's primary timestamp source. - /// - /// # Note - /// - /// May not be initialized at the moment of calling. - fn timestamp_source(&self) -> &dyn TimestampSource; + // /// Returns a reference to the platform's interrupt controller. + // /// + // /// # Note + // /// + // /// May not be initialized at the moment of calling. + // fn interrupt_controller(&self) -> &dyn InterruptController; + + // /// Returns the platform's primary timestamp source. + // /// + // /// # Note + // /// + // /// May not be initialized at the moment of calling. + // fn timestamp_source(&self) -> &dyn TimestampSource; } diff --git a/src/main.rs b/src/main.rs index 3d739565..bd99c5ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,7 +39,7 @@ fn panic_handler(_pi: &core::panic::PanicInfo) -> ! { loop {} } // -// pub mod device; +pub mod device; // pub mod fs; pub mod mem; // pub mod panic; From aec161bc4794b2e8039c11d06c40b779aa13623e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 30 Jul 2023 16:40:30 +0300 Subject: [PATCH 025/211] x86_64: physical memory init --- Cargo.toml | 5 +- src/arch/aarch64/mod.rs | 1 - src/arch/mod.rs | 2 + src/arch/x86_64/boot/mod.rs | 44 +++++++--- src/arch/x86_64/exception.rs | 102 +++++++++++++++++++++++ src/arch/x86_64/gdt.rs | 111 +++++++++++++++++++++++++ src/arch/x86_64/intrinsics.rs | 12 +++ src/arch/x86_64/mod.rs | 70 +++++++++++++++- src/arch/x86_64/table/fixed.rs | 16 +++- src/arch/x86_64/vectors.S | 139 ++++++++++++++++++++++++++++++++ src/device/display/linear_fb.rs | 9 ++- src/device/platform.rs | 27 +++---- src/main.rs | 10 ++- src/mem/mod.rs | 20 +++-- src/mem/phys/mod.rs | 15 +++- 15 files changed, 536 insertions(+), 47 deletions(-) create mode 100644 src/arch/x86_64/exception.rs create mode 100644 src/arch/x86_64/gdt.rs create mode 100644 src/arch/x86_64/intrinsics.rs create mode 100644 src/arch/x86_64/vectors.S diff --git a/Cargo.toml b/Cargo.toml index 06cfa7f2..97f54b7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,13 +12,14 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } # atomic_enum = "0.2.0" bitflags = "2.3.3" -# linked_list_allocator = "0.10.5" -# spinning_top = "0.2.5" +linked_list_allocator = "0.10.5" +spinning_top = "0.2.5" # static_assertions = "1.1.0" # tock-registers = "0.8.1" cfg-if = "1.0.0" bitmap-font = { version = "0.3.0" } embedded-graphics = "0.8.0" +git-version = "0.3.5" [dependencies.elf] version = "0.7.2" diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 2525daf1..a8ca6e3c 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -9,7 +9,6 @@ use plat_qemu::PLATFORM; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - absolute_address, arch::{ aarch64::{boot::CPU_INIT_FENCE, cpu::Cpu, devtree::FdtMemoryRegionIter, smp::CPU_COUNT}, Architecture, diff --git a/src/arch/mod.rs b/src/arch/mod.rs index c8231cd5..122a746d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -7,12 +7,14 @@ cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod aarch64; + pub use aarch64::intrinsics::absolute_address; pub use aarch64::plat_qemu::{QemuPlatform as PlatformImpl, PLATFORM}; pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; use abi::error::Error; use cfg_if::cfg_if; } else if #[cfg(target_arch = "x86_64")] { + #[macro_use] pub mod x86_64; pub use x86_64::{ diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 45446adf..a096326d 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,14 +1,23 @@ use core::arch::global_asm; +use git_version::git_version; use yboot_proto::{ - v1::FramebufferOption, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, - PROTOCOL_VERSION_1, + v1::{FramebufferOption, MemoryMap}, + LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, }; use crate::{ - arch::{Architecture, ArchitectureImpl}, + arch::{ + x86_64::{exception, gdt}, + Architecture, ArchitectureImpl, + }, debug, device::platform::Platform, + mem::{ + heap, + phys::{self, PageUsage}, + ConvertAddress, + }, }; use super::ARCHITECTURE; @@ -35,6 +44,8 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { }, kernel_virt_offset: KERNEL_VIRT_OFFSET as _, + memory_map: MemoryMap { address: 0, len: 0 }, + opt_framebuffer: FramebufferOption { req_width: 640, req_height: 480, @@ -59,16 +70,29 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { ARCHITECTURE.init_primary_debug_sink(); debug::init(); + infoln!("Yggdrasil kernel git {} starting", git_version!()); - let mut i = 0; - loop { - debugln!("{} {}", env!("PROFILE"), i); - i += 1; + if YBOOT_DATA.memory_map.address > 0xFFFFFFFF { + errorln!("Unhandled case: memory map is above 4GiB"); + loop { + ArchitectureImpl::wait_for_interrupt(); + } } - // loop { - // ArchitectureImpl::wait_for_interrupt(); - // } + gdt::init(); + exception::init_exceptions(0); + + // Setup physical memory allocation + ArchitectureImpl::init_physical_memory(&YBOOT_DATA.memory_map); + + // Allocate memory for the kernel heap + let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) + .expect("Could not allocate a block for heap"); + heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + + loop { + ArchitectureImpl::wait_for_interrupt(); + } } global_asm!( diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs new file mode 100644 index 00000000..6d562456 --- /dev/null +++ b/src/arch/x86_64/exception.rs @@ -0,0 +1,102 @@ +use core::{arch::global_asm, mem::size_of_val}; + +use crate::arch::{Architecture, ArchitectureImpl}; + +#[derive(Debug)] +#[repr(C)] +pub struct ExceptionFrame { + rax: usize, + rcx: usize, + rdx: usize, + rbx: usize, + rsi: usize, + rdi: usize, + rbp: usize, + r8: usize, + r9: usize, + r10: usize, + r11: usize, + r12: usize, + r13: usize, + r14: usize, + r15: usize, +} + +#[derive(Clone, Copy)] +#[repr(packed)] +struct Entry { + base_lo: u16, + selector: u16, + __res0: u8, + flags: u8, + base_hi: u16, + base_ex: u32, + __res1: u32, +} + +#[repr(packed)] +struct Pointer { + limit: u16, + offset: usize, +} + +const SIZE: usize = 256; + +impl Entry { + const PRESENT: u8 = 1 << 7; + const INT32: u8 = 0xE; + + const NULL: Self = Self { + base_lo: 0, + base_hi: 0, + base_ex: 0, + selector: 0, + flags: 0, + __res0: 0, + __res1: 0, + }; + + const fn new(base: usize, selector: u16, flags: u8) -> Self { + Self { + base_lo: (base & 0xFFFF) as u16, + base_hi: ((base >> 16) & 0xFFFF) as u16, + base_ex: (base >> 32) as u32, + selector, + flags, + __res0: 0, + __res1: 0, + } + } +} + +static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; + +extern "C" fn __x86_64_exception_handler() { + errorln!("An exception occurrred"); + loop { + ArchitectureImpl::wait_for_interrupt(); + } +} + +pub unsafe fn init_exceptions(_cpu_index: usize) { + extern "C" { + static __x86_64_exception_vectors: [usize; 32]; + } + + for (i, &entry) in __x86_64_exception_vectors.iter().enumerate() { + IDT[i] = Entry::new(entry, 0x08, Entry::PRESENT | Entry::INT32); + } + + let idtr = Pointer { + limit: size_of_val(&IDT) as u16 - 1, + offset: &IDT as *const _ as usize, + }; + + core::arch::asm!("wbinvd; lidt ({0})", in(reg) &idtr, options(att_syntax)); +} + +global_asm!( + include_str!("vectors.S"), + exception_handler = sym __x86_64_exception_handler, + options(att_syntax) +); diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs new file mode 100644 index 00000000..9a817c80 --- /dev/null +++ b/src/arch/x86_64/gdt.rs @@ -0,0 +1,111 @@ +// TODO TSS + +use core::mem::size_of_val; + +#[repr(packed)] +struct Entry { + limit_lo: u16, + base_lo: u16, + base_mi: u8, + access: u8, + flags: u8, + base_hi: u8, +} + +#[repr(packed)] +struct Pointer { + limit: u16, + offset: usize, +} + +impl Entry { + const FLAG_LONG: u8 = 1 << 5; + const ACC_PRESENT: u8 = 1 << 7; + const ACC_SYSTEM: u8 = 1 << 4; + const ACC_EXECUTE: u8 = 1 << 3; + const ACC_WRITE: u8 = 1 << 1; + const ACC_RING3: u8 = 3 << 5; + const ACC_ACCESS: u8 = 1 << 0; + + const NULL: Self = Self { + base_lo: 0, + base_mi: 0, + base_hi: 0, + access: 0, + flags: 0, + limit_lo: 0, + }; + + const fn new(base: u32, limit: u32, flags: u8, access: u8) -> Self { + Self { + base_lo: (base & 0xFFFF) as u16, + base_mi: ((base >> 16) & 0xFF) as u8, + base_hi: ((base >> 24) & 0xFF) as u8, + access, + flags: (flags & 0xF0) | (((limit >> 16) & 0xF) as u8), + limit_lo: (limit & 0xFFFF) as u16, + } + } +} + +const SIZE: usize = 3; + +// TODO per-CPU +static mut GDT: [Entry; SIZE] = [ + // 0x00, NULL + Entry::NULL, + // 0x08, Ring0 CS64 + Entry::new( + 0, + 0, + Entry::FLAG_LONG, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_EXECUTE, + ), + // 0x10, Ring0 DS64 + Entry::new( + 0, + 0, + 0, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_WRITE, + ), +]; + +pub unsafe fn init() { + let gdtr = Pointer { + limit: size_of_val(&GDT) as u16 - 1, + offset: &GDT as *const _ as usize, + }; + + core::arch::asm!( + r#" + wbinvd + lgdt ({0}) + + // Have to use iretq here + mov %rsp, %rcx + leaq 1f(%rip), %rax + + // SS:RSP + pushq $0x10 + pushq %rcx + + // RFLAGS + pushfq + + // CS:RIP + pushq $0x08 + pushq %rax + iretq + 1: + mov $0x10, %ax + mov %ax, %ds + mov %ax, %es + mov %ax, %fs + mov %ax, %gs + mov %ax, %ss + "#, + in(reg) &gdtr, + out("rax") _, + options(att_syntax) + ); +} diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs new file mode 100644 index 00000000..1deca692 --- /dev/null +++ b/src/arch/x86_64/intrinsics.rs @@ -0,0 +1,12 @@ +/// Returns an absolute address to the given symbol + +#[macro_export] +macro_rules! absolute_address { + ($sym:expr) => {{ + let mut _x: usize; + unsafe { + core::arch::asm!("movabsq ${1}, {0}", out(reg) _x, sym $sym, options(att_syntax)); + } + _x + }}; +} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 922cd604..ded7fdb9 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,5 +1,5 @@ use abi::error::Error; -use yboot_proto::v1::FramebufferOption; +use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap}; use crate::{ arch::x86_64::table::{init_fixed_tables, KERNEL_TABLES}, @@ -8,14 +8,59 @@ use crate::{ display::{fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, platform::Platform, }, + mem::phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, util::OneTimeInit, }; use super::Architecture; +#[macro_use] +pub mod intrinsics; + pub mod boot; +pub mod exception; +pub mod gdt; pub mod table; +pub trait AbstractAvailableRegion { + fn start_address(&self) -> usize; + fn page_count(&self) -> usize; +} + +pub trait AbstractMemoryMap<'a>: 'a { + type AvailableRegion: AbstractAvailableRegion; + type Iter: Iterator + Clone; + + fn reserved_range(&self) -> PhysicalMemoryRegion; + fn iter(&self) -> Self::Iter; +} + +impl AbstractAvailableRegion for T { + fn start_address(&self) -> usize { + ::start_address(self) + } + + fn page_count(&self) -> usize { + ::page_count(self) + } +} + +impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T { + type AvailableRegion = T::Entry; + type Iter = T::Iter; + + fn reserved_range(&self) -> PhysicalMemoryRegion { + PhysicalMemoryRegion { + base: self.data_physical_base(), + size: (self.data_size() + 0xFFF) & !0xFFF, + } + } + + fn iter(&self) -> Self::Iter { + ::iter_with_offset(self, X86_64::KERNEL_VIRT_OFFSET) + } +} + pub struct X86_64 { yboot_framebuffer: OneTimeInit, } @@ -61,6 +106,8 @@ impl Architecture for X86_64 { } impl Platform for X86_64 { + const KERNEL_PHYS_BASE: usize = 0x400000; + unsafe fn init(&'static self, _is_bsp: bool) -> Result<(), Error> { Ok(()) } @@ -100,6 +147,27 @@ impl Platform for X86_64 { } } +impl X86_64 { + unsafe fn init_physical_memory<'a, M: AbstractMemoryMap<'a>>(memory_map: &M) { + // Reserve the lower 8MiB of memory + reserve_region( + "lower-memory", + PhysicalMemoryRegion { + base: 0, + size: 8 << 21, + }, + ); + // Reserve memory map + reserve_region("memory-map", memory_map.reserved_range()); + + phys::init_from_iter(memory_map.iter().map(|r| PhysicalMemoryRegion { + base: r.start_address(), + size: r.page_count() * 0x1000, + })) + .expect("Failed to initialize the physical memory manager"); + } +} + pub static ARCHITECTURE: X86_64 = X86_64 { yboot_framebuffer: OneTimeInit::new(), }; diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs index cdae45f4..8f3fa41b 100644 --- a/src/arch/x86_64/table/fixed.rs +++ b/src/arch/x86_64/table/fixed.rs @@ -24,6 +24,7 @@ pub struct FixedTables { // 512 entries device_l3: PageTable, + device_l2i: usize, device_l3i: usize, } @@ -43,6 +44,7 @@ impl FixedTables { device_l2: PageTable::zeroed(), device_l3: PageTable::zeroed(), + device_l2i: 1, device_l3i: 0, } } @@ -51,8 +53,20 @@ impl FixedTables { if count > 512 * 512 { panic!("Unsupported device memory mapping size"); } else if count > 512 { + let count = (count + 511) / 512; // 2MiB mappings - todo!(); + if self.device_l2i + count > 512 { + return Err(Error::OutOfMemory); + } + + let virt = DEVICE_VIRT_OFFSET + (self.device_l2i << 21); + for i in 0..count { + self.device_l2[self.device_l2i + i] = + PageEntry::block(phys, PageAttributes::WRITABLE); + } + self.device_l2i += count; + + Ok(virt) } else { // 4KiB mappings if self.device_l3i + count > 512 { diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S new file mode 100644 index 00000000..c816655f --- /dev/null +++ b/src/arch/x86_64/vectors.S @@ -0,0 +1,139 @@ +.macro ISR_NERR, n +__x86_64_exc_\n: + cli + pushq $0 + pushq $\n + jmp __x86_64_exc_common +.endm + +.macro ISR_YERR, n +__x86_64_exc_\n: + cli + pushq $\n + jmp __x86_64_exc_common +.endm + +// 16 general-purpose registers +.set PT_REGS_SIZE, 15 * 8 + +.macro EXC_SAVE_STATE + sub $PT_REGS_SIZE, %rsp + mov %rax, 0(%rsp) + mov %rcx, 8(%rsp) + mov %rdx, 16(%rsp) + mov %rbx, 32(%rsp) + mov %rsi, 36(%rsp) + mov %rdi, 40(%rsp) + mov %rbp, 48(%rsp) + + mov %r8, 52(%rsp) + mov %r9, 56(%rsp) + mov %r10, 60(%rsp) + mov %r11, 64(%rsp) + mov %r12, 68(%rsp) + mov %r13, 72(%rsp) + mov %r14, 76(%rsp) + mov %r15, 80(%rsp) +.endm + +.macro EXC_RESTORE_STATE + mov 0(%rsp), %rax + mov 8(%rsp), %rcx + mov 16(%rsp), %rdx + mov 32(%rsp), %rbx + mov 36(%rsp), %rsi + mov 40(%rsp), %rdi + mov 48(%rsp), %rbp + + mov 52(%rsp), %r8 + mov 56(%rsp), %r9 + mov 60(%rsp), %r10 + mov 64(%rsp), %r11 + mov 68(%rsp), %r12 + mov 72(%rsp), %r13 + mov 76(%rsp), %r14 + mov 80(%rsp), %r15 +.endm + +.global __x86_64_exception_vectors +.section .text +__x86_64_exc_common: + EXC_SAVE_STATE + mov %rsp, %rdi + call {exception_handler} + + // TODO +1: + cli + hlt + jmp 1b + +ISR_NERR 0 +ISR_NERR 1 +ISR_NERR 2 +ISR_NERR 3 +ISR_NERR 4 +ISR_NERR 5 +ISR_NERR 6 +ISR_NERR 7 +ISR_YERR 8 +ISR_NERR 9 +ISR_YERR 10 +ISR_YERR 11 +ISR_YERR 12 +ISR_YERR 13 +ISR_YERR 14 +ISR_NERR 15 +ISR_NERR 16 +ISR_YERR 17 +ISR_NERR 18 +ISR_NERR 19 +ISR_NERR 20 +ISR_NERR 21 +ISR_NERR 22 +ISR_NERR 23 +ISR_NERR 24 +ISR_NERR 25 +ISR_NERR 26 +ISR_NERR 27 +ISR_NERR 28 +ISR_NERR 29 +ISR_YERR 30 +ISR_NERR 31 + +.section .rodata +.global __x86_64_exception_vectors +.p2align 4 +__x86_64_exception_vectors: + .quad __x86_64_exc_0 + .quad __x86_64_exc_1 + .quad __x86_64_exc_2 + .quad __x86_64_exc_3 + .quad __x86_64_exc_4 + .quad __x86_64_exc_5 + .quad __x86_64_exc_6 + .quad __x86_64_exc_7 + .quad __x86_64_exc_8 + .quad __x86_64_exc_9 + .quad __x86_64_exc_10 + .quad __x86_64_exc_11 + .quad __x86_64_exc_12 + .quad __x86_64_exc_13 + .quad __x86_64_exc_14 + .quad __x86_64_exc_15 + .quad __x86_64_exc_16 + .quad __x86_64_exc_17 + .quad __x86_64_exc_18 + .quad __x86_64_exc_19 + .quad __x86_64_exc_20 + .quad __x86_64_exc_21 + .quad __x86_64_exc_22 + .quad __x86_64_exc_23 + .quad __x86_64_exc_24 + .quad __x86_64_exc_25 + .quad __x86_64_exc_26 + .quad __x86_64_exc_27 + .quad __x86_64_exc_28 + .quad __x86_64_exc_29 + .quad __x86_64_exc_30 + .quad __x86_64_exc_31 diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 1abd302e..0aa86ecf 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -39,9 +39,14 @@ impl LinearFramebuffer { stride, }; - Ok(Self { + let res = Self { inner: IrqSafeSpinlock::new(inner), - }) + }; + + // Clear the screen + res.lock().fill_rows(0, height, 0); + + Ok(res) } // TODO doesn't actually lock diff --git a/src/device/platform.rs b/src/device/platform.rs index ee637776..762adc26 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -12,7 +12,7 @@ pub trait Platform { // type IrqNumber; /// Address, to which the kernel is expected to be loaded for this platform - // const KERNEL_PHYS_BASE: usize; + const KERNEL_PHYS_BASE: usize; /// Initializes the platform devices to their usable state. /// @@ -21,27 +21,24 @@ pub trait Platform { /// Unsafe to call if the platform has already been initialized. unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error>; + // TODO use init_debug instead and let the platform code handle it + /// Initializes the primary output device to provide the debugging output as early as possible. + /// + /// # Safety + /// + /// Unsafe to call if the device has already been initialized. unsafe fn init_primary_debug_sink(&self); - // /// Initializes the primary serial device to provide the debugging output as early as possible. - // /// - // /// # Safety - // /// - // /// Unsafe to call if the device has already been initialized. - // unsafe fn init_primary_serial(&self); - /// Returns a display name for the platform fn name(&self) -> &'static str; + /// Returns a reference to the primary debug output device. + /// + /// # Note + /// + /// May not be initialized at the moment of calling. fn primary_debug_sink(&self) -> Option<&dyn DebugSink>; - // /// Returns a reference to the primary serial device. - // /// - // /// # Note - // /// - // /// May not be initialized at the moment of calling. - // fn primary_serial(&self) -> Option<&dyn SerialDevice>; - // /// Returns a reference to the platform's interrupt controller. // /// // /// # Note diff --git a/src/main.rs b/src/main.rs index bd99c5ad..32cb0823 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,7 +27,7 @@ extern crate yggdrasil_abi as abi; // use task::process::Process; // use vfs::{Filesystem, IoContext, VnodeRef}; // -// extern crate alloc; +extern crate alloc; // #[macro_use] pub mod debug; @@ -36,7 +36,13 @@ pub mod arch; #[panic_handler] fn panic_handler(_pi: &core::panic::PanicInfo) -> ! { - loop {} + use arch::{Architecture, ArchitectureImpl}; + + fatalln!("KERNEL PANIC"); + + loop { + ArchitectureImpl::wait_for_interrupt(); + } } // pub mod device; diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 5c8f2c7a..d5cf0ad7 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,27 +1,25 @@ //! Memory management utilities and types -use core::{alloc::Layout, mem::size_of}; +// use core::{alloc::Layout, mem::size_of}; // use abi::error::Error; // -// use crate::{ -// arch::{Architecture, ArchitectureImpl, PlatformImpl}, -// device::platform::Platform, -// }; +use crate::{ + arch::{Architecture, ArchitectureImpl, PlatformImpl}, + device::platform::Platform, +}; // // use self::table::AddressSpace; pub mod device; -// pub mod heap; -// pub mod phys; +pub mod heap; +pub mod phys; pub mod table; /// Kernel's physical load address -// pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; +pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; /// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + /// [KERNEL_VIRT_OFFSET]) -// pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; -// TODO fix this -pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; +pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; /// Interface for converting between address spaces. /// diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index a19f435f..c79e681d 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -4,7 +4,6 @@ use core::{iter::StepBy, mem::size_of, ops::Range}; use abi::error::Error; use crate::{ - absolute_address, debug::LogLevel, mem::{ phys::reserved::{is_reserved, reserve_region}, @@ -16,6 +15,10 @@ use crate::{ use self::manager::PhysicalMemoryManager; +// Enumerating lots of pages is slow and I'm too lazy right now to write a better algorithm, so +// capping the page count helps +const PHYS_MEMORY_PAGE_CAP: usize = 65536; + pub mod manager; pub mod reserved; @@ -229,6 +232,10 @@ pub unsafe fn init_from_iter + Clone>( let mut page_count = 0; for region in it { + if page_count >= PHYS_MEMORY_PAGE_CAP { + break; + } + for page in region.pages() { if is_reserved(page) { continue; @@ -236,10 +243,14 @@ pub unsafe fn init_from_iter + Clone>( manager.add_available_page(page); page_count += 1; + + if page_count >= PHYS_MEMORY_PAGE_CAP { + break; + } } } - infoln!("{} available pages", page_count); + infoln!("{} available pages ({}KiB)", page_count, page_count * 4); PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager)); Ok(()) From 3a795f02296d12d769d549f73162cac2dfcfa67a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 1 Aug 2023 11:09:56 +0300 Subject: [PATCH 026/211] x86-64: implement Local+I/O APICs and PS/2 kbd --- Cargo.toml | 4 +- src/arch/x86_64/apic/ioapic.rs | 246 +++++++++++++++++++++++++++++ src/arch/x86_64/apic/local.rs | 118 ++++++++++++++ src/arch/x86_64/apic/mod.rs | 83 ++++++++++ src/arch/x86_64/boot/mod.rs | 21 ++- src/arch/x86_64/cpu.rs | 66 ++++++++ src/arch/x86_64/exception.rs | 48 +++++- src/arch/x86_64/gdt.rs | 12 +- src/arch/x86_64/intrinsics.rs | 39 ++++- src/arch/x86_64/mod.rs | 127 ++++++++++++++- src/arch/x86_64/peripherals/mod.rs | 3 + src/arch/x86_64/peripherals/ps2.rs | 102 ++++++++++++ src/arch/x86_64/registers/mod.rs | 102 ++++++++++++ src/arch/x86_64/table/fixed.rs | 10 ++ src/arch/x86_64/table/mod.rs | 33 ++-- src/arch/x86_64/vectors.S | 63 ++++++++ src/debug.rs | 58 +++---- src/device/display/fb_console.rs | 6 +- src/device/display/linear_fb.rs | 16 ++ src/device/display/mod.rs | 7 + src/device/interrupt.rs | 55 ++++--- src/device/mod.rs | 2 +- src/device/platform.rs | 17 +- src/main.rs | 15 +- src/mem/device.rs | 1 + src/panic.rs | 15 +- 26 files changed, 1162 insertions(+), 107 deletions(-) create mode 100644 src/arch/x86_64/apic/ioapic.rs create mode 100644 src/arch/x86_64/apic/local.rs create mode 100644 src/arch/x86_64/apic/mod.rs create mode 100644 src/arch/x86_64/cpu.rs create mode 100644 src/arch/x86_64/peripherals/mod.rs create mode 100644 src/arch/x86_64/peripherals/ps2.rs create mode 100644 src/arch/x86_64/registers/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 97f54b7f..c67503f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,12 @@ bitflags = "2.3.3" linked_list_allocator = "0.10.5" spinning_top = "0.2.5" # static_assertions = "1.1.0" -# tock-registers = "0.8.1" +tock-registers = "0.8.1" cfg-if = "1.0.0" bitmap-font = { version = "0.3.0" } embedded-graphics = "0.8.0" git-version = "0.3.5" +acpi = "4.1.1" [dependencies.elf] version = "0.7.2" @@ -33,3 +34,4 @@ aarch64-cpu = "9.3.1" [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } +aml = "0.16.4" diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs new file mode 100644 index 00000000..181c7eff --- /dev/null +++ b/src/arch/x86_64/apic/ioapic.rs @@ -0,0 +1,246 @@ +//! x86-64 I/O APIC driver implementation +use abi::error::Error; +use acpi::platform::interrupt::{Apic as AcpiApic, Polarity, TriggerMode}; + +use crate::{ + arch::x86_64::apic::local::BSP_APIC_ID, + device::{ + interrupt::{ExternalInterruptController, InterruptSource}, + Device, + }, + mem::ConvertAddress, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +use super::{IrqNumber, Table, APIC_EXTERNAL_OFFSET}; + +// IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries +const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET; + +const REG_IOAPIC_VERSION: u32 = 0x01; +const REG_REDIRECTION_BASE: u32 = 0x10; + +const ENTRY_LOW_MASK: u32 = 1 << 16; +// const ENTRY_LOW_TRIGGER_LEVEL: u32 = 1 << 15; +// const ENTRY_LOW_POLARITY_LOW: u32 = 1 << 13; +const ENTRY_LOW_DESTINATION_LOGICAL: u32 = 1 << 11; + +const ENTRY_HIGH_APIC_ID_SHIFT: u32 = 24; + +#[derive(Clone, Copy, PartialEq)] +enum ExternalTriggerMode { + Edge, + Level, +} + +#[allow(dead_code)] +#[derive(Clone, Copy)] +struct IsaRedirection { + gsi_index: u32, + polarity: bool, + trigger: ExternalTriggerMode, +} + +struct Regs { + base: usize, +} + +struct Inner { + regs: Regs, + max_gsi: u32, +} + +/// I/O APIC interface. Provides a way to route and control how interrupts from external devices +/// are handled. +pub struct IoApic { + inner: OneTimeInit>, + isa_redirections: OneTimeInit<[Option; 16]>, + + table: IrqSafeSpinlock, +} + +impl Regs { + #[inline] + fn read(&self, reg: u32) -> u32 { + let ptr = self.base as *mut u32; + + unsafe { + ptr.write_volatile(reg & 0xFF); + ptr.add(4).read_volatile() + } + } + + #[inline] + fn write(&self, reg: u32, value: u32) { + let ptr = self.base as *mut u32; + + unsafe { + ptr.write_volatile(reg & 0xFF); + ptr.add(4).write_volatile(value); + } + } +} + +impl Inner { + fn map_gsi(&mut self, gsi: u32, vector: u32, apic_id: u32) -> Result<(), Error> { + assert!(gsi < self.max_gsi); + assert!(vector < 0x100); + + debugln!("map_irq gsi{}, vec{}, apic{}", gsi, vector, apic_id); + + let mut low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2); + let mut high = self.regs.read(REG_REDIRECTION_BASE + gsi * 2 + 1); + + // Vector + low &= !0xFF; + low |= vector; + // Destination - physical + low &= !ENTRY_LOW_DESTINATION_LOGICAL; + + // Destination APIC ID + high &= !(0xFF << ENTRY_HIGH_APIC_ID_SHIFT); + high |= apic_id << ENTRY_HIGH_APIC_ID_SHIFT; + + self.regs.write(REG_REDIRECTION_BASE + gsi * 2, low); + self.regs.write(REG_REDIRECTION_BASE + gsi * 2 + 1, high); + + Ok(()) + } + + fn set_gsi_enabled(&mut self, gsi: u32, enabled: bool) { + assert!(gsi < self.max_gsi); + + let low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2); + if enabled { + debugln!("Unmask GSI #{}", gsi); + self.regs + .write(REG_REDIRECTION_BASE + gsi * 2, low & !ENTRY_LOW_MASK) + } else { + self.regs + .write(REG_REDIRECTION_BASE + gsi * 2, low | ENTRY_LOW_MASK); + } + } +} + +impl Device for IoApic { + unsafe fn init(&self) -> Result<(), Error> { + Ok(()) + } + + fn name(&self) -> &'static str { + "I/O APIC" + } +} + +impl ExternalInterruptController for IoApic { + type IrqNumber = IrqNumber; + + fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { + let mut inner = self.inner.get().lock(); + let gsi = self.translate_irq(irq); + inner.set_gsi_enabled(gsi, true); + Ok(()) + } + + fn register_handler( + &self, + irq: Self::IrqNumber, + handler: &'static (dyn InterruptSource + Sync), + ) -> Result<(), Error> { + let mut inner = self.inner.get().lock(); + let mut table = self.table.lock(); + + // Allocate a vector + let table_vector = table.least_loaded(); + let gsi_target_vector = (table_vector as u32) + IO_APIC_VECTOR_OFFSET; + let bsp_apic = *BSP_APIC_ID.get(); + + debugln!("Binding {:?} to {}:{}", irq, bsp_apic, table_vector); + table.add_handler(table_vector, handler)?; + + let gsi = self.translate_irq(irq); + inner.map_gsi(gsi, gsi_target_vector, *BSP_APIC_ID.get())?; + + Ok(()) + } +} + +impl IoApic { + /// Constructs an uninitialized I/O APIC interface + pub const fn new() -> Self { + Self { + inner: OneTimeInit::new(), + isa_redirections: OneTimeInit::new(), + table: IrqSafeSpinlock::new(Table::new()), + } + } + + /// Handles an interrupt with given GSI number + pub fn handle_irq(&self, gsi: u32) { + let table = self.table.lock(); + + if let Some(handler) = table.vectors[gsi as usize] { + drop(table); + handler.handle_irq().expect("IRQ handler failed"); + } else { + panic!("No handler set for GSI #{}", gsi); + } + } + + /// Initializes the I/O APIC interface using information provided by ACPI + pub fn init_from_acpi(&self, info: &AcpiApic) -> Result<(), Error> { + let io_apic = info.io_apics.first().unwrap(); + + infoln!("I/O APIC at {:#x}", io_apic.address); + + let mut isa_redirections = [None; 16]; + + for redir in info.interrupt_source_overrides.iter() { + let index = redir.isa_source as usize; + let polarity = match redir.polarity { + Polarity::ActiveLow => false, + // TODO this may not be correct + Polarity::ActiveHigh | Polarity::SameAsBus => true, + }; + let trigger = match redir.trigger_mode { + TriggerMode::Edge | TriggerMode::SameAsBus => ExternalTriggerMode::Edge, + TriggerMode::Level => ExternalTriggerMode::Level, + }; + + debugln!( + "ISA IRQ #{} -> GSI #{}", + index, + redir.global_system_interrupt + ); + isa_redirections[index].replace(IsaRedirection { + gsi_index: redir.global_system_interrupt, + polarity, + trigger, + }); + } + + let regs = Regs { + base: unsafe { (io_apic.address as usize).virtualize() }, + }; + + let max_gsi = (regs.read(REG_IOAPIC_VERSION) >> 16) & 0xFF; + + let inner = Inner { regs, max_gsi }; + + self.inner.init(IrqSafeSpinlock::new(inner)); + self.isa_redirections.init(isa_redirections); + + Ok(()) + } + + fn translate_irq(&self, irq: IrqNumber) -> u32 { + let redir = self.isa_redirections.get(); + match irq { + IrqNumber::Isa(isa) => redir[isa as usize] + .map(|t| t.gsi_index) + .unwrap_or(isa as u32), + IrqNumber::Gsi(gsi) => gsi as _, + } + } +} diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs new file mode 100644 index 00000000..07c6f07f --- /dev/null +++ b/src/arch/x86_64/apic/local.rs @@ -0,0 +1,118 @@ +//! x86-64 Local APIC driver implementation +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite, WriteOnly}, +}; + +use crate::{arch::x86_64::registers::MSR_IA32_APIC_BASE, mem::ConvertAddress, util::OneTimeInit}; + +use super::APIC_TIMER_VECTOR; + +const TIMER_INTERVAL: u32 = 150000; + +/// When initialized, contains the Local APIC ID of the bootstrap processor +pub static BSP_APIC_ID: OneTimeInit = OneTimeInit::new(); + +register_bitfields! { + u32, + Id [ + ApicId OFFSET(24) NUMBITS(8) [] + ], + SpuriousVector [ + SoftwareEnable OFFSET(8) NUMBITS(1), + ], + TimerLocalVectorEntry [ + Vector OFFSET(0) NUMBITS(8) [], + Mask OFFSET(16) NUMBITS(1) [ + Masked = 1, + Unmasked = 0 + ], + TimerMode OFFSET(17) NUMBITS(1) [ + Periodic = 1, + OneShot = 0 + ] + ] +} + +register_structs! { + #[allow(non_snake_case, missing_docs)] + Regs { + (0x00 => _0), + (0x20 => Id: ReadOnly), + (0x24 => _1), + (0xB0 => EndOfInterrupt: WriteOnly), + (0xB4 => _2), + (0xF0 => SpuriousVector: ReadWrite), + (0xF4 => _3), + (0x320 => TimerLocalVectorEntry: ReadWrite), + (0x324 => _4), + (0x380 => TimerInitCount: ReadWrite), + (0x384 => _5), + (0x390 => TimerCurrentCount: ReadOnly), + (0x394 => _6), + (0x3E0 => TimerDivideConfig: ReadWrite), + (0x3E4 => _7), + (0x530 => @END), + } +} + +/// Per-processor local APIC interface +pub struct LocalApic { + regs: &'static Regs, +} + +impl LocalApic { + /// Constructs a new instance of Local APIC. + /// + /// # Safety + /// + /// Only meant to be called once per processor during their init. + pub unsafe fn new() -> Self { + let base = unsafe { Self::base().virtualize() }; + let regs = unsafe { &*(base as *const Regs) }; + + let id = regs.Id.read(Id::ApicId); + + if Self::is_bootstrap_cpu() { + BSP_APIC_ID.init(id); + } + + Self::enable(); + + // Enable timer + regs.TimerInitCount.set(TIMER_INTERVAL); + regs.TimerDivideConfig.set(0x3); + + regs.TimerLocalVectorEntry.write( + TimerLocalVectorEntry::Vector.val(APIC_TIMER_VECTOR + 32) + + TimerLocalVectorEntry::Mask::Unmasked + + TimerLocalVectorEntry::TimerMode::Periodic, + ); + + regs.SpuriousVector + .modify(SpuriousVector::SoftwareEnable::SET); + + Self { regs } + } + + /// Signals local APIC that we've handled the IRQ + pub fn clear_interrupt(&self) { + self.regs.EndOfInterrupt.set(0); + } + + #[inline] + fn base() -> usize { + MSR_IA32_APIC_BASE.read_base() as usize + } + + #[inline] + fn is_bootstrap_cpu() -> bool { + MSR_IA32_APIC_BASE.read(MSR_IA32_APIC_BASE::BootstrapCpuCore) != 0 + } + + #[inline] + fn enable() { + MSR_IA32_APIC_BASE.modify(MSR_IA32_APIC_BASE::ApicEnable::SET); + } +} diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs new file mode 100644 index 00000000..83610997 --- /dev/null +++ b/src/arch/x86_64/apic/mod.rs @@ -0,0 +1,83 @@ +//! x86-64 APIC interface (Local + I/O) + +use abi::error::Error; + +use crate::{ + arch::{x86_64::cpu::Cpu, PLATFORM}, + device::interrupt::IrqHandler, +}; + +use super::exception::{self, ExceptionFrame}; + +pub mod ioapic; +pub mod local; + +// I/O APIC 0..MAX_EXTERNAL_VECTORS range is mapped to BSP Local APIC 2.. + +/// Fixed IRQ vector for Local APIC timer +pub const APIC_TIMER_VECTOR: u32 = 0; +/// Start of the I/O APIC IRQ range +pub const APIC_EXTERNAL_OFFSET: u32 = 2; +/// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC +pub const MAX_EXTERNAL_VECTORS: u32 = 16; + +/// x86-64 interrupt number wrapper +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum IrqNumber { + /// Legacy (ISA) interrupt. Can have value in range 0..16. + Isa(u8), + /// Global system interrupt. Means an external interrupt for I/O APIC. + Gsi(u8), +} + +struct Table { + vectors: [Option; MAX_EXTERNAL_VECTORS as usize], +} + +impl Table { + pub const fn new() -> Self { + Self { + vectors: [None; MAX_EXTERNAL_VECTORS as usize], + } + } + + pub fn add_handler(&mut self, vector: usize, handler: IrqHandler) -> Result<(), Error> { + let old = self.vectors[vector].replace(handler); + assert!(old.is_none()); + Ok(()) + } + + pub fn least_loaded(&self) -> usize { + self.vectors.iter().position(|p| p.is_none()).unwrap() + } +} + +/// Fills the IDT with interrupt vectors for this APIC +pub fn setup_vectors(idt: &mut [exception::Entry]) { + extern "C" { + static __x86_64_apic_vectors: [usize; 16]; + } + + for (i, &entry) in unsafe { __x86_64_apic_vectors.iter() }.enumerate() { + idt[i] = exception::Entry::new( + entry, + 0x08, + exception::Entry::PRESENT | exception::Entry::INT32, + ); + } +} + +/// Main handler for APIC interrupts +pub extern "C" fn __x86_64_apic_irq_handler(vector: u32, _frame: *mut ExceptionFrame) { + let cpu = Cpu::local(); + cpu.local_apic().clear_interrupt(); + + if vector == APIC_TIMER_VECTOR { + // TODO + } else if (APIC_EXTERNAL_OFFSET..APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS).contains(&vector) + { + PLATFORM.ioapic.handle_irq(vector - APIC_EXTERNAL_OFFSET); + } else { + panic!("Got an interrupt on undefined vector: {}", vector); + } +} diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index a096326d..f87c4f76 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,3 +1,4 @@ +//! x86-64 boot and entry functions use core::arch::global_asm; use git_version::git_version; @@ -8,21 +9,20 @@ use yboot_proto::{ use crate::{ arch::{ - x86_64::{exception, gdt}, - Architecture, ArchitectureImpl, + x86_64::{apic::local::LocalApic, cpu::Cpu, exception, gdt}, + Architecture, ArchitectureImpl, PLATFORM, }, debug, device::platform::Platform, mem::{ heap, phys::{self, PageUsage}, - ConvertAddress, + ConvertAddress, KERNEL_VIRT_OFFSET, }, }; use super::ARCHITECTURE; -pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; const BOOT_STACK_SIZE: usize = 65536; #[repr(C, align(0x20))] @@ -46,6 +46,8 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { memory_map: MemoryMap { address: 0, len: 0 }, + rsdp_address: 0, + opt_framebuffer: FramebufferOption { req_width: 640, req_height: 480, @@ -79,6 +81,11 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { } } + if YBOOT_DATA.rsdp_address != 0 { + infoln!("ACPI RSDP at {:#x}", YBOOT_DATA.rsdp_address); + PLATFORM.init_rsdp(YBOOT_DATA.rsdp_address as _); + } + gdt::init(); exception::init_exceptions(0); @@ -90,6 +97,12 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { .expect("Could not allocate a block for heap"); heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + // Also initializes local APIC + Cpu::init_local(LocalApic::new(), 0); + + PLATFORM.init(true).unwrap(); + + ArchitectureImpl::set_interrupt_mask(false); loop { ArchitectureImpl::wait_for_interrupt(); } diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs new file mode 100644 index 00000000..f7d9382c --- /dev/null +++ b/src/arch/x86_64/cpu.rs @@ -0,0 +1,66 @@ +//! Per-CPU information and data structures +use core::ptr::null_mut; + +use alloc::boxed::Box; +use tock_registers::interfaces::Writeable; + +use crate::arch::x86_64::registers::MSR_IA32_KERNEL_GS_BASE; + +use super::apic::local::LocalApic; + +/// Per-CPU data structure, only visible to the executing CPU +#[repr(C, align(0x10))] +pub struct Cpu { + // 0x00 + this: *mut Cpu, + + local_apic: LocalApic, + id: u32, +} + +impl Cpu { + /// Initializes the per-CPU data structure. + /// + /// # Safety + /// + /// Only meant to be called once per each CPU during their init. + pub unsafe fn init_local(local_apic: LocalApic, id: u32) { + let this = Box::new(Cpu { + this: null_mut(), + local_apic, + id, + }); + let this = Box::into_raw(this); + (*this).this = this; + + MSR_IA32_KERNEL_GS_BASE.set(this as u64); + core::arch::asm!("swapgs"); + MSR_IA32_KERNEL_GS_BASE.set(this as u64); + } + + /// Returns this CPU's local data structure + #[inline(always)] + pub fn local<'a>() -> &'a Self { + Self::get_local().unwrap() + } + + /// Returns this CPU's local data structure or None if it hasn't been set up yet + #[inline(always)] + pub fn get_local<'a>() -> Option<&'a Self> { + let mut addr: usize; + unsafe { + core::arch::asm!("movq %gs:(0), {0}", out(reg) addr, options(att_syntax)); + (addr as *const Cpu).as_ref() + } + } + + /// Returns this CPU's number + pub fn id(&self) -> u32 { + self.id + } + + /// Returns the local APIC associated with this CPU + pub fn local_apic(&self) -> &LocalApic { + &self.local_apic + } +} diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 6d562456..aff1cd7d 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -1,7 +1,12 @@ +//! x86-64 exception and interrupt handling use core::{arch::global_asm, mem::size_of_val}; -use crate::arch::{Architecture, ArchitectureImpl}; +use crate::arch::{ + x86_64::apic::{self, __x86_64_apic_irq_handler}, + Architecture, ArchitectureImpl, +}; +/// Set of registers saved when taking an exception/interrupt #[derive(Debug)] #[repr(C)] pub struct ExceptionFrame { @@ -20,11 +25,15 @@ pub struct ExceptionFrame { r13: usize, r14: usize, r15: usize, + exc_number: usize, + exc_code: usize, } +/// Exception table entry +#[allow(dead_code)] #[derive(Clone, Copy)] #[repr(packed)] -struct Entry { +pub struct Entry { base_lo: u16, selector: u16, __res0: u8, @@ -34,6 +43,7 @@ struct Entry { __res1: u32, } +#[allow(dead_code)] #[repr(packed)] struct Pointer { limit: u16, @@ -43,10 +53,13 @@ struct Pointer { const SIZE: usize = 256; impl Entry { - const PRESENT: u8 = 1 << 7; - const INT32: u8 = 0xE; + /// Entry is valid + pub const PRESENT: u8 = 1 << 7; + /// Entry is a 32-bit interrupt + pub const INT32: u8 = 0xE; - const NULL: Self = Self { + /// Empty entry + pub const NULL: Self = Self { base_lo: 0, base_hi: 0, base_ex: 0, @@ -56,7 +69,8 @@ impl Entry { __res1: 0, }; - const fn new(base: usize, selector: u16, flags: u8) -> Self { + /// Constructs an interrupt table entry + pub const fn new(base: usize, selector: u16, flags: u8) -> Self { Self { base_lo: (base & 0xFFFF) as u16, base_hi: ((base >> 16) & 0xFFFF) as u16, @@ -71,13 +85,28 @@ impl Entry { static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; -extern "C" fn __x86_64_exception_handler() { - errorln!("An exception occurrred"); +extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { + let frame = unsafe { &*frame }; + errorln!("Exception {}", frame.exc_number); + + if frame.exc_number == 14 { + let cr2: usize; + unsafe { + core::arch::asm!("mov %cr2, {0}", out(reg) cr2, options(att_syntax)); + } + errorln!("Page fault at {:#x}", cr2); + } + loop { ArchitectureImpl::wait_for_interrupt(); } } +/// Initializes the interrupt descriptor table for the given CPU. +/// +/// # Safety +/// +/// Only meant to be called once per each CPU during their init. pub unsafe fn init_exceptions(_cpu_index: usize) { extern "C" { static __x86_64_exception_vectors: [usize; 32]; @@ -87,6 +116,8 @@ pub unsafe fn init_exceptions(_cpu_index: usize) { IDT[i] = Entry::new(entry, 0x08, Entry::PRESENT | Entry::INT32); } + apic::setup_vectors(&mut IDT[32..]); + let idtr = Pointer { limit: size_of_val(&IDT) as u16 - 1, offset: &IDT as *const _ as usize, @@ -98,5 +129,6 @@ pub unsafe fn init_exceptions(_cpu_index: usize) { global_asm!( include_str!("vectors.S"), exception_handler = sym __x86_64_exception_handler, + apic_irq_handler = sym __x86_64_apic_irq_handler, options(att_syntax) ); diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 9a817c80..7978e59c 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,7 +1,8 @@ +//! x86-64 Global Descriptor Table interface // TODO TSS - use core::mem::size_of_val; +#[allow(dead_code)] #[repr(packed)] struct Entry { limit_lo: u16, @@ -12,6 +13,7 @@ struct Entry { base_hi: u8, } +#[allow(dead_code)] #[repr(packed)] struct Pointer { limit: u16, @@ -24,7 +26,10 @@ impl Entry { const ACC_SYSTEM: u8 = 1 << 4; const ACC_EXECUTE: u8 = 1 << 3; const ACC_WRITE: u8 = 1 << 1; + + #[allow(unused)] const ACC_RING3: u8 = 3 << 5; + #[allow(unused)] const ACC_ACCESS: u8 = 1 << 0; const NULL: Self = Self { @@ -70,6 +75,11 @@ static mut GDT: [Entry; SIZE] = [ ), ]; +/// Initializes the global descriptor table. +/// +/// # Safety +/// +/// Only meant to be called by the CPUs during their early init. pub unsafe fn init() { let gdtr = Pointer { limit: size_of_val(&GDT) as u16 - 1, diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index 1deca692..71375ac7 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -1,5 +1,8 @@ -/// Returns an absolute address to the given symbol +//! x86-64 architecture helper functions +use core::marker::PhantomData; + +/// Returns an absolute address to the given symbol #[macro_export] macro_rules! absolute_address { ($sym:expr) => {{ @@ -10,3 +13,37 @@ macro_rules! absolute_address { _x }}; } + +/// Wrapper struct providing access to an x86 I/O port +#[derive(Clone, Debug)] +#[repr(transparent)] +pub struct IoPort { + address: u16, + _pd: PhantomData, +} + +impl IoPort { + /// Constructs a new I/O port interface + pub const fn new(address: u16) -> Self { + Self { + address, + _pd: PhantomData, + } + } + + /// Writes a byte into the port + pub fn write(&self, value: u8) { + unsafe { + core::arch::asm!("outb %al, %dx", in("al") value, in("dx") self.address, options(att_syntax)); + } + } + + /// Reads a byte from the port + pub fn read(&self) -> u8 { + let value: u8; + unsafe { + core::arch::asm!("inb %dx, %al", in("dx") self.address, out("al") value, options(att_syntax)); + } + value + } +} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index ded7fdb9..ce4e913a 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,4 +1,8 @@ +//! x86-64 architecture and platform implementation +use core::ptr::NonNull; + use abi::error::Error; +use acpi::{AcpiHandler, AcpiTables, InterruptModel, PhysicalMapping}; use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap}; use crate::{ @@ -6,32 +10,55 @@ use crate::{ debug::DebugSink, device::{ display::{fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, + interrupt::{ExternalInterruptController, InterruptSource}, platform::Platform, + Device, + }, + mem::{ + phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, + ConvertAddress, }, - mem::phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, util::OneTimeInit, }; +use self::{ + apic::{ioapic::IoApic, IrqNumber}, + intrinsics::IoPort, + peripherals::ps2::PS2Controller, +}; + use super::Architecture; #[macro_use] pub mod intrinsics; +pub mod apic; pub mod boot; +pub mod cpu; pub mod exception; pub mod gdt; +pub mod peripherals; +pub mod registers; pub mod table; +/// Helper trait to provide abstract access to available memory regions pub trait AbstractAvailableRegion { + /// Returns page-aligned physical start address of the region fn start_address(&self) -> usize; + /// Returns the page count (rounded down) of this region fn page_count(&self) -> usize; } +/// Helper trait to provide abstract access to memory maps pub trait AbstractMemoryMap<'a>: 'a { + /// Available memory region type contained within this memory map type AvailableRegion: AbstractAvailableRegion; + /// Iterator type returned by [Self::iter] type Iter: Iterator + Clone; + /// Returns the physical memory range which contains this memory map fn reserved_range(&self) -> PhysicalMemoryRegion; + /// Returns an iterator over the available memory regions fn iter(&self) -> Self::Iter; } @@ -61,8 +88,43 @@ impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T { } } +#[derive(Clone, Copy)] +struct AcpiHandlerImpl; + +impl AcpiHandler for AcpiHandlerImpl { + // No actual address space modification is performed + unsafe fn map_physical_region( + &self, + physical_address: usize, + size: usize, + ) -> PhysicalMapping { + if physical_address <= 0xFFFFFFFF { + PhysicalMapping::new( + physical_address, + NonNull::new_unchecked(physical_address.virtualize() as *mut T), + size, + size, + *self, + ) + } else { + todo!() + } + } + + // Unmap nothing, these addresses are "virtualized" to high address space + fn unmap_physical_region(_region: &PhysicalMapping) {} +} + +/// x86-64 architecture + platform implementation pub struct X86_64 { yboot_framebuffer: OneTimeInit, + rsdp_phys_base: OneTimeInit, + + // Static devices with dynamic addresses + ioapic: IoApic, + + // Static devices + ps2: PS2Controller, } impl Architecture for X86_64 { @@ -106,9 +168,34 @@ impl Architecture for X86_64 { } impl Platform for X86_64 { + type IrqNumber = apic::IrqNumber; + const KERNEL_PHYS_BASE: usize = 0x400000; - unsafe fn init(&'static self, _is_bsp: bool) -> Result<(), Error> { + unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { + // self.apic.init()?; + + if is_bsp { + Self::disable_8259(); + + // Initialize I/O APIC from ACPI + let rsdp = *self.rsdp_phys_base.get(); + let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap(); + let platform_info = acpi_tables.platform_info().unwrap(); + + let InterruptModel::Apic(apic_info) = &platform_info.interrupt_model else { + panic!("Processor does not have APIC"); + }; + + self.ioapic.init_from_acpi(apic_info)?; + + // Initialize static devices + self.ps2.init()?; + + // Enable IRQs for the devices + self.ps2.init_irq()?; + } + Ok(()) } @@ -145,6 +232,12 @@ impl Platform for X86_64 { None } } + + fn interrupt_controller( + &self, + ) -> &dyn ExternalInterruptController { + &self.ioapic + } } impl X86_64 { @@ -166,10 +259,40 @@ impl X86_64 { })) .expect("Failed to initialize the physical memory manager"); } + + fn init_rsdp(&self, phys: usize) { + self.rsdp_phys_base.init(phys); + } + + unsafe fn disable_8259() { + // TODO should I make a module for 8259 if I don't even use it? + let pic_master_cmd = IoPort::new(0x20); + let pic_master_data = IoPort::new(0x21); + let pic_slave_cmd = IoPort::new(0xA0); + let pic_slave_data = IoPort::new(0xA1); + + // Remap PIC IRQ vectors to 32.. + pic_master_cmd.write(0x11); + pic_slave_cmd.write(0x11); + + pic_master_data.write(32); + pic_slave_data.write(32 + 8); + + pic_slave_data.write(0xFF); + pic_master_data.write(0xFF); + + pic_master_cmd.write(0x20); + pic_slave_cmd.write(0x20); + } } +/// x86-64 architecture + platform implementation pub static ARCHITECTURE: X86_64 = X86_64 { + rsdp_phys_base: OneTimeInit::new(), yboot_framebuffer: OneTimeInit::new(), + ioapic: IoApic::new(), + + ps2: PS2Controller::new(IrqNumber::Isa(1), IrqNumber::Isa(12), 0x64, 0x60), }; static LINEAR_FB: OneTimeInit = OneTimeInit::new(); diff --git a/src/arch/x86_64/peripherals/mod.rs b/src/arch/x86_64/peripherals/mod.rs new file mode 100644 index 00000000..8bb8ca17 --- /dev/null +++ b/src/arch/x86_64/peripherals/mod.rs @@ -0,0 +1,3 @@ +//! x86-64 platform peripheral drivers + +pub mod ps2; diff --git a/src/arch/x86_64/peripherals/ps2.rs b/src/arch/x86_64/peripherals/ps2.rs new file mode 100644 index 00000000..5ddcf923 --- /dev/null +++ b/src/arch/x86_64/peripherals/ps2.rs @@ -0,0 +1,102 @@ +//! Intel 8042 PS/2 controller driver implemenation +use abi::error::Error; + +use crate::{ + arch::{ + x86_64::{apic::IrqNumber, intrinsics::IoPort}, + PLATFORM, + }, + device::{interrupt::InterruptSource, platform::Platform, Device}, +}; + +/// PS/2 controller driver +pub struct PS2Controller { + command: IoPort, + data: IoPort, + primary_irq: IrqNumber, + #[allow(unused)] + auxiliary_irq: IrqNumber, +} + +impl InterruptSource for PS2Controller { + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = PLATFORM.interrupt_controller(); + + intc.register_handler(self.primary_irq, self)?; + + // Disable PS/2 devices from sending any further data + self.send_command(0xAD); + self.send_command(0xA7); + + // Flush the buffer + while self.command.read() & Self::STATUS_OUTPUT_FULL != 0 { + self.data.read(); + } + + // Enable primary port + self.send_command(0xAE); + + intc.enable_irq(self.primary_irq)?; + + Ok(()) + } + + fn handle_irq(&self) -> Result { + let status = self.command.read(); + + if status & 1 == 0 { + return Ok(false); + } + + let mut key = self.data.read(); + + if key == 0xE0 { + key = self.data.read(); + } + + self.data.read(); + + if key < 128 { + debugln!("Got key {:#x}", key); + } + + Ok(true) + } +} + +impl Device for PS2Controller { + unsafe fn init(&self) -> Result<(), Error> { + Ok(()) + } + + fn name(&self) -> &'static str { + "PS/2 Controller" + } +} + +impl PS2Controller { + const STATUS_OUTPUT_FULL: u8 = 1 << 0; + const STATUS_INPUT_FULL: u8 = 1 << 1; + + /// Constructs a new instance of the device + pub const fn new( + primary_irq: IrqNumber, + auxiliary_irq: IrqNumber, + cmd_port: u16, + data_port: u16, + ) -> Self { + Self { + primary_irq, + auxiliary_irq, + command: IoPort::new(cmd_port), + data: IoPort::new(data_port), + } + } + + fn send_command(&self, cmd: u8) { + while self.command.read() & Self::STATUS_INPUT_FULL != 0 { + core::hint::spin_loop(); + } + self.command.write(cmd); + } +} diff --git a/src/arch/x86_64/registers/mod.rs b/src/arch/x86_64/registers/mod.rs new file mode 100644 index 00000000..fb6d6b48 --- /dev/null +++ b/src/arch/x86_64/registers/mod.rs @@ -0,0 +1,102 @@ +//! Helper types for interfacing with x86-64 registers + +macro_rules! msr_impl_read { + ($t:ident, $addr:expr, $register:ty) => { + impl tock_registers::interfaces::Readable for $t { + type T = u64; + type R = $register; + + #[inline] + fn get(&self) -> u64 { + let (high, low): (u32, u32); + unsafe { + core::arch::asm!( + "rdmsr", + in("ecx") $addr, + out("eax") low, + out("edx") high, + options(att_syntax) + ); + } + ((high as u64) << 32) | (low as u64) + } + } + }; + + ($t:ident, $addr:expr) => { msr_impl_read!($t, $addr, ()); }; +} + +macro_rules! msr_impl_write { + ($t:ident, $addr:expr, $register:ty) => { + impl tock_registers::interfaces::Writeable for $t { + type T = u64; + type R = $register; + + #[inline] + fn set(&self, value: u64) { + let low = value as u32; + let high = (value >> 32) as u32; + unsafe { + core::arch::asm!( + "wrmsr", + in("ecx") $addr, + in("eax") low, + in("edx") high, + options(att_syntax) + ); + } + } + } + }; + + ($t:ident, $addr:expr) => { msr_impl_write!($t, $addr, ()); }; +} + +mod msr_ia32_kernel_gs_base { + const ADDR: u32 = 0xC0000102; + pub struct Reg; + + msr_impl_read!(Reg, ADDR); + msr_impl_write!(Reg, ADDR); + + /// IA32_KERNEL_GS_BASE model-specific register. Provides the base address for %gs-relative + /// loads/stores. + pub const MSR_IA32_KERNEL_GS_BASE: Reg = Reg; +} + +mod msr_ia32_apic_base { + use tock_registers::{interfaces::Readable, register_bitfields}; + + register_bitfields! { + u64, + #[allow(missing_docs)] + #[doc = "IA32_APIC_BASE model-specific register"] + pub MSR_IA32_APIC_BASE [ + #[doc = "Contains a virtual page number of the Local APIC base address for this processor"] + AddressPage OFFSET(12) NUMBITS(40) [], + #[doc = "If set, the APIC is enabled"] + ApicEnable OFFSET(11) NUMBITS(1) [], + #[doc = "If set, this CPU is a bootstrap processor"] + BootstrapCpuCore OFFSET(8) NUMBITS(1) [], + ] + } + + const ADDR: u32 = 0x0000001B; + pub struct Reg; + + msr_impl_read!(Reg, ADDR, MSR_IA32_APIC_BASE::Register); + msr_impl_write!(Reg, ADDR, MSR_IA32_APIC_BASE::Register); + + impl Reg { + #[inline] + pub fn read_base(&self) -> u64 { + self.read(MSR_IA32_APIC_BASE::AddressPage) << 12 + } + } + + /// IA32_APIC_BASE model-specific register + pub const MSR_IA32_APIC_BASE: Reg = Reg; +} + +pub use msr_ia32_apic_base::MSR_IA32_APIC_BASE; +pub use msr_ia32_kernel_gs_base::MSR_IA32_KERNEL_GS_BASE; diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs index 8f3fa41b..2ab6e03f 100644 --- a/src/arch/x86_64/table/fixed.rs +++ b/src/arch/x86_64/table/fixed.rs @@ -11,6 +11,7 @@ const KERNEL_PD_COUNT: usize = 4; const DEVICE_MAPPING_L1I: usize = KERNEL_PD_COUNT + 1; const DEVICE_VIRT_OFFSET: usize = (DEVICE_MAPPING_L1I << 30) + KERNEL_VIRT_OFFSET; +/// Fixed tables for x86-64. Provide device mappings and static kernel mapping. pub struct FixedTables { // Common l0: PageTable, @@ -29,6 +30,7 @@ pub struct FixedTables { } impl FixedTables { + /// Constructs a set of empty translation tables pub const fn zeroed() -> Self { Self { // Global @@ -49,6 +51,7 @@ impl FixedTables { } } + /// Maps a specified count of physical memory pages to the device virtual address space pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result { if count > 512 * 512 { panic!("Unsupported device memory mapping size"); @@ -84,13 +87,20 @@ impl FixedTables { } } + /// Returns the physical address of the fixed PML4 pub fn physical_address(&self) -> usize { self.l0.physical_address() } } +/// Instance of fixed translation tables pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed(); +/// Initializes the fixed translation tables. +/// +/// # Safety +/// +/// Only meant to be called by BSP during early init. pub unsafe fn init_fixed_tables() { // Kernel L2 for i in 0..512 * KERNEL_PD_COUNT { diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/table/mod.rs index 672d1289..1200bc40 100644 --- a/src/arch/x86_64/table/mod.rs +++ b/src/arch/x86_64/table/mod.rs @@ -1,3 +1,5 @@ +//! x86-64 virtual memory management implementation + use core::{ marker::PhantomData, ops::{Index, IndexMut}, @@ -15,33 +17,41 @@ use crate::mem::{ }; bitflags! { + /// Describes how each page table entry is mapped pub struct PageAttributes: u64 { + /// When set, the mapping is considered valid and pointing somewhere const PRESENT = 1 << 0; + /// For tables, allows writes to further translation levels, for pages/blocks, allows + /// writes to the region covered by the entry const WRITABLE = 1 << 1; + /// When set for L2 entries, the mapping specifies a 2MiB page instead of a page table + /// reference const BLOCK = 1 << 7; } } +/// Represents a single virtual address space mapping depending on its translation level #[derive(Clone, Copy)] #[repr(transparent)] pub struct PageEntry(u64, PhantomData); +/// Table describing a single level of address translation #[derive(Clone, Copy)] #[repr(C, align(0x1000))] pub struct PageTable { data: [PageEntry; 512], } -// L0: PML4, 512GiB page +/// Translation level 0 (PML4): Entry is 512GiB table #[derive(Clone, Copy)] pub struct L0; -// L1: PDPT, 1GiB page +/// Translation level 1 (PDPT): Entry is 1GiB table #[derive(Clone, Copy)] pub struct L1; -// L2: Page directory, 2MiB page +/// Translation level 2 (Page directory): Entry is 2MiB block/table #[derive(Clone, Copy)] pub struct L2; -// L3: Page table, 4KiB page +/// Translation level 3 (Page table): Entry is 4KiB page #[derive(Clone, Copy)] pub struct L3; @@ -55,17 +65,12 @@ impl NonTerminalEntryLevel for L2 { type NextLevel = L3; } -// #[repr(C)] -// pub struct AddressSpace { -// l0: *mut PageTable, -// } - impl const EntryLevel for L0 { - fn index(addr: usize) -> usize { + fn index(_addr: usize) -> usize { todo!() } - fn page_offset(addr: usize) -> usize { + fn page_offset(_addr: usize) -> usize { todo!() } } @@ -101,6 +106,7 @@ impl const EntryLevel for L3 { } impl PageEntry { + /// Constructs a mapping which points to a 4KiB page pub fn page(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (attrs | PageAttributes::PRESENT).bits(), @@ -110,6 +116,7 @@ impl PageEntry { } impl PageEntry { + /// Constructs a mapping which points to a 2MiB block pub fn block(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), @@ -119,6 +126,7 @@ impl PageEntry { } impl PageEntry { + /// Constructs a mapping which points to a next-level table pub fn table(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::WRITABLE).bits(), @@ -128,16 +136,19 @@ impl PageEntry { } impl PageEntry { + /// An entry that is not mapped pub const INVALID: Self = Self(0, PhantomData); } impl PageTable { + /// Constructs a page table filled with invalid (non-present) entries pub const fn zeroed() -> Self { Self { data: [PageEntry::INVALID; 512], } } + /// Returns the physical address of this table pub fn physical_address(&self) -> usize { unsafe { (self.data.as_ptr() as usize).physicalize() } } diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index c816655f..e176efb0 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -13,6 +13,26 @@ __x86_64_exc_\n: jmp __x86_64_exc_common .endm +.macro apic_vector, n +__x86_64_apic_irq_\n: + cli + + // Push dummy error codes + // pushq $0 + // pushq $0 + + EXC_SAVE_STATE + mov $\n, %rdi + mov %rsp, %rsi + call {apic_irq_handler} + EXC_RESTORE_STATE + + // Remove dummy error codes + // add $16, %rsp + + iretq +.endm + // 16 general-purpose registers .set PT_REGS_SIZE, 15 * 8 @@ -53,9 +73,13 @@ __x86_64_exc_\n: mov 72(%rsp), %r13 mov 76(%rsp), %r14 mov 80(%rsp), %r15 + + addq $PT_REGS_SIZE, %rsp .endm .global __x86_64_exception_vectors +.global __x86_64_apic_vectors + .section .text __x86_64_exc_common: EXC_SAVE_STATE @@ -101,6 +125,24 @@ ISR_NERR 29 ISR_YERR 30 ISR_NERR 31 +apic_vector 0 +apic_vector 1 +apic_vector 2 +apic_vector 3 +apic_vector 4 +apic_vector 5 +apic_vector 6 +apic_vector 7 +apic_vector 8 +apic_vector 9 +apic_vector 10 +apic_vector 11 +apic_vector 12 +apic_vector 13 +apic_vector 14 +apic_vector 15 + + .section .rodata .global __x86_64_exception_vectors .p2align 4 @@ -137,3 +179,24 @@ __x86_64_exception_vectors: .quad __x86_64_exc_29 .quad __x86_64_exc_30 .quad __x86_64_exc_31 + +.p2align 4 +__x86_64_apic_vectors: + .quad __x86_64_apic_irq_0 + .quad __x86_64_apic_irq_1 + .quad __x86_64_apic_irq_2 + .quad __x86_64_apic_irq_3 + .quad __x86_64_apic_irq_4 + .quad __x86_64_apic_irq_5 + .quad __x86_64_apic_irq_6 + .quad __x86_64_apic_irq_7 + .quad __x86_64_apic_irq_8 + .quad __x86_64_apic_irq_9 + .quad __x86_64_apic_irq_10 + .quad __x86_64_apic_irq_11 + .quad __x86_64_apic_irq_12 + .quad __x86_64_apic_irq_13 + .quad __x86_64_apic_irq_14 + .quad __x86_64_apic_irq_15 + +.section .text diff --git a/src/debug.rs b/src/debug.rs index 6db53a88..449656cd 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -27,9 +27,12 @@ pub enum LogLevel { Fatal, } +/// Generic interface for debug output pub trait DebugSink { + /// Sends a single byte to the output fn putc(&self, c: u8) -> Result<(), Error>; + /// Sends a string of bytes to the output fn puts(&self, s: &str) -> Result<(), Error> { for &byte in s.as_bytes() { self.putc(byte)?; @@ -37,8 +40,9 @@ pub trait DebugSink { Ok(()) } - fn supports_color(&self) -> bool { - return false; + /// Returns `true` if the device supports vt100-like control sequences + fn supports_control_sequences(&self) -> bool { + false } } @@ -86,27 +90,27 @@ debug_tpl!($ fatal, fatalln, Fatal); #[no_mangle] static DEBUG_PRINTER: OneTimeInit> = OneTimeInit::new(); -impl LogLevel { - fn log_prefix(self) -> &'static str { - match self { - LogLevel::Debug => "", - LogLevel::Info => "\x1b[36m\x1b[1m", - LogLevel::Warning => "\x1b[33m\x1b[1m", - LogLevel::Error => "\x1b[31m\x1b[1m", - LogLevel::Fatal => "\x1b[38;2;255;0;0m\x1b[1m", - } - } - - fn log_suffix(self) -> &'static str { - match self { - LogLevel::Debug => "", - LogLevel::Info => "\x1b[0m", - LogLevel::Warning => "\x1b[0m", - LogLevel::Error => "\x1b[0m", - LogLevel::Fatal => "\x1b[0m", - } - } -} +// impl LogLevel { +// fn log_prefix(self) -> &'static str { +// match self { +// LogLevel::Debug => "", +// LogLevel::Info => "\x1b[36m\x1b[1m", +// LogLevel::Warning => "\x1b[33m\x1b[1m", +// LogLevel::Error => "\x1b[31m\x1b[1m", +// LogLevel::Fatal => "\x1b[38;2;255;0;0m\x1b[1m", +// } +// } +// +// fn log_suffix(self) -> &'static str { +// match self { +// LogLevel::Debug => "", +// LogLevel::Info => "\x1b[0m", +// LogLevel::Warning => "\x1b[0m", +// LogLevel::Error => "\x1b[0m", +// LogLevel::Fatal => "\x1b[0m", +// } +// } +// } impl fmt::Write for DebugPrinter { fn write_str(&mut self, s: &str) -> fmt::Result { @@ -152,12 +156,12 @@ pub fn init() { // } } -pub fn init_with_sink(sink: &'static dyn DebugSink) { - DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { sink })); -} +// pub fn init_with_sink(sink: &'static dyn DebugSink) { +// DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { sink })); +// } #[doc(hidden)] -pub fn debug_internal(args: Arguments, level: LogLevel) { +pub fn debug_internal(args: Arguments, _level: LogLevel) { use fmt::Write; if DEBUG_PRINTER.is_initialized() { diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 32cdeea6..d9d271db 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -1,3 +1,5 @@ +//! Framebuffer console driver + use abi::error::Error; use bitmap_font::{BitmapFont, TextStyle}; use embedded_graphics::{ @@ -22,11 +24,13 @@ struct Inner { height_chars: u32, } +/// Framebuffer console device wrapper pub struct FramebufferConsole { inner: IrqSafeSpinlock, } impl FramebufferConsole { + /// Constructs an instance of console from its framebuffer reference pub fn from_framebuffer( framebuffer: &'static LinearFramebuffer, font: &'static BitmapFont<'static>, @@ -114,7 +118,7 @@ impl DebugSink for FramebufferConsole { Ok(()) } - fn supports_color(&self) -> bool { + fn supports_control_sequences(&self) -> bool { false } } diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 0aa86ecf..62abdf20 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -1,3 +1,5 @@ +//! Abstract linear framebuffer device implementation + use core::ops::{Index, IndexMut}; use abi::error::Error; @@ -12,17 +14,24 @@ struct Inner { stride: usize, } +#[doc(hidden)] pub struct FramebufferAccess { dimensions: DisplayDimensions, base: usize, stride: usize, } +/// Linear framebuffer wrapper pub struct LinearFramebuffer { inner: IrqSafeSpinlock, } impl LinearFramebuffer { + /// Constructs a liner framebuffer struct from its components. + /// + /// # Safety + /// + /// Unsafe: the caller must ensure the validity of all the arguments. pub unsafe fn from_physical_bits( base: usize, size: usize, @@ -49,6 +58,11 @@ impl LinearFramebuffer { Ok(res) } + /// Temporary function to provide framebuffer access + /// + /// # Safety + /// + /// Unsafe: access is not synchronized // TODO doesn't actually lock pub unsafe fn lock(&self) -> FramebufferAccess { let inner = self.inner.lock(); @@ -78,6 +92,7 @@ impl DisplayDevice for LinearFramebuffer { } impl FramebufferAccess { + /// Copies `count` rows starting from `src_row` to `dst_row` pub fn copy_rows(&mut self, src_row: u32, dst_row: u32, count: u32) { use core::ffi::c_void; extern "C" { @@ -108,6 +123,7 @@ impl FramebufferAccess { } } + /// Fills the specified number of pixel rows with given pixel value pub fn fill_rows(&mut self, start_row: u32, count: u32, value: u32) { use core::ffi::c_void; extern "C" { diff --git a/src/device/display/mod.rs b/src/device/display/mod.rs index 7f718c4f..26db455f 100644 --- a/src/device/display/mod.rs +++ b/src/device/display/mod.rs @@ -1,14 +1,21 @@ +//! Display device interfaces + use super::Device; pub mod fb_console; pub mod linear_fb; +/// Resolution of the display device #[derive(Clone, Copy, Debug)] pub struct DisplayDimensions { + /// Width of the display in pixels pub width: u32, + /// Height of the display in pixels pub height: u32, } +/// Abstract display device interface pub trait DisplayDevice: Device { + /// Returns the dimensions of the display in its current mode fn dimensions(&self) -> DisplayDimensions; } diff --git a/src/device/interrupt.rs b/src/device/interrupt.rs index b2a531f7..e673b226 100644 --- a/src/device/interrupt.rs +++ b/src/device/interrupt.rs @@ -3,10 +3,11 @@ use core::marker::PhantomData; use abi::error::Error; -use crate::arch::CpuMessage; - use super::Device; +/// Convenience type alias for a static IRQ handler +pub type IrqHandler = &'static (dyn InterruptSource + Sync); + /// Specifies the target(s) of interprocessor interrupt delivery #[derive(Clone, Copy, PartialEq, Debug)] pub enum IpiDeliveryTarget { @@ -25,12 +26,39 @@ pub trait InterruptSource: Device { /// The caller must ensure the function hasn't been called before. unsafe fn init_irq(&'static self) -> Result<(), Error>; - /// Handles the interrupt raised by the device - fn handle_irq(&self) -> Result<(), Error>; + /// Handles the interrupt raised by the device. Returns `true` if the interrupt was handled, + /// `false` if device itself sent no interrupt (may be possible if several interrupt handlers + /// share the same IRQ vector). + fn handle_irq(&self) -> Result; } -/// Interface for a device responsible for routing and handling IRQs -pub trait InterruptController: Device { +// TODO not yet ready for this +// /// Controls per-CPU interrupt delivery +// pub trait LocalInterruptController { +// type Vector; +// type Id; +// +// /// Allocates a vector for external IRQ +// unsafe fn allocate_irq_vector(&self) -> Result; +// +// /// Returns the unique ID of this local interrupt controller +// fn id(&self) -> Self::Id; +// +// /// Sends a message to the requested set of CPUs through an interprocessor interrupt. +// /// +// /// # Note +// /// +// /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations +// /// may impose narrower restrictions. +// /// +// /// # Safety +// /// +// /// As the call may alter the flow of execution on CPUs, this function is unsafe. +// unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; +// } + +/// Interface for a device responsible for routing and handling IRQs from external sources +pub trait ExternalInterruptController: Device { /// Interrupt number wrapper type type IrqNumber; @@ -43,21 +71,6 @@ pub trait InterruptController: Device { /// Enables given interrupt number/vector fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error>; - - /// Handles all pending interrupts on this controller - fn handle_pending_irqs<'irq>(&'irq self, ic: &IrqContext<'irq>); - - /// Sends a message to the requested set of CPUs through an interprocessor interrupt. - /// - /// # Note - /// - /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations - /// may impose narrower restrictions. - /// - /// # Safety - /// - /// As the call may alter the flow of execution on CPUs, this function is unsafe. - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; } /// Token type to indicate that the code is being run from an interrupt handler diff --git a/src/device/mod.rs b/src/device/mod.rs index d00e96e1..537f6a8a 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,8 +1,8 @@ //! Device management and interfaces use abi::error::Error; -// pub mod interrupt; pub mod display; +pub mod interrupt; pub mod platform; // pub mod serial; // pub mod timer; diff --git a/src/device/platform.rs b/src/device/platform.rs index 762adc26..84a62f8b 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -4,12 +4,12 @@ use abi::error::Error; use crate::debug::DebugSink; -// use super::{interrupt::InterruptController, serial::SerialDevice, timer::TimestampSource}; +use super::interrupt::ExternalInterruptController; /// Platform interface for interacting with a general hardware set pub trait Platform { /// Interrupt number type for the platform - // type IrqNumber; + type IrqNumber; /// Address, to which the kernel is expected to be loaded for this platform const KERNEL_PHYS_BASE: usize; @@ -39,12 +39,13 @@ pub trait Platform { /// May not be initialized at the moment of calling. fn primary_debug_sink(&self) -> Option<&dyn DebugSink>; - // /// Returns a reference to the platform's interrupt controller. - // /// - // /// # Note - // /// - // /// May not be initialized at the moment of calling. - // fn interrupt_controller(&self) -> &dyn InterruptController; + /// Returns a reference to the platform's interrupt controller. + /// + /// # Note + /// + /// May not be initialized at the moment of calling. + fn interrupt_controller(&self) + -> &dyn ExternalInterruptController; // /// Returns the platform's primary timestamp source. // /// diff --git a/src/main.rs b/src/main.rs index 32cb0823..a1f76675 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,27 +28,16 @@ extern crate yggdrasil_abi as abi; // use vfs::{Filesystem, IoContext, VnodeRef}; // extern crate alloc; -// + #[macro_use] pub mod debug; #[macro_use] pub mod arch; -#[panic_handler] -fn panic_handler(_pi: &core::panic::PanicInfo) -> ! { - use arch::{Architecture, ArchitectureImpl}; - - fatalln!("KERNEL PANIC"); - - loop { - ArchitectureImpl::wait_for_interrupt(); - } -} -// pub mod device; // pub mod fs; pub mod mem; -// pub mod panic; +pub mod panic; // pub mod proc; pub mod sync; // pub mod syscall; diff --git a/src/mem/device.rs b/src/mem/device.rs index d689a6eb..d7530111 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -39,6 +39,7 @@ impl DeviceMemory { Ok(Self { name, base, size }) } + /// Returns the base address of this mapping #[inline] pub fn base(&self) -> usize { self.base diff --git a/src/panic.rs b/src/panic.rs index 8bdcb080..ff673bd9 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,9 +2,8 @@ use core::sync::atomic::{AtomicBool, Ordering}; use crate::{ - arch::{Architecture, ArchitectureImpl, CpuMessage, PLATFORM}, + arch::{Architecture, ArchitectureImpl}, debug::{debug_internal, LogLevel}, - device::{interrupt::IpiDeliveryTarget, platform::Platform}, sync::SpinFence, }; @@ -38,12 +37,12 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .is_ok() { // Let other CPUs know we're screwed - unsafe { - PLATFORM - .interrupt_controller() - .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) - .ok(); - } + // unsafe { + // PLATFORM + // .interrupt_controller() + // .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) + // .ok(); + // } log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); log_print_raw!(LogLevel::Fatal, "Kernel panic "); From a9b9c71e47e8896f00f028c51a3508db0dcce786 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 1 Aug 2023 18:05:10 +0300 Subject: [PATCH 027/211] x86-64: ring0/ring3 tasks + syscall --- Cargo.toml | 2 +- src/arch/x86_64/apic/mod.rs | 2 +- src/arch/x86_64/boot/mod.rs | 15 +- src/arch/x86_64/context.S | 120 ++++++++ src/arch/x86_64/context.rs | 190 +++++++++++++ src/arch/x86_64/cpu.rs | 37 ++- src/arch/x86_64/cpuid.rs | 80 ++++++ src/arch/x86_64/exception.rs | 49 +++- src/arch/x86_64/gdt.rs | 82 +++++- src/arch/x86_64/mod.rs | 4 +- src/arch/x86_64/registers/mod.rs | 201 ++++++++++++++ src/arch/x86_64/table/fixed.rs | 4 + src/arch/x86_64/table/mod.rs | 48 +++- src/arch/x86_64/vectors.S | 111 +++++--- src/main.rs | 4 +- src/mem/table.rs | 2 +- src/sync.rs | 17 +- src/task/mod.rs | 83 +++++- src/task/process.rs | 457 +++++++++++++++---------------- src/task/sched.rs | 36 ++- 20 files changed, 1226 insertions(+), 318 deletions(-) create mode 100644 src/arch/x86_64/context.S create mode 100644 src/arch/x86_64/context.rs create mode 100644 src/arch/x86_64/cpuid.rs diff --git a/Cargo.toml b/Cargo.toml index c67503f2..e04376ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } # vfs = { path = "lib/vfs" } # memfs = { path = "lib/memfs" } -# atomic_enum = "0.2.0" +atomic_enum = "0.2.0" bitflags = "2.3.3" linked_list_allocator = "0.10.5" spinning_top = "0.2.5" diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 83610997..cc20f8ab 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -73,7 +73,7 @@ pub extern "C" fn __x86_64_apic_irq_handler(vector: u32, _frame: *mut ExceptionF cpu.local_apic().clear_interrupt(); if vector == APIC_TIMER_VECTOR { - // TODO + unsafe { Cpu::local().queue().yield_cpu() } } else if (APIC_EXTERNAL_OFFSET..APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS).contains(&vector) { PLATFORM.ioapic.handle_irq(vector - APIC_EXTERNAL_OFFSET); diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index f87c4f76..4ec5e314 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -9,7 +9,7 @@ use yboot_proto::{ use crate::{ arch::{ - x86_64::{apic::local::LocalApic, cpu::Cpu, exception, gdt}, + x86_64::{apic::local::LocalApic, cpu::Cpu, cpuid, exception}, Architecture, ArchitectureImpl, PLATFORM, }, debug, @@ -19,6 +19,7 @@ use crate::{ phys::{self, PageUsage}, ConvertAddress, KERNEL_VIRT_OFFSET, }, + task, }; use super::ARCHITECTURE; @@ -73,6 +74,7 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { debug::init(); infoln!("Yggdrasil kernel git {} starting", git_version!()); + cpuid::feature_gate(); if YBOOT_DATA.memory_map.address > 0xFFFFFFFF { errorln!("Unhandled case: memory map is above 4GiB"); @@ -86,9 +88,6 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { PLATFORM.init_rsdp(YBOOT_DATA.rsdp_address as _); } - gdt::init(); - exception::init_exceptions(0); - // Setup physical memory allocation ArchitectureImpl::init_physical_memory(&YBOOT_DATA.memory_map); @@ -99,13 +98,13 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { // Also initializes local APIC Cpu::init_local(LocalApic::new(), 0); + exception::init_exceptions(0); PLATFORM.init(true).unwrap(); - ArchitectureImpl::set_interrupt_mask(false); - loop { - ArchitectureImpl::wait_for_interrupt(); - } + task::init().expect("Failed to initialize the scheduler"); + + task::enter() } global_asm!( diff --git a/src/arch/x86_64/context.S b/src/arch/x86_64/context.S new file mode 100644 index 00000000..0f392db4 --- /dev/null +++ b/src/arch/x86_64/context.S @@ -0,0 +1,120 @@ +.macro SAVE_TASK_STATE + sub ${context_size}, %rsp + + mov %rbx, 0(%rsp) + mov %r12, 8(%rsp) + mov %r13, 16(%rsp) + mov %r14, 24(%rsp) + mov %r15, 32(%rsp) + // TODO save %fs + mov %rbp, 48(%rsp) + + mov %cr3, %rbx + mov %rbx, 56(%rsp) +.endm + +.macro LOAD_TASK_STATE + mov 56(%rsp), %rbx + mov %rbx, %cr3 + + mov 0(%rsp), %rbx + mov 8(%rsp), %r12 + mov 16(%rsp), %r13 + mov 24(%rsp), %r14 + mov 32(%rsp), %r15 + // TODO + // mov 40(%rsp), %fs + mov 48(%rsp), %rbp + + add ${context_size}, %rsp +.endm + +.global __x86_64_task_enter_user +.global __x86_64_task_enter_kernel +.global __x86_64_enter_task +.global __x86_64_switch_task + +.section .text + +__x86_64_task_enter_user: + // User stack pointer + popq %rcx + // Argument + popq %rdi + // Entry address + popq %rax + + // SS:RSP + pushq $0x1B + pushq %rcx + + // RFLAGS + pushq $0x200 + + // CS:RIP + pushq $0x23 + pushq %rax + + iretq + +__x86_64_task_enter_kernel: + // Argument + popq %rdi + // Entry address + popq %rax + + // Enable IRQ in RFLAGS + pushfq + popq %rdx + or $(1 << 9), %rdx + + mov %rsp, %rcx + + // SS:RSP + pushq $0x10 + pushq %rcx + + // RFLAGS + pushq %rdx + + // CS:RIP + pushq $0x08 + pushq %rax + + iretq + +// %rsi - from struct ptr, %rdi - to struct ptr +__x86_64_switch_task: + SAVE_TASK_STATE + mov %rsp, 0(%rsi) + + // TSS.RSP0 + mov 8(%rdi), %rax + // Kernel stack + mov 0(%rdi), %rdi + + mov %rdi, %rsp + + // Load TSS.RSP0 + mov %gs:(8), %rdi + mov %rax, 4(%rdi) + + LOAD_TASK_STATE + + ret + +// %rdi - to struct ptr +__x86_64_enter_task: + // TSS.RSP0 + mov 8(%rdi), %rax + // Kernel stack + mov 0(%rdi), %rdi + + mov %rdi, %rsp + + // Load TSS.RSP0 + mov %gs:(8), %rdi + mov %rax, 4(%rdi) + + LOAD_TASK_STATE + ret diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs new file mode 100644 index 00000000..9af78eb3 --- /dev/null +++ b/src/arch/x86_64/context.rs @@ -0,0 +1,190 @@ +//! x86-64 task context setup and switching +use core::{arch::global_asm, cell::UnsafeCell}; + +use abi::error::Error; +use alloc::boxed::Box; + +use crate::{ + arch::x86_64::table::KERNEL_TABLES, + mem::{ + phys::{self, PageUsage}, + ConvertAddress, + }, +}; + +struct StackBuilder { + base: usize, + sp: usize, +} + +#[repr(C, align(0x10))] +struct Inner { + // 0x00 + sp: usize, + // 0x08 + tss_rsp0: usize, +} + +/// x86-64 implementation of a task context +#[allow(dead_code)] +pub struct TaskContext { + inner: UnsafeCell, + stack_base: usize, + stack_size: usize, +} + +unsafe impl Sync for TaskContext {} + +// 8 registers + return address (which is not included) +const COMMON_CONTEXT_SIZE: usize = 8 * 8; + +impl StackBuilder { + fn new(base: usize, size: usize) -> Self { + Self { + base, + sp: base + size, + } + } + + fn push(&mut self, value: usize) { + if self.sp == self.base { + panic!(); + } + self.sp -= 8; + unsafe { + (self.sp as *mut usize).write_volatile(value); + } + } + + fn _skip(&mut self, count: usize) { + self.sp -= count * 8; + if self.sp < self.base { + panic!(); + } + } + + fn build(self) -> usize { + self.sp + } + + fn init_common(&mut self, entry: usize, cr3: usize) { + self.push(entry); // address for ret + + // End of common context + self.push(cr3); // %cr3 + + self.push(0); // %rbp + self.push(0); // %fs (TODO) + self.push(0); // %r15 + self.push(0); // %r14 + self.push(0); // %r13 + self.push(0); // %r12 + self.push(0); // %rbx + } +} + +impl TaskContext { + /// Constructs a kernel-space task context + pub fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { + const KERNEL_TASK_PAGES: usize = 4; + let stack_base = unsafe { + phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() + }; + + let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); + + // Entry and argument + stack.push(entry as _); + stack.push(arg); + + stack.init_common(__x86_64_task_enter_kernel as _, unsafe { + KERNEL_TABLES.physical_address() + }); + + let sp = stack.build(); + infoln!("Kernel stack {:#x}", sp); + + // TODO stack is leaked + + Ok(Self { + inner: UnsafeCell::new(Inner { sp, tss_rsp0: 0 }), + stack_base, + stack_size: KERNEL_TASK_PAGES * 0x1000, + }) + } + + /// Constructs a user thread context. The caller is responsible for allocating the userspace + /// stack and setting up a valid address space for the context. + pub fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result { + const USER_TASK_PAGES: usize = 8; + let stack_base = + unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + stack.push(entry as _); + stack.push(arg); + stack.push(user_stack_sp); + + stack.init_common(__x86_64_task_enter_user as _, cr3); + + let sp = stack.build(); + let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; + debugln!("TSS.RSP0 = {:#x}", rsp0); + + Ok(Self { + inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + stack_base, + stack_size: USER_TASK_PAGES * 0x1000, + }) + } + + /// Constructs a safe wrapper process to execute a kernel-space closure + pub fn kernel_closure(f: F) -> Result { + extern "C" fn closure_wrapper(closure_addr: usize) -> ! { + let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; + closure(); + todo!("Process termination"); + } + + let closure = Box::new(f); + debugln!("closure: {:p}", closure); + Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) + } + + /// Performs an entry into a context. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + pub unsafe fn enter(&self) -> ! { + __x86_64_enter_task(self.inner.get()) + } + + /// Performs a context switch between two contexts. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + pub unsafe fn switch(&self, from: &Self) { + let to = self.inner.get(); + let from = from.inner.get(); + + if to != from { + __x86_64_switch_task(to, from) + } + } +} + +extern "C" { + fn __x86_64_task_enter_kernel(); + fn __x86_64_task_enter_user(); + fn __x86_64_enter_task(to: *mut Inner) -> !; + fn __x86_64_switch_task(to: *mut Inner, from: *mut Inner); +} + +global_asm!( + include_str!("context.S"), + context_size = const COMMON_CONTEXT_SIZE, + options(att_syntax) +); diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index f7d9382c..52689d83 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -4,7 +4,11 @@ use core::ptr::null_mut; use alloc::boxed::Box; use tock_registers::interfaces::Writeable; -use crate::arch::x86_64::registers::MSR_IA32_KERNEL_GS_BASE; +use crate::{ + arch::x86_64::{gdt, registers::MSR_IA32_KERNEL_GS_BASE}, + task::sched::CpuQueue, + util::OneTimeInit, +}; use super::apic::local::LocalApic; @@ -13,9 +17,15 @@ use super::apic::local::LocalApic; pub struct Cpu { // 0x00 this: *mut Cpu, + // 0x08, used in assembly + tss_address: usize, + // 0x10, used in assembly for temporary RSP storage + tmp_address: usize, + // 0x08 + id: u32, local_apic: LocalApic, - id: u32, + queue: OneTimeInit<&'static CpuQueue>, } impl Cpu { @@ -25,10 +35,16 @@ impl Cpu { /// /// Only meant to be called once per each CPU during their init. pub unsafe fn init_local(local_apic: LocalApic, id: u32) { + let tss_address = gdt::init(id); + let this = Box::new(Cpu { this: null_mut(), - local_apic, + tss_address, + tmp_address: 0, id, + + local_apic, + queue: OneTimeInit::new(), }); let this = Box::into_raw(this); (*this).this = this; @@ -44,6 +60,11 @@ impl Cpu { Self::get_local().unwrap() } + /// Returns the system ID of the CPU + pub fn local_id() -> u32 { + Self::local().id() + } + /// Returns this CPU's local data structure or None if it hasn't been set up yet #[inline(always)] pub fn get_local<'a>() -> Option<&'a Self> { @@ -63,4 +84,14 @@ impl Cpu { pub fn local_apic(&self) -> &LocalApic { &self.local_apic } + + /// Returns the CPU's execution queue + pub fn queue(&self) -> &'static CpuQueue { + self.queue.get() + } + + /// Sets up the local CPU's execution queue + pub fn init_queue(&self, queue: &'static CpuQueue) { + self.queue.init(queue); + } } diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs new file mode 100644 index 00000000..e7736a82 --- /dev/null +++ b/src/arch/x86_64/cpuid.rs @@ -0,0 +1,80 @@ +//! x86-64 CPUID interface +use tock_registers::interfaces::ReadWriteable; + +use crate::arch::x86_64::registers::CR4; + +use super::registers::XCR0; + +unsafe fn cpuid(eax: u32, result: &mut [u32]) { + core::arch::asm!( + r#" + push %rbx + cpuid + mov %ebx, {0:e} + pop %rbx + "#, + out(reg) result[0], + out("edx") result[1], + out("ecx") result[2], + in("eax") eax, + options(att_syntax) + ); +} + +type RequiredBit = (u32, &'static str); + +const EAX1_ECX_REQUIRED_FEATURES: &[RequiredBit] = &[ + (1 << 0, "SSE3"), + (1 << 19, "SSE4.1"), + (1 << 20, "SSE4.2"), + (1 << 24, "TSC"), + (1 << 26, "XSAVE"), + (1 << 28, "AVX"), +]; +const EAX1_EDX_REQUIRED_FEATURES: &[RequiredBit] = &[ + (1 << 0, "FPU"), + (1 << 3, "PSE"), + (1 << 4, "TSC (%edx)"), + (1 << 5, "MSR"), + (1 << 6, "PAE"), + (1 << 9, "APIC"), + (1 << 13, "PGE"), + (1 << 23, "MMX"), + (1 << 24, "FXSR"), + (1 << 25, "SSE"), + (1 << 26, "SSE2"), +]; + +fn enable_cr4_features() { + // TODO maybe also include FSGSBASE here? + CR4.modify(CR4::OSXSAVE::SET + CR4::OSFXSR::SET + CR4::PGE::SET); +} + +fn enable_xcr0_features() { + XCR0.modify(XCR0::X87::SET + XCR0::SSE::SET + XCR0::AVX::SET); +} + +/// Checks for the features required by the kernel and enables them +pub fn feature_gate() { + // TODO the compiler may have generated instructions from SSE/AVX sets by now, find some way to + // perform this as early as possible + let mut data = [0; 3]; + unsafe { + cpuid(1, &mut data); + } + + for (bit, name) in EAX1_ECX_REQUIRED_FEATURES { + if data[2] & bit == 0 { + panic!("Required feature not supported: {}", name); + } + } + for (bit, name) in EAX1_EDX_REQUIRED_FEATURES { + if data[1] & bit == 0 { + panic!("Required feature not supported: {}", name); + } + } + + // Enable the SSE/AVX features + enable_cr4_features(); + enable_xcr0_features(); +} diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index aff1cd7d..aaf4cf5c 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -1,15 +1,20 @@ //! x86-64 exception and interrupt handling use core::{arch::global_asm, mem::size_of_val}; +use tock_registers::interfaces::{ReadWriteable, Writeable}; + use crate::arch::{ - x86_64::apic::{self, __x86_64_apic_irq_handler}, + x86_64::{ + apic::{self, __x86_64_apic_irq_handler}, + registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, + }, Architecture, ArchitectureImpl, }; -/// Set of registers saved when taking an exception/interrupt +/// Set of registers saved by most of the exception/syscall/irq handlers #[derive(Debug)] #[repr(C)] -pub struct ExceptionFrame { +pub struct CommonFrame { rax: usize, rcx: usize, rdx: usize, @@ -25,8 +30,26 @@ pub struct ExceptionFrame { r13: usize, r14: usize, r15: usize, + _0: usize, +} + +/// Set of registers saved when taking an exception/interrupt +#[derive(Debug)] +#[repr(C)] +pub struct ExceptionFrame { + c: CommonFrame, exc_number: usize, exc_code: usize, + rip: usize, + cs: usize, +} + +/// Set of registers saved when taking a syscall instruction +#[derive(Debug)] +#[repr(C)] +pub struct SyscallFrame { + user_sp: usize, + c: CommonFrame, } /// Exception table entry @@ -88,6 +111,7 @@ static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { let frame = unsafe { &*frame }; errorln!("Exception {}", frame.exc_number); + errorln!("At {:#x}:{:#x}", frame.cs, frame.rip); if frame.exc_number == 14 { let cr2: usize; @@ -102,6 +126,11 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { } } +extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { + let frame = unsafe { &*frame }; + debugln!("Syscall {}", frame.c.rax); +} + /// Initializes the interrupt descriptor table for the given CPU. /// /// # Safety @@ -110,6 +139,7 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { pub unsafe fn init_exceptions(_cpu_index: usize) { extern "C" { static __x86_64_exception_vectors: [usize; 32]; + fn __x86_64_syscall_vector(); } for (i, &entry) in __x86_64_exception_vectors.iter().enumerate() { @@ -124,11 +154,24 @@ pub unsafe fn init_exceptions(_cpu_index: usize) { }; core::arch::asm!("wbinvd; lidt ({0})", in(reg) &idtr, options(att_syntax)); + + // Initialize syscall vector + MSR_IA32_LSTAR.set(__x86_64_syscall_vector as u64); + MSR_IA32_SFMASK.write(MSR_IA32_SFMASK::IF::Masked); + MSR_IA32_STAR.write( + // On sysret, CS = val + 16 (0x23), SS = val + 8 (0x1B) + MSR_IA32_STAR::SYSRET_CS_SS.val(0x1B - 8) + + // On syscall, CS = val (0x08), SS = val + 8 (0x10) + MSR_IA32_STAR::SYSCALL_CS_SS.val(0x08), + ); + + MSR_IA32_EFER.modify(MSR_IA32_EFER::SCE::Enable); } global_asm!( include_str!("vectors.S"), exception_handler = sym __x86_64_exception_handler, apic_irq_handler = sym __x86_64_apic_irq_handler, + syscall_handler = sym __x86_64_syscall_handler, options(att_syntax) ); diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 7978e59c..7eab6b35 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,6 +1,6 @@ //! x86-64 Global Descriptor Table interface // TODO TSS -use core::mem::size_of_val; +use core::mem::{size_of, size_of_val}; #[allow(dead_code)] #[repr(packed)] @@ -13,6 +13,26 @@ struct Entry { base_hi: u8, } +#[allow(dead_code)] +#[repr(packed)] +struct Tss { + _0: u32, + rsp0: u64, + rsp1: u64, + rsp2: u64, + _1: u32, + ist1: u64, + ist2: u64, + ist3: u64, + ist4: u64, + ist5: u64, + ist6: u64, + ist7: u64, + _2: u64, + _3: u16, + iopb_base: u16, +} + #[allow(dead_code)] #[repr(packed)] struct Pointer { @@ -20,6 +40,26 @@ struct Pointer { offset: usize, } +impl Tss { + const NULL: Self = Self { + _0: 0, + rsp0: 0, + rsp1: 0, + rsp2: 0, + _1: 0, + ist1: 0, + ist2: 0, + ist3: 0, + ist4: 0, + ist5: 0, + ist6: 0, + ist7: 0, + _2: 0, + _3: 0, + iopb_base: size_of::() as u16, + }; +} + impl Entry { const FLAG_LONG: u8 = 1 << 5; const ACC_PRESENT: u8 = 1 << 7; @@ -53,9 +93,12 @@ impl Entry { } } -const SIZE: usize = 3; +const SIZE: usize = 7; // TODO per-CPU +#[no_mangle] +static mut TSS: Tss = Tss::NULL; + static mut GDT: [Entry; SIZE] = [ // 0x00, NULL Entry::NULL, @@ -73,6 +116,23 @@ static mut GDT: [Entry; SIZE] = [ 0, Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_WRITE, ), + // 0x18 (0x1B), Ring3 DS64 + Entry::new( + 0, + 0, + 0, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_RING3 | Entry::ACC_WRITE, + ), + // 0x20 (0x23), Ring3 CS64 + Entry::new( + 0, + 0, + Entry::FLAG_LONG, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_RING3 | Entry::ACC_EXECUTE, + ), + // 0x28, TSS + Entry::NULL, + Entry::NULL, ]; /// Initializes the global descriptor table. @@ -80,7 +140,18 @@ static mut GDT: [Entry; SIZE] = [ /// # Safety /// /// Only meant to be called by the CPUs during their early init. -pub unsafe fn init() { +pub unsafe fn init(_id: u32) -> usize { + let tss_addr = &TSS as *const _ as usize; + + GDT[5] = Entry::new( + tss_addr as u32, + size_of_val(&TSS) as u32 - 1, + Entry::FLAG_LONG, + Entry::ACC_ACCESS | Entry::ACC_PRESENT | Entry::ACC_EXECUTE, + ); + let tss_upper = &mut GDT[6] as *mut _ as *mut u64; + tss_upper.write_unaligned((tss_addr >> 32) as u64); + let gdtr = Pointer { limit: size_of_val(&GDT) as u16 - 1, offset: &GDT as *const _ as usize, @@ -113,9 +184,14 @@ pub unsafe fn init() { mov %ax, %fs mov %ax, %gs mov %ax, %ss + + mov $0x28, %ax + ltr %ax "#, in(reg) &gdtr, out("rax") _, options(att_syntax) ); + + tss_addr } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index ce4e913a..212caaf0 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -34,7 +34,9 @@ pub mod intrinsics; pub mod apic; pub mod boot; +pub mod context; pub mod cpu; +pub mod cpuid; pub mod exception; pub mod gdt; pub mod peripherals; @@ -160,7 +162,7 @@ impl Architecture for X86_64 { fn interrupt_mask() -> bool { let mut flags: u64; unsafe { - core::arch::asm!("pushfd; pop {0}", out(reg) flags, options(att_syntax)); + core::arch::asm!("pushfq; pop {0}", out(reg) flags, options(att_syntax)); } // If IF is zero, interrupts are disabled (masked) flags & (1 << 9) == 0 diff --git a/src/arch/x86_64/registers/mod.rs b/src/arch/x86_64/registers/mod.rs index fb6d6b48..a1d063c2 100644 --- a/src/arch/x86_64/registers/mod.rs +++ b/src/arch/x86_64/registers/mod.rs @@ -98,5 +98,206 @@ mod msr_ia32_apic_base { pub const MSR_IA32_APIC_BASE: Reg = Reg; } +mod msr_ia32_sfmask { + use tock_registers::register_bitfields; + + register_bitfields! { + u64, + #[allow(missing_docs)] + pub MSR_IA32_SFMASK [ + IF OFFSET(9) NUMBITS(1) [ + Masked = 1, + Unmasked = 0 + ] + ] + } + + const ADDR: u32 = 0xC0000084; + pub struct Reg; + + msr_impl_read!(Reg, ADDR, MSR_IA32_SFMASK::Register); + msr_impl_write!(Reg, ADDR, MSR_IA32_SFMASK::Register); + + /// IA32_SFMASK model-specific register + pub const MSR_IA32_SFMASK: Reg = Reg; +} + +mod msr_ia32_star { + use tock_registers::register_bitfields; + + register_bitfields! { + u64, + #[allow(missing_docs)] + pub MSR_IA32_STAR [ + SYSCALL_CS_SS OFFSET(32) NUMBITS(16) [], + SYSRET_CS_SS OFFSET(48) NUMBITS(16) [], + ] + } + + const ADDR: u32 = 0xC0000081; + pub struct Reg; + + msr_impl_read!(Reg, ADDR, MSR_IA32_STAR::Register); + msr_impl_write!(Reg, ADDR, MSR_IA32_STAR::Register); + + /// IA32_STAR model-specific register + pub const MSR_IA32_STAR: Reg = Reg; +} + +mod msr_ia32_lstar { + const ADDR: u32 = 0xC0000082; + pub struct Reg; + + msr_impl_read!(Reg, ADDR); + msr_impl_write!(Reg, ADDR); + + /// IA32_LSTAR model-specific register + pub const MSR_IA32_LSTAR: Reg = Reg; +} + +mod msr_ia32_efer { + use tock_registers::register_bitfields; + + register_bitfields! { + u64, + #[allow(missing_docs)] + pub MSR_IA32_EFER [ + // If set, support for SYSCALL/SYSRET instructions is enabled + SCE OFFSET(0) NUMBITS(1) [ + Enable = 1, + Disable = 0 + ] + ] + } + + const ADDR: u32 = 0xC0000080; + pub struct Reg; + + msr_impl_read!(Reg, ADDR, MSR_IA32_EFER::Register); + msr_impl_write!(Reg, ADDR, MSR_IA32_EFER::Register); + + /// IA32_EFER Extended Feature Enable model-specific Register + pub const MSR_IA32_EFER: Reg = Reg; +} + +mod cr4 { + use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, + }; + + register_bitfields! { + u64, + #[allow(missing_docs)] + pub CR4 [ + /// If set, XSAVE and extended processor states are enabled + OSXSAVE OFFSET(18) NUMBITS(1) [], + /// Indicates OS support for FXSAVE and FXRSTOR instructions + OSFXSR OFFSET(9) NUMBITS(1) [], + /// If set, "page global" attribute is enabled + PGE OFFSET(8) NUMBITS(1) [], + ] + } + + pub struct Reg; + + impl Readable for Reg { + type T = u64; + type R = CR4::Register; + + fn get(&self) -> Self::T { + let value: u64; + unsafe { + core::arch::asm!("mov %cr4, {0}", out(reg) value, options(att_syntax)); + } + value + } + } + + impl Writeable for Reg { + type T = u64; + type R = CR4::Register; + + fn set(&self, value: Self::T) { + unsafe { + core::arch::asm!("mov {0}, %cr4", in(reg) value, options(att_syntax)); + } + } + } + + /// x86-64 control register 4 + pub const CR4: Reg = Reg; +} + +mod xcr0 { + use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, + }; + + register_bitfields! { + u64, + #[allow(missing_docs)] + pub XCR0 [ + /// If set, x87 FPU/MMX is enabled + X87 OFFSET(0) NUMBITS(1) [], + /// If set, XSAVE support for MXCSR and XMM registers is enabled + SSE OFFSET(1) NUMBITS(1) [], + /// If set, AVX is enabled and XSAVE supports YMM upper halves + AVX OFFSET(2) NUMBITS(1) [], + ] + } + + pub struct Reg; + + impl Readable for Reg { + type T = u64; + type R = XCR0::Register; + + fn get(&self) -> Self::T { + let eax: u32; + let edx: u32; + unsafe { + core::arch::asm!( + "xgetbv", + in("ecx") 0, + out("eax") eax, + out("edx") edx, + options(att_syntax) + ); + } + ((edx as u64) << 32) | (eax as u64) + } + } + + impl Writeable for Reg { + type T = u64; + type R = XCR0::Register; + + fn set(&self, value: Self::T) { + let eax = value as u32; + let edx = (value >> 32) as u32; + unsafe { + core::arch::asm!( + "xsetbv", + in("ecx") 0, + in("eax") eax, + in("edx") edx, + options(att_syntax) + ); + } + } + } + + /// Extended control register for SSE/AVX/FPU configuration + pub const XCR0: Reg = Reg; +} + +pub use cr4::CR4; pub use msr_ia32_apic_base::MSR_IA32_APIC_BASE; +pub use msr_ia32_efer::MSR_IA32_EFER; pub use msr_ia32_kernel_gs_base::MSR_IA32_KERNEL_GS_BASE; +pub use msr_ia32_lstar::MSR_IA32_LSTAR; +pub use msr_ia32_sfmask::MSR_IA32_SFMASK; +pub use msr_ia32_star::MSR_IA32_STAR; +pub use xcr0::XCR0; diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs index 2ab6e03f..ef6a59dd 100644 --- a/src/arch/x86_64/table/fixed.rs +++ b/src/arch/x86_64/table/fixed.rs @@ -91,6 +91,10 @@ impl FixedTables { pub fn physical_address(&self) -> usize { self.l0.physical_address() } + + pub fn clone_into(&self, target: &mut PageTable) { + target[511] = self.l0[511]; + } } /// Instance of fixed translation tables diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/table/mod.rs index 1200bc40..3265c756 100644 --- a/src/arch/x86_64/table/mod.rs +++ b/src/arch/x86_64/table/mod.rs @@ -5,6 +5,7 @@ use core::{ ops::{Index, IndexMut}, }; +use abi::error::Error; use bitflags::bitflags; mod fixed; @@ -12,6 +13,7 @@ mod fixed; pub use fixed::{init_fixed_tables, KERNEL_TABLES}; use crate::mem::{ + phys::{self, PageUsage}, table::{EntryLevel, NonTerminalEntryLevel}, ConvertAddress, }; @@ -27,9 +29,19 @@ bitflags! { /// When set for L2 entries, the mapping specifies a 2MiB page instead of a page table /// reference const BLOCK = 1 << 7; + /// For tables, allows user access to further translation levels, for pages/blocks, allows + /// user access to the region covered by the entry + const USER = 1 << 2; } } +/// Represents a process or kernel address space. Because x86-64 does not have cool stuff like +/// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space. +#[repr(C)] +pub struct AddressSpace { + l0: *mut PageTable, +} + /// Represents a single virtual address space mapping depending on its translation level #[derive(Clone, Copy)] #[repr(transparent)] @@ -109,7 +121,7 @@ impl PageEntry { /// Constructs a mapping which points to a 4KiB page pub fn page(phys: usize, attrs: PageAttributes) -> Self { Self( - (phys as u64) | (attrs | PageAttributes::PRESENT).bits(), + (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::USER).bits(), PhantomData, ) } @@ -119,7 +131,9 @@ impl PageEntry { /// Constructs a mapping which points to a 2MiB block pub fn block(phys: usize, attrs: PageAttributes) -> Self { Self( - (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), + (phys as u64) + | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK | PageAttributes::USER) + .bits(), PhantomData, ) } @@ -129,7 +143,12 @@ impl PageEntry { /// Constructs a mapping which points to a next-level table pub fn table(phys: usize, attrs: PageAttributes) -> Self { Self( - (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::WRITABLE).bits(), + (phys as u64) + | (attrs + | PageAttributes::PRESENT + | PageAttributes::WRITABLE + | PageAttributes::USER) + .bits(), PhantomData, ) } @@ -167,3 +186,26 @@ impl IndexMut for PageTable { &mut self.data[index] } } + +impl AddressSpace { + /// Allocates an empty address space with all entries marked as non-present + pub fn new_empty() -> Result { + let l0 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable }; + + for i in 0..512 { + unsafe { + (*l0)[i] = PageEntry::INVALID; + } + } + unsafe { + KERNEL_TABLES.clone_into(&mut *l0); + } + + Ok(Self { l0 }) + } + + /// Returns the physical address of the root table + pub fn physical_address(&self) -> usize { + unsafe { (self.l0 as usize).physicalize() } + } +} diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index e176efb0..1285a9f2 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -1,39 +1,5 @@ -.macro ISR_NERR, n -__x86_64_exc_\n: - cli - pushq $0 - pushq $\n - jmp __x86_64_exc_common -.endm - -.macro ISR_YERR, n -__x86_64_exc_\n: - cli - pushq $\n - jmp __x86_64_exc_common -.endm - -.macro apic_vector, n -__x86_64_apic_irq_\n: - cli - - // Push dummy error codes - // pushq $0 - // pushq $0 - - EXC_SAVE_STATE - mov $\n, %rdi - mov %rsp, %rsi - call {apic_irq_handler} - EXC_RESTORE_STATE - - // Remove dummy error codes - // add $16, %rsp - - iretq -.endm - -// 16 general-purpose registers +// vi: ft=asm : +// 15 general-purpose registers .set PT_REGS_SIZE, 15 * 8 .macro EXC_SAVE_STATE @@ -77,8 +43,49 @@ __x86_64_apic_irq_\n: addq $PT_REGS_SIZE, %rsp .endm +.macro ISR_NERR, n +__x86_64_exc_\n: + cli + pushq $0 + pushq $\n + jmp __x86_64_exc_common +.endm + +.macro ISR_YERR, n +__x86_64_exc_\n: + cli + pushq $\n + jmp __x86_64_exc_common +.endm + +.macro apic_vector, n +__x86_64_apic_irq_\n: + cli + + // Push dummy error codes + // pushq $0 + // pushq $0 + + EXC_SAVE_STATE + + mov $0x10, %ax + mov %ax, %ss + + mov $\n, %rdi + mov %rsp, %rsi + call {apic_irq_handler} + + EXC_RESTORE_STATE + + // Remove dummy error codes + // add $16, %rsp + + iretq +.endm + .global __x86_64_exception_vectors .global __x86_64_apic_vectors +.global __x86_64_syscall_vector .section .text __x86_64_exc_common: @@ -142,6 +149,38 @@ apic_vector 13 apic_vector 14 apic_vector 15 +__x86_64_syscall_vector: + // On entry: + // %rcx - userspace %rip + // %r11 - rflags + + // Store user RSP + // TODO: eliminate magic %gs-relative addresses + mov %rsp, %gs:(16) + // Load the task's RSP0 from TSS + mov %gs:(8), %rsp + mov 4(%rsp), %rsp + + // Store the state + EXC_SAVE_STATE + // Store user stack pointer + mov %gs:(16), %rax + pushq %rax + + mov %rsp, %rdi + call {syscall_handler} + + // Restore user stack pointer + popq %rax + mov %rax, %gs:(16) + // Restore the state + EXC_RESTORE_STATE + + // %rcx and %r11 now contain the expected values + // Restore user RSP + mov %gs:(16), %rsp + + sysretq .section .rodata .global __x86_64_exception_vectors diff --git a/src/main.rs b/src/main.rs index a1f76675..08986974 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ const_mut_refs, let_chains )] -#![allow(clippy::new_without_default)] +#![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] #![warn(missing_docs)] #![no_std] #![no_main] @@ -41,7 +41,7 @@ pub mod panic; // pub mod proc; pub mod sync; // pub mod syscall; -// pub mod task; +pub mod task; pub mod util; // // fn setup_root() -> Result { diff --git a/src/mem/table.rs b/src/mem/table.rs index 317b4262..4e57cbd3 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -6,7 +6,7 @@ cfg_if! { if #[cfg(target_arch = "aarch64")] { pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::table::{PageAttributes}; + pub use crate::arch::x86_64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; } } diff --git a/src/sync.rs b/src/sync.rs index baf080c9..d32b31cd 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -5,6 +5,8 @@ use core::{ sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; +use crate::arch::{Architecture, ArchitectureImpl}; + /// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is /// met. pub struct SpinFence { @@ -13,7 +15,7 @@ pub struct SpinFence { /// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation /// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. -pub struct IrqGuard(u64); +pub struct IrqGuard(bool); struct SpinlockInner { value: UnsafeCell, @@ -129,16 +131,19 @@ impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { impl IrqGuard { /// Saves the current IRQ state and masks them pub fn acquire() -> Self { - Self(0) - // let this = Self(DAIF.get()); - // DAIF.modify(DAIF::I::SET); - // this + let mask = ArchitectureImpl::interrupt_mask(); + unsafe { + ArchitectureImpl::set_interrupt_mask(true); + } + Self(mask) } } impl Drop for IrqGuard { fn drop(&mut self) { - // DAIF.set(self.0); + unsafe { + ArchitectureImpl::set_interrupt_mask(self.0); + } } } diff --git a/src/task/mod.rs b/src/task/mod.rs index 3b7c0985..0756e545 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -1,14 +1,27 @@ //! Multitasking and process/thread management interfaces -use core::sync::atomic::Ordering; -use aarch64_cpu::registers::MPIDR_EL1; +#![allow(dead_code)] + use abi::error::Error; use alloc::{rc::Rc, vec::Vec}; -use tock_registers::interfaces::Readable; +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub use crate::arch::aarch64::{context::TaskContext, cpu::Cpu}; + } else if #[cfg(target_arch = "x86_64")] { + pub use crate::arch::x86_64::{context::TaskContext, cpu::Cpu}; + } +} use crate::{ - arch::aarch64::{context::TaskContext, cpu::Cpu, smp::CPU_COUNT}, - kernel_main, + mem::{ + phys::{self, PageUsage}, + table::AddressSpace, + ConvertAddress, + }, + //arch::aarch64::{context::TaskContext, cpu::Cpu, smp::CPU_COUNT}, + // kernel_main, sync::{IrqSafeSpinlock, SpinFence}, task::sched::CpuQueue, }; @@ -69,15 +82,69 @@ pub fn spawn_kernel_closure(f: F) -> Result<(), Error> Ok(()) } +unsafe fn spawn_user_in_kernel(entry: usize, arg: usize) -> Result<(), Error> { + let stack = phys::alloc_pages_contiguous(4, PageUsage::Used)?; + let stack_pointer = stack + 4 * 0x1000; + + let space = AddressSpace::new_empty()?; + + let user_ctx = TaskContext::user( + entry, + arg, + space.physical_address(), + stack_pointer.virtualize(), + )?; + let proc = Process::new_with_context(Some(space), user_ctx); + + proc.enqueue_somewhere(); + + Ok(()) +} + +extern "C" fn f1(arg: usize) -> ! { + loop { + unsafe { + core::arch::asm!("syscall", in("rax") arg, options(att_syntax), clobber_abi("C")); + } + + for _ in 0..100000 { + unsafe { + core::arch::asm!("nop"); + } + } + } +} + /// Sets up CPU queues and gives them some processes to run pub fn init() -> Result<(), Error> { - let cpu_count = CPU_COUNT.load(Ordering::Acquire); + // XXX + // let cpu_count = CPU_COUNT.load(Ordering::Acquire); + let cpu_count = 1; // Create a queue for each CPU sched::init_queues(Vec::from_iter((0..cpu_count).map(|_| CpuQueue::new()))); // Spawn kernel main task - spawn_kernel_closure(kernel_main)?; + unsafe { + spawn_user_in_kernel(f1 as usize, 0).unwrap(); + } + unsafe { + spawn_user_in_kernel(f1 as usize, 1).unwrap(); + } + + // spawn_kernel_closure(move || loop { + // debugln!("A"); + // for _ in 0..100000 { + // core::hint::spin_loop(); + // } + // })?; + + // spawn_kernel_closure(move || loop { + // debugln!("B"); + // for _ in 0..100000 { + // core::hint::spin_loop(); + // } + // })?; Ok(()) } @@ -94,7 +161,7 @@ pub fn init() -> Result<(), Error> { pub unsafe fn enter() -> ! { static AP_CAN_ENTER: SpinFence = SpinFence::new(); - let cpu_id = MPIDR_EL1.get() & 0xF; + let cpu_id = Cpu::local_id(); if cpu_id != 0 { // Wait until BSP allows us to enter diff --git a/src/task/process.rs b/src/task/process.rs index f6417b33..95a53ed8 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,32 +1,26 @@ //! Process data structures use core::{ - mem::size_of, ops::Deref, sync::atomic::{AtomicU32, Ordering}, }; -use aarch64_cpu::registers::DAIF; -use abi::{ - error::Error, - process::{ExitCode, Signal, SignalEntryData}, -}; -use alloc::{collections::VecDeque, rc::Rc}; +// use aarch64_cpu::registers::DAIF; +use alloc::rc::Rc; use atomic_enum::atomic_enum; -use tock_registers::interfaces::Readable; -use vfs::VnodeRef; use crate::{ - arch::aarch64::{context::TaskContext, cpu::Cpu, exception::ExceptionFrame}, - mem::{table::AddressSpace, ForeignPointer}, - proc::{ - io::ProcessIo, - wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, - }, + mem::table::AddressSpace, + // arch::aarch64::{context::TaskContext, cpu::Cpu, exception::ExceptionFrame}, + // mem::{table::AddressSpace, ForeignPointer}, + // proc::{ + // io::ProcessIo, + // wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, + // }, sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; -use super::{sched::CpuQueue, ProcessId, PROCESSES}; +use super::{sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; /// Represents the states a process can be at some point in time #[atomic_enum] @@ -50,16 +44,17 @@ pub struct SignalEntry { } struct ProcessInner { - pending_wait: Option<&'static Wait>, - wait_status: WaitStatus, - exit_status: i32, + // XXX + // pending_wait: Option<&'static Wait>, + // wait_status: WaitStatus, + // exit_status: i32, - session_id: ProcessId, - group_id: ProcessId, - session_terminal: Option, + // session_id: ProcessId, + // group_id: ProcessId, + // session_terminal: Option, - signal_entry: Option, - signal_stack: VecDeque, + // signal_entry: Option, + // signal_stack: VecDeque, } /// Process data and state structure @@ -71,11 +66,9 @@ pub struct Process { state: AtomicProcessState, cpu_id: AtomicU32, space: Option, - inner: IrqSafeSpinlock, - - /// I/O state of the task - pub io: IrqSafeSpinlock, + // /// I/O state of the task + // pub io: IrqSafeSpinlock, } /// Guard type that provides [Process] operations only available for current processes @@ -97,29 +90,30 @@ impl Process { space, inner: IrqSafeSpinlock::new(ProcessInner { - pending_wait: None, - wait_status: WaitStatus::Done, - exit_status: 0, + // XXX + // pending_wait: None, + // wait_status: WaitStatus::Done, + // exit_status: 0, - session_id: 0, - group_id: 0, - session_terminal: None, + // session_id: 0, + // group_id: 0, + // session_terminal: None, - signal_entry: None, - signal_stack: VecDeque::new(), + // signal_entry: None, + // signal_stack: VecDeque::new(), }), - - io: IrqSafeSpinlock::new(ProcessIo::new()), + // XXX + // io: IrqSafeSpinlock::new(ProcessIo::new()), }); let id = unsafe { PROCESSES.lock().push(this.clone()) }; this.id.init(id); - { - let mut inner = this.inner.lock(); - inner.session_id = id; - inner.group_id = id; - } + // { + // let mut inner = this.inner.lock(); + // inner.session_id = id; + // inner.group_id = id; + // } this } @@ -168,35 +162,36 @@ impl Process { self.space.as_ref() } - /// Replaces the task's session terminal device with another one - pub fn set_session_terminal(&self, terminal: VnodeRef) { - self.inner.lock().session_terminal.replace(terminal); - } + // XXX + // /// Replaces the task's session terminal device with another one + // pub fn set_session_terminal(&self, terminal: VnodeRef) { + // self.inner.lock().session_terminal.replace(terminal); + // } - /// Removes the task's current terminal - pub fn clear_session_terminal(&self) -> Option { - self.inner.lock().session_terminal.take() - } + // /// Removes the task's current terminal + // pub fn clear_session_terminal(&self) -> Option { + // self.inner.lock().session_terminal.take() + // } - /// Returns the current terminal of the task - pub fn session_terminal(&self) -> Option { - self.inner.lock().session_terminal.clone() - } + // /// Returns the current terminal of the task + // pub fn session_terminal(&self) -> Option { + // self.inner.lock().session_terminal.clone() + // } - /// Sets the session ID of the task - pub fn set_session_id(&self, sid: ProcessId) { - self.inner.lock().session_id = sid; - } + // /// Sets the session ID of the task + // pub fn set_session_id(&self, sid: ProcessId) { + // self.inner.lock().session_id = sid; + // } - /// Sets the process group ID of the task - pub fn set_group_id(&self, gid: ProcessId) { - self.inner.lock().group_id = gid; - } + // /// Sets the process group ID of the task + // pub fn set_group_id(&self, gid: ProcessId) { + // self.inner.lock().group_id = gid; + // } - /// Returns the process group ID of the task - pub fn group_id(&self) -> ProcessId { - self.inner.lock().group_id - } + // /// Returns the process group ID of the task + // pub fn group_id(&self) -> ProcessId { + // self.inner.lock().group_id + // } /// Selects a suitable CPU queue and submits the process for execution. /// @@ -249,43 +244,44 @@ impl Process { ProcessState::Suspended => (), ProcessState::Terminated => panic!("Process is terminated"), ProcessState::Running => { - let cpu_id = self.cpu_id.load(Ordering::Acquire); - let local_cpu_id = Cpu::local_id(); - let queue = Cpu::local().queue(); + todo!(); + // let cpu_id = self.cpu_id.load(Ordering::Acquire); + // let local_cpu_id = Cpu::local_id(); + // let queue = Cpu::local().queue(); - if cpu_id == local_cpu_id { - // Suspending a process running on local CPU - unsafe { queue.yield_cpu() } - } else { - todo!(); - } + // if cpu_id == local_cpu_id { + // // Suspending a process running on local CPU + // unsafe { queue.yield_cpu() } + // } else { + // todo!(); + // } } } } - /// Returns current wait status of the task - pub fn wait_status(&self) -> WaitStatus { - self.inner.lock().wait_status - } + // /// Returns current wait status of the task + // pub fn wait_status(&self) -> WaitStatus { + // self.inner.lock().wait_status + // } - /// Updates the wait status for the task. - /// - /// # Safety - /// - /// This function is only meant to be called on waiting tasks, otherwise atomicity is not - /// guaranteed. - pub unsafe fn set_wait_status(&self, status: WaitStatus) { - self.inner.lock().wait_status = status; - } + // /// Updates the wait status for the task. + // /// + // /// # Safety + // /// + // /// This function is only meant to be called on waiting tasks, otherwise atomicity is not + // /// guaranteed. + // pub unsafe fn set_wait_status(&self, status: WaitStatus) { + // self.inner.lock().wait_status = status; + // } - /// Returns an exit code if the process exited, [None] if it didn't - pub fn get_exit_status(&self) -> Option { - if self.state() == ProcessState::Terminated { - Some(ExitCode::from(self.inner.lock().exit_status)) - } else { - None - } - } + // /// Returns an exit code if the process exited, [None] if it didn't + // pub fn get_exit_status(&self) -> Option { + // if self.state() == ProcessState::Terminated { + // Some(ExitCode::from(self.inner.lock().exit_status)) + // } else { + // None + // } + // } /// Returns the [Process] currently executing on local CPU, None if idling. pub fn get_current() -> Option { @@ -304,71 +300,71 @@ impl Process { Self::get_current().unwrap() } - /// Handles the cleanup of an exited process - pub fn handle_exit(&self) { - // Queue lock is still held - assert_eq!(self.state(), ProcessState::Terminated); + // /// Handles the cleanup of an exited process + // pub fn handle_exit(&self) { + // // Queue lock is still held + // assert_eq!(self.state(), ProcessState::Terminated); - // TODO cancel Wait if a process was killed while suspended? - { - let inner = self.inner.lock(); - let exit_status = ExitCode::from(inner.exit_status); - debugln!( - "Handling exit of #{} with status {:?}", - self.id(), - exit_status - ); + // // TODO cancel Wait if a process was killed while suspended? + // { + // let inner = self.inner.lock(); + // let exit_status = ExitCode::from(inner.exit_status); + // debugln!( + // "Handling exit of #{} with status {:?}", + // self.id(), + // exit_status + // ); - // TODO cleanup address space - // if let Some(space) = self.get_address_space() { - // } + // // TODO cleanup address space + // // if let Some(space) = self.get_address_space() { + // // } - self.io.lock().handle_exit(); - } + // self.io.lock().handle_exit(); + // } - // Notify any waiters we're done - PROCESS_EXIT_WAIT.wakeup_all(); - } + // // Notify any waiters we're done + // PROCESS_EXIT_WAIT.wakeup_all(); + // } - /// Raises an asynchronous signal for the target process - pub fn try_set_signal(self: &Rc, signal: Signal) -> Result<(), Error> { - { - let mut inner = self.inner.lock(); - inner.wait_status = WaitStatus::Interrupted; - inner.signal_stack.push_back(signal); - } + // /// Raises an asynchronous signal for the target process + // pub fn try_set_signal(self: &Rc, signal: Signal) -> Result<(), Error> { + // { + // let mut inner = self.inner.lock(); + // inner.wait_status = WaitStatus::Interrupted; + // inner.signal_stack.push_back(signal); + // } - if self.state() == ProcessState::Suspended { - self.clone().enqueue_somewhere(); - } + // if self.state() == ProcessState::Suspended { + // self.clone().enqueue_somewhere(); + // } - Ok(()) - } + // Ok(()) + // } - /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. - pub fn inherit(&self, parent: &Rc) -> Result<(), Error> { - let mut our_inner = self.inner.lock(); - let their_inner = parent.inner.lock(); + // /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. + // pub fn inherit(&self, parent: &Rc) -> Result<(), Error> { + // let mut our_inner = self.inner.lock(); + // let their_inner = parent.inner.lock(); - our_inner.session_id = their_inner.session_id; - our_inner.group_id = their_inner.group_id; - our_inner.session_terminal = their_inner.session_terminal.clone(); + // our_inner.session_id = their_inner.session_id; + // our_inner.group_id = their_inner.group_id; + // our_inner.session_terminal = their_inner.session_terminal.clone(); - Ok(()) - } + // Ok(()) + // } - /// Raises a signal for the specified process group - pub fn signal_group(group_id: ProcessId, signal: Signal) { - let processes = PROCESSES.lock(); - for (_, proc) in processes.data.iter() { - let inner = proc.inner.lock(); - if proc.state() != ProcessState::Terminated && inner.group_id == group_id { - debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); - drop(inner); - proc.try_set_signal(signal).unwrap(); - } - } - } + // /// Raises a signal for the specified process group + // pub fn signal_group(group_id: ProcessId, signal: Signal) { + // let processes = PROCESSES.lock(); + // for (_, proc) in processes.data.iter() { + // let inner = proc.inner.lock(); + // if proc.state() != ProcessState::Terminated && inner.group_id == group_id { + // debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); + // drop(inner); + // proc.try_set_signal(signal).unwrap(); + // } + // } + // } } impl Drop for Process { @@ -384,98 +380,99 @@ impl CurrentProcess { /// /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. pub unsafe fn new(inner: Rc) -> Self { - assert_eq!(DAIF.read(DAIF::I), 1); + // XXX + // assert_eq!(DAIF.read(DAIF::I), 1); Self(inner) } - /// Sets up a pending wait for the process. - /// - /// # Safety - /// - /// This function is only meant to be called in no-IRQ context and when caller can guarantee - /// the task won't get scheduled to a CPU in such state. - pub unsafe fn setup_wait(&self, wait: &'static Wait) { - let mut inner = self.inner.lock(); - inner.pending_wait.replace(wait); - inner.wait_status = WaitStatus::Pending; - } + // /// Sets up a pending wait for the process. + // /// + // /// # Safety + // /// + // /// This function is only meant to be called in no-IRQ context and when caller can guarantee + // /// the task won't get scheduled to a CPU in such state. + // pub unsafe fn setup_wait(&self, wait: &'static Wait) { + // let mut inner = self.inner.lock(); + // inner.pending_wait.replace(wait); + // inner.wait_status = WaitStatus::Pending; + // } - /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. - /// - /// # Safety - /// - /// This function is only meant to be called right before returning from an userspace - /// exception handler. - pub unsafe fn handle_signal(&self, frame: &mut ExceptionFrame) { - let mut inner = self.inner.lock(); - if let Some(signal) = inner.signal_stack.pop_front() { - let Some(entry) = inner.signal_entry.clone() else { - todo!(); - }; + // /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. + // /// + // /// # Safety + // /// + // /// This function is only meant to be called right before returning from an userspace + // /// exception handler. + // pub unsafe fn handle_signal(&self, frame: &mut ExceptionFrame) { + // let mut inner = self.inner.lock(); + // if let Some(signal) = inner.signal_stack.pop_front() { + // let Some(entry) = inner.signal_entry.clone() else { + // todo!(); + // }; - debugln!( - "Enter signal handler from: pc={:#x}, sp={:#x}", - frame.elr_el1, - frame.sp_el0 - ); + // debugln!( + // "Enter signal handler from: pc={:#x}, sp={:#x}", + // frame.elr_el1, + // frame.sp_el0 + // ); - // TODO check if really in a syscall, lol - let syscall_return = -(u32::from(Error::Interrupted) as isize); - frame.r[0] = syscall_return as u64; + // // TODO check if really in a syscall, lol + // let syscall_return = -(u32::from(Error::Interrupted) as isize); + // frame.r[0] = syscall_return as u64; - // Setup signal frame - let usp = (entry.stack - size_of::()) & !0xF; - let frame_ptr = usp as *mut SignalEntryData; + // // Setup signal frame + // let usp = (entry.stack - size_of::()) & !0xF; + // let frame_ptr = usp as *mut SignalEntryData; - let saved_frame = frame.to_saved_frame(); - frame_ptr.write_foreign_volatile( - self.address_space(), - SignalEntryData { - signal, - frame: saved_frame, - }, - ); + // let saved_frame = frame.to_saved_frame(); + // frame_ptr.write_foreign_volatile( + // self.address_space(), + // SignalEntryData { + // signal, + // frame: saved_frame, + // }, + // ); - // Setup return to signal handler - debugln!( - "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", - entry.entry, - usp, - entry.stack - ); - frame.sp_el0 = usp as _; - frame.elr_el1 = entry.entry as _; + // // Setup return to signal handler + // debugln!( + // "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", + // entry.entry, + // usp, + // entry.stack + // ); + // frame.sp_el0 = usp as _; + // frame.elr_el1 = entry.entry as _; - // Pass the frame pointer as an argument to signal handler entry - frame.r[0] = usp as _; - } - } + // // Pass the frame pointer as an argument to signal handler entry + // frame.r[0] = usp as _; + // } + // } - /// Configures signal entry information for the process - pub fn set_signal_entry(&self, entry: usize, stack: usize) { - let mut inner = self.inner.lock(); - inner.signal_entry.replace(SignalEntry { entry, stack }); - } + // /// Configures signal entry information for the process + // pub fn set_signal_entry(&self, entry: usize, stack: usize) { + // let mut inner = self.inner.lock(); + // inner.signal_entry.replace(SignalEntry { entry, stack }); + // } - /// Terminate the current process - pub fn exit(&self, status: ExitCode) { - self.inner.lock().exit_status = status.into(); - let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); - assert_eq!(current_state, ProcessState::Running); - infoln!("Process {} exited with code {:?}", self.id(), status); + // /// Terminate the current process + // pub fn exit(&self, status: ExitCode) { + // self.inner.lock().exit_status = status.into(); + // let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); + // assert_eq!(current_state, ProcessState::Running); + // infoln!("Process {} exited with code {:?}", self.id(), status); - match current_state { - ProcessState::Suspended => { - todo!(); - } - ProcessState::Ready => todo!(), - ProcessState::Running => { - self.handle_exit(); - unsafe { Cpu::local().queue().yield_cpu() } - } - ProcessState::Terminated => todo!(), - } - } + // match current_state { + // ProcessState::Suspended => { + // todo!(); + // } + // ProcessState::Ready => todo!(), + // ProcessState::Running => { + // self.handle_exit(); + // unsafe { Cpu::local().queue().yield_cpu() } + // } + // ProcessState::Terminated => todo!(), + // } + // } } impl Deref for CurrentProcess { diff --git a/src/task/sched.rs b/src/task/sched.rs index dbdd26f9..fa277024 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -1,18 +1,18 @@ //! Per-CPU queue implementation -use aarch64_cpu::registers::CNTPCT_EL0; +// use aarch64_cpu::registers::CNTPCT_EL0; use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; -use tock_registers::interfaces::Readable; +use cfg_if::cfg_if; use crate::{ - arch::aarch64::{context::TaskContext, cpu::Cpu}, + // arch::aarch64::{context::TaskContext, cpu::Cpu}, sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, util::OneTimeInit, }; use super::{ process::{CurrentProcess, Process, ProcessState}, - ProcessId, + Cpu, ProcessId, TaskContext, }; /// Per-CPU statistics @@ -49,7 +49,19 @@ static QUEUES: OneTimeInit> = OneTimeInit::new(); #[naked] extern "C" fn __idle(_x: usize) -> ! { unsafe { - core::arch::asm!("1: nop; b 1b", options(noreturn)); + cfg_if! { + if #[cfg(target_arch = "aarch64")] { + core::arch::asm!("1: nop; b 1b", options(noreturn)); + } else if #[cfg(target_arch = "x86_64")] { + core::arch::asm!(r#" + 1: + nop + jmp 1b + "#, options(noreturn, att_syntax)); + } else { + core::arch::asm!("", options(noreturn)); + } + } } } @@ -113,8 +125,8 @@ impl CpuQueue { /// Only meant to be called from [crate::task::enter()] function. pub unsafe fn enter(&self) -> ! { // Start from idle thread to avoid having a Rc stuck here without getting dropped - let t = CNTPCT_EL0.get(); - self.lock().stats.measure_time = t; + // let t = CNTPCT_EL0.get(); + // self.lock().stats.measure_time = t; self.idle.enter() } @@ -127,9 +139,9 @@ impl CpuQueue { pub unsafe fn yield_cpu(&self) { let mut inner = self.inner.lock(); - let t = CNTPCT_EL0.get(); - let delta = t - inner.stats.measure_time; - inner.stats.measure_time = t; + // let t = CNTPCT_EL0.get(); + // let delta = t - inner.stats.measure_time; + // inner.stats.measure_time = t; let current = inner.current.clone(); @@ -139,9 +151,9 @@ impl CpuQueue { } inner.queue.push_back(current.clone()); - inner.stats.cpu_time += delta; + // inner.stats.cpu_time += delta; } else { - inner.stats.idle_time += delta; + // inner.stats.idle_time += delta; } let next = inner.next_ready_task(); From c9af9b143ab81f901b04e3dc5e1a0f15ec369cef Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 2 Aug 2023 19:53:54 +0300 Subject: [PATCH 028/211] x86-64: single-processor userspace multitasking --- Cargo.toml | 4 +- src/arch/x86_64/apic/mod.rs | 19 +- src/arch/x86_64/boot/mod.rs | 37 +- src/arch/x86_64/context.S | 5 + src/arch/x86_64/context.rs | 3 + src/arch/x86_64/cpuid.rs | 2 +- src/arch/x86_64/exception.rs | 342 ++++++++++--- src/arch/x86_64/mod.rs | 31 ++ src/arch/x86_64/peripherals/ps2/codeset.rs | 6 + .../x86_64/peripherals/{ps2.rs => ps2/mod.rs} | 42 +- src/arch/x86_64/syscall.S | 175 +++++++ src/arch/x86_64/syscall.rs | 157 ++++++ src/arch/x86_64/table/mod.rs | 196 +++++++- src/arch/x86_64/vectors.S | 150 +++--- src/debug.rs | 8 +- src/device/input.rs | 5 + src/device/mod.rs | 7 +- src/device/serial/mod.rs | 1 + src/device/tty.rs | 78 ++- src/fs/devfs.rs | 4 + src/main.rs | 145 +++--- src/mem/mod.rs | 376 +++++++------- src/mem/phys/reserved.rs | 2 +- src/mem/table.rs | 16 +- src/proc/elf.rs | 12 +- src/proc/exec.rs | 48 +- src/proc/io.rs | 5 +- src/proc/wait.rs | 72 +-- src/syscall/mod.rs | 150 +++--- src/task/mod.rs | 59 +-- src/task/process.rs | 472 +++++++++--------- src/task/sched.rs | 17 +- 32 files changed, 1820 insertions(+), 826 deletions(-) create mode 100644 src/arch/x86_64/peripherals/ps2/codeset.rs rename src/arch/x86_64/peripherals/{ps2.rs => ps2/mod.rs} (67%) create mode 100644 src/arch/x86_64/syscall.S create mode 100644 src/arch/x86_64/syscall.rs create mode 100644 src/device/input.rs diff --git a/Cargo.toml b/Cargo.toml index e04376ac..b67152c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -# vfs = { path = "lib/vfs" } -# memfs = { path = "lib/memfs" } +vfs = { path = "lib/vfs" } +memfs = { path = "lib/memfs" } atomic_enum = "0.2.0" bitflags = "2.3.3" diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index cc20f8ab..5608cded 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -5,9 +5,10 @@ use abi::error::Error; use crate::{ arch::{x86_64::cpu::Cpu, PLATFORM}, device::interrupt::IrqHandler, + task::process::Process, }; -use super::exception::{self, ExceptionFrame}; +use super::exception::{self, ExceptionFrame, IrqFrame}; pub mod ioapic; pub mod local; @@ -67,8 +68,7 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) { } } -/// Main handler for APIC interrupts -pub extern "C" fn __x86_64_apic_irq_handler(vector: u32, _frame: *mut ExceptionFrame) { +fn apic_irq_inner(vector: u32) { let cpu = Cpu::local(); cpu.local_apic().clear_interrupt(); @@ -81,3 +81,16 @@ pub extern "C" fn __x86_64_apic_irq_handler(vector: u32, _frame: *mut ExceptionF panic!("Got an interrupt on undefined vector: {}", vector); } } + +/// Main handler for APIC interrupts +pub extern "C" fn __x86_64_apic_irq_handler(vector: u32, frame: *mut IrqFrame) { + let frame = unsafe { &mut *frame }; + + apic_irq_inner(vector); + + if let Some(process) = Process::get_current() { + unsafe { + process.handle_signal(frame); + } + } +} diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 4ec5e314..8d2d6e7a 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -9,11 +9,12 @@ use yboot_proto::{ use crate::{ arch::{ - x86_64::{apic::local::LocalApic, cpu::Cpu, cpuid, exception}, + x86_64::{apic::local::LocalApic, cpu::Cpu, cpuid, exception, syscall}, Architecture, ArchitectureImpl, PLATFORM, }, debug, device::platform::Platform, + fs::{devfs, Initrd, INITRD_DATA}, mem::{ heap, phys::{self, PageUsage}, @@ -48,6 +49,8 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { memory_map: MemoryMap { address: 0, len: 0 }, rsdp_address: 0, + initrd_address: 0, + initrd_size: 0, opt_framebuffer: FramebufferOption { req_width: 640, @@ -61,6 +64,34 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { }, }; +fn setup_initrd() { + let initrd_start = unsafe { YBOOT_DATA.initrd_address } as usize; + let initrd_end = initrd_start + unsafe { YBOOT_DATA.initrd_size } as usize; + + if initrd_start == 0 || initrd_end <= initrd_start { + infoln!("No initrd loaded"); + return; + } + + let start_aligned = initrd_start & !0xFFF; + let end_aligned = initrd_end & !0xFFF; + + let data = unsafe { + core::slice::from_raw_parts( + initrd_start.virtualize() as *const _, + initrd_end - initrd_start, + ) + }; + + let initrd = Initrd { + phys_page_start: start_aligned, + phys_page_len: end_aligned - start_aligned, + data, + }; + + INITRD_DATA.init(initrd); +} + unsafe extern "C" fn __x86_64_upper_entry() -> ! { ArchitectureImpl::set_interrupt_mask(true); @@ -89,6 +120,7 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { } // Setup physical memory allocation + setup_initrd(); ArchitectureImpl::init_physical_memory(&YBOOT_DATA.memory_map); // Allocate memory for the kernel heap @@ -98,8 +130,11 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { // Also initializes local APIC Cpu::init_local(LocalApic::new(), 0); + syscall::init_syscall(); exception::init_exceptions(0); + devfs::init(); + PLATFORM.init(true).unwrap(); task::init().expect("Failed to initialize the scheduler"); diff --git a/src/arch/x86_64/context.S b/src/arch/x86_64/context.S index 0f392db4..ffcbe2af 100644 --- a/src/arch/x86_64/context.S +++ b/src/arch/x86_64/context.S @@ -1,3 +1,5 @@ +// vi: set ft=asm : + .macro SAVE_TASK_STATE sub ${context_size}, %rsp @@ -63,6 +65,9 @@ __x86_64_task_enter_kernel: // Entry address popq %rax + // Alignment word + fake return address to terminate "call chain" + pushq $0 + // Enable IRQ in RFLAGS pushfq popq %rdx diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 9af78eb3..f25f17ad 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -84,6 +84,9 @@ impl StackBuilder { } impl TaskContext { + pub const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; + pub const USER_STACK_EXTRA_ALIGN: usize = 8; + /// Constructs a kernel-space task context pub fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 4; diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index e7736a82..160d9174 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -27,7 +27,7 @@ const EAX1_ECX_REQUIRED_FEATURES: &[RequiredBit] = &[ (1 << 0, "SSE3"), (1 << 19, "SSE4.1"), (1 << 20, "SSE4.2"), - (1 << 24, "TSC"), + // (1 << 24, "TSC"), (1 << 26, "XSAVE"), (1 << 28, "AVX"), ]; diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index aaf4cf5c..39c529ce 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -1,55 +1,225 @@ //! x86-64 exception and interrupt handling use core::{arch::global_asm, mem::size_of_val}; +use abi::{ + arch::SavedFrame, + primitive_enum, + process::{Signal, SignalEntryData}, + syscall::SyscallFunction, +}; use tock_registers::interfaces::{ReadWriteable, Writeable}; -use crate::arch::{ - x86_64::{ - apic::{self, __x86_64_apic_irq_handler}, - registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, +use crate::{ + arch::{ + x86_64::{ + apic::{self, __x86_64_apic_irq_handler}, + registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, + }, + Architecture, ArchitectureImpl, + }, + mem::table::AddressSpace, + syscall::raw_syscall_handler, + task::{ + process::{Process, TaskFrame}, + Cpu, }, - Architecture, ArchitectureImpl, }; -/// Set of registers saved by most of the exception/syscall/irq handlers +primitive_enum! { + pub enum ExceptionKind: u64 { + DivisionError = 0, + Debug = 1, + NonMaskableInterrupt = 2, + Breakpoint = 3, + Overflow = 4, + BoundRangeExceeded = 5, + InvalidOpcode = 6, + DeviceNotAvailable = 7, + DoubleFault = 8, + InvalidTss = 10, + SegmentNotPresent = 11, + StackSegmentFault = 12, + GeneralProtectionFault = 13, + PageFault = 14, + FpuException = 16, + AlignmentCheck = 17, + MachineCheck = 18, + SimdFpuException = 19, + VirtualizationException = 20, + ControlProtectionException = 21, + + Unknown = 99, + } +} + +impl ExceptionKind { + pub fn ring3_possible(&self) -> bool { + match self { + Self::DivisionError + | Self::Debug + | Self::Breakpoint + | Self::Overflow + | Self::BoundRangeExceeded + | Self::InvalidOpcode + | Self::GeneralProtectionFault + | Self::PageFault + | Self::FpuException + | Self::AlignmentCheck + | Self::SimdFpuException => true, + _ => false, + } + } +} + +impl TaskFrame for IrqFrame { + fn store(&self) -> SavedFrame { + todo!() + } + + fn restore(&mut self, saved: &SavedFrame) { + todo!() + } + + fn argument(&self) -> u64 { + todo!(); + } + + fn user_ip(&self) -> usize { + todo!() + } + + fn user_sp(&self) -> usize { + todo!() + } + + fn set_argument(&mut self, value: u64) { + self.rdi = value; + } + + fn set_return_value(&mut self, value: u64) { + todo!() + } + + fn set_user_ip(&mut self, value: usize) { + todo!() + } + + fn set_user_sp(&mut self, value: usize) { + todo!() + } +} + +impl TaskFrame for ExceptionFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.rdx, + rbx: self.rbx, + rsi: self.rsi, + rdi: self.rdi, + rbp: self.rbp, + r8: self.r8, + r9: self.r9, + r10: self.r10, + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.rip, + user_sp: self.rsp, + rflags: self.rflags, + } + } + + fn restore(&mut self, saved: &SavedFrame) { + todo!() + } + + fn argument(&self) -> u64 { + 0 + } + + fn user_sp(&self) -> usize { + self.rsp as _ + } + + fn user_ip(&self) -> usize { + self.rip as _ + } + + fn set_user_sp(&mut self, value: usize) { + self.rsp = value as _; + } + + fn set_user_ip(&mut self, value: usize) { + self.rip = value as _; + } + + fn set_return_value(&mut self, _value: u64) { + // Not in syscall, do not overwrite + } + + fn set_argument(&mut self, value: u64) { + self.rdi = value; + } +} + #[derive(Debug)] #[repr(C)] -pub struct CommonFrame { - rax: usize, - rcx: usize, - rdx: usize, - rbx: usize, - rsi: usize, - rdi: usize, - rbp: usize, - r8: usize, - r9: usize, - r10: usize, - r11: usize, - r12: usize, - r13: usize, - r14: usize, - r15: usize, - _0: usize, +pub struct IrqFrame { + rax: u64, + rcx: u64, + rdx: u64, + rbx: u64, + rsi: u64, + rdi: u64, + rbp: u64, + r8: u64, + r9: u64, + r10: u64, + r11: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, + + rip: u64, + cs: u64, + rflags: u64, + rsp: u64, + ss: u64, } /// Set of registers saved when taking an exception/interrupt #[derive(Debug)] #[repr(C)] pub struct ExceptionFrame { - c: CommonFrame, - exc_number: usize, - exc_code: usize, - rip: usize, - cs: usize, -} + rax: u64, + rcx: u64, + rdx: u64, + rbx: u64, + rsi: u64, + rdi: u64, + rbp: u64, + r8: u64, + r9: u64, + r10: u64, + r11: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, -/// Set of registers saved when taking a syscall instruction -#[derive(Debug)] -#[repr(C)] -pub struct SyscallFrame { - user_sp: usize, - c: CommonFrame, + exc_number: u64, + exc_code: u64, + + rip: u64, + cs: u64, + rflags: u64, + rsp: u64, + ss: u64, } /// Exception table entry @@ -108,27 +278,87 @@ impl Entry { static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; -extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { - let frame = unsafe { &*frame }; - errorln!("Exception {}", frame.exc_number); - errorln!("At {:#x}:{:#x}", frame.cs, frame.rip); +fn user_exception_inner(kind: ExceptionKind, frame: &mut ExceptionFrame) { + let process = Process::current(); - if frame.exc_number == 14 { - let cr2: usize; - unsafe { - core::arch::asm!("mov %cr2, {0}", out(reg) cr2, options(att_syntax)); + warnln!("{:?} in #{} {:?}", kind, process.id(), process.name()); + warnln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); + warnln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); + + match kind { + ExceptionKind::PageFault => { + let cr2: usize; + unsafe { + core::arch::asm!("mov %cr2, {0}", out(reg) cr2, options(att_syntax)); + } + + warnln!("CR2 = {:#x}", cr2); + + process.raise_signal(Signal::MemoryAccessViolation); + return; } - errorln!("Page fault at {:#x}", cr2); + ExceptionKind::InvalidOpcode => { + process.raise_signal(Signal::Aborted); + return; + } + _ => todo!("No handler for exception: {:?}", kind), } - loop { - ArchitectureImpl::wait_for_interrupt(); + // errorln!("Exception {}", frame.exc_number); + // errorln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); + // errorln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); + + // if let Some(cpu) = Cpu::get_local() { + // let current = cpu.queue().current_process(); + // if let Some(current) = current { + // errorln!("In process #{} {:?}", current.id(), current.name()); + // } + // } + + // debugln!("Registers:\n{:#x?}", &frame); + + // if frame.exc_number == 6 { + // // Invalid opcode + // warnln!("Invalid opcode"); + // } + + // if frame.exc_number == 14 { + // } + + // loop { + // ArchitectureImpl::wait_for_interrupt(); + // } +} + +fn kernel_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) -> ! { + todo!() +} + +extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { + let frame = unsafe { &mut *frame }; + let kind = ExceptionKind::try_from(frame.exc_number).unwrap_or(ExceptionKind::Unknown); + + if kind.ring3_possible() && frame.cs == 0x23 { + user_exception_inner(kind, frame); + + unsafe { + Process::current().handle_signal(frame); + } + } else { + kernel_exception_inner(kind, frame) } } -extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { - let frame = unsafe { &*frame }; - debugln!("Syscall {}", frame.c.rax); +pub unsafe fn handle_signal_exit(frame: &mut F) { + // TODO validate the argument + let saved_data = &*(frame.argument() as *const SignalEntryData); + infoln!( + "Handling signal exit to ip={:#x}, sp={:#x}", + saved_data.frame.user_ip, + saved_data.frame.user_sp + ); + + frame.restore(&saved_data.frame); } /// Initializes the interrupt descriptor table for the given CPU. @@ -139,7 +369,6 @@ extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { pub unsafe fn init_exceptions(_cpu_index: usize) { extern "C" { static __x86_64_exception_vectors: [usize; 32]; - fn __x86_64_syscall_vector(); } for (i, &entry) in __x86_64_exception_vectors.iter().enumerate() { @@ -154,24 +383,11 @@ pub unsafe fn init_exceptions(_cpu_index: usize) { }; core::arch::asm!("wbinvd; lidt ({0})", in(reg) &idtr, options(att_syntax)); - - // Initialize syscall vector - MSR_IA32_LSTAR.set(__x86_64_syscall_vector as u64); - MSR_IA32_SFMASK.write(MSR_IA32_SFMASK::IF::Masked); - MSR_IA32_STAR.write( - // On sysret, CS = val + 16 (0x23), SS = val + 8 (0x1B) - MSR_IA32_STAR::SYSRET_CS_SS.val(0x1B - 8) + - // On syscall, CS = val (0x08), SS = val + 8 (0x10) - MSR_IA32_STAR::SYSCALL_CS_SS.val(0x08), - ); - - MSR_IA32_EFER.modify(MSR_IA32_EFER::SCE::Enable); } global_asm!( include_str!("vectors.S"), exception_handler = sym __x86_64_exception_handler, apic_irq_handler = sym __x86_64_apic_irq_handler, - syscall_handler = sym __x86_64_syscall_handler, options(att_syntax) ); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 212caaf0..5dad6708 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -10,10 +10,16 @@ use crate::{ debug::DebugSink, device::{ display::{fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, + input::KeyboardDevice, interrupt::{ExternalInterruptController, InterruptSource}, platform::Platform, + tty::CombinedTerminal, Device, }, + fs::{ + devfs::{self, CharDeviceType}, + INITRD_DATA, + }, mem::{ phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, ConvertAddress, @@ -41,6 +47,7 @@ pub mod exception; pub mod gdt; pub mod peripherals; pub mod registers; +pub mod syscall; pub mod table; /// Helper trait to provide abstract access to available memory regions @@ -122,6 +129,9 @@ pub struct X86_64 { yboot_framebuffer: OneTimeInit, rsdp_phys_base: OneTimeInit, + // TODO make this fully dynamic? + combined_terminal: OneTimeInit, + // Static devices with dynamic addresses ioapic: IoApic, @@ -196,6 +206,14 @@ impl Platform for X86_64 { // Enable IRQs for the devices self.ps2.init_irq()?; + + // Add a terminal to the devfs + // TODO this is ugly + let combined_terminal = CombinedTerminal::new(&self.ps2, FB_CONSOLE.get()); + self.combined_terminal.init(combined_terminal); + self.ps2.attach(self.combined_terminal.get()); + + devfs::add_char_device(self.combined_terminal.get(), CharDeviceType::TtyRegular)?; } Ok(()) @@ -255,6 +273,16 @@ impl X86_64 { // Reserve memory map reserve_region("memory-map", memory_map.reserved_range()); + if let Some(initrd) = INITRD_DATA.try_get() { + reserve_region( + "initrd", + PhysicalMemoryRegion { + base: initrd.phys_page_start, + size: initrd.phys_page_len, + }, + ); + } + phys::init_from_iter(memory_map.iter().map(|r| PhysicalMemoryRegion { base: r.start_address(), size: r.page_count() * 0x1000, @@ -292,6 +320,9 @@ impl X86_64 { pub static ARCHITECTURE: X86_64 = X86_64 { rsdp_phys_base: OneTimeInit::new(), yboot_framebuffer: OneTimeInit::new(), + + combined_terminal: OneTimeInit::new(), + ioapic: IoApic::new(), ps2: PS2Controller::new(IrqNumber::Isa(1), IrqNumber::Isa(12), 0x64, 0x60), diff --git a/src/arch/x86_64/peripherals/ps2/codeset.rs b/src/arch/x86_64/peripherals/ps2/codeset.rs new file mode 100644 index 00000000..ceedeecd --- /dev/null +++ b/src/arch/x86_64/peripherals/ps2/codeset.rs @@ -0,0 +1,6 @@ +pub static CODE_SET_1_00: &[u8] = &[ + 0x00, b'\x1b', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'-', b'=', b'\x7f', + b'\t', b'q', b'w', b'e', b'r', b't', b'y', b'u', b'i', b'o', b'p', b'[', b']', b'\n', 0x00, + b'a', b's', b'd', b'f', b'g', b'h', b'j', b'k', b'l', b';', b'\'', b'`', 0x00, b'\\', b'z', + b'x', b'c', b'v', b'b', b'n', b'm', b',', b'.', b'/', 0x00, +]; diff --git a/src/arch/x86_64/peripherals/ps2.rs b/src/arch/x86_64/peripherals/ps2/mod.rs similarity index 67% rename from src/arch/x86_64/peripherals/ps2.rs rename to src/arch/x86_64/peripherals/ps2/mod.rs index 5ddcf923..f194ed25 100644 --- a/src/arch/x86_64/peripherals/ps2.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -6,9 +6,17 @@ use crate::{ x86_64::{apic::IrqNumber, intrinsics::IoPort}, PLATFORM, }, - device::{interrupt::InterruptSource, platform::Platform, Device}, + device::{ + input::KeyboardDevice, interrupt::InterruptSource, platform::Platform, tty::TtyDevice, + Device, + }, + sync::IrqSafeSpinlock, }; +use self::codeset::CODE_SET_1_00; + +mod codeset; + /// PS/2 controller driver pub struct PS2Controller { command: IoPort, @@ -16,6 +24,26 @@ pub struct PS2Controller { primary_irq: IrqNumber, #[allow(unused)] auxiliary_irq: IrqNumber, + + tty: IrqSafeSpinlock>>, +} + +fn translate(codeset: &[u8], key: u8) -> Option { + if key as usize > codeset.len() { + return None; + } + let value = codeset[key as usize]; + if value != 0 { + Some(value) + } else { + None + } +} + +impl KeyboardDevice for PS2Controller { + fn attach(&self, terminal: &'static dyn TtyDevice<16>) { + self.tty.lock().replace(terminal); + } } impl InterruptSource for PS2Controller { @@ -52,12 +80,21 @@ impl InterruptSource for PS2Controller { if key == 0xE0 { key = self.data.read(); + infoln!("TODO: handle 0xE0"); + return Ok(true); } self.data.read(); if key < 128 { - debugln!("Got key {:#x}", key); + let Some(code) = translate(CODE_SET_1_00, key) else { + return Ok(true); + }; + let terminal = self.tty.lock(); + + if let Some(terminal) = &*terminal { + terminal.recv_byte(code); + } } Ok(true) @@ -90,6 +127,7 @@ impl PS2Controller { auxiliary_irq, command: IoPort::new(cmd_port), data: IoPort::new(data_port), + tty: IrqSafeSpinlock::new(None), } } diff --git a/src/arch/x86_64/syscall.S b/src/arch/x86_64/syscall.S new file mode 100644 index 00000000..1bb7b1d0 --- /dev/null +++ b/src/arch/x86_64/syscall.S @@ -0,0 +1,175 @@ +.global __x86_64_syscall_vector + +.set REG_RAX, 0 * 8 +.set REG_RDI, 1 * 8 +.set REG_RSI, 2 * 8 +.set REG_RDX, 3 * 8 +.set REG_R10, 4 * 8 +.set REG_R8, 5 * 8 +.set REG_R9, 6 * 8 + +.set REG_RCX, 7 * 8 +.set REG_R11, 8 * 8 + +.set REG_USER_IP, 9 * 8 +.set REG_USER_SP, 10 * 8 +.set REG_USER_FLAGS, 11 * 8 + +.set REG_RBX, 12 * 8 +.set REG_RBP, 13 * 8 // +.set REG_R12, 14 * 8 // +.set REG_R13, 15 * 8 // Overwritten by iret +.set REG_R14, 16 * 8 // +.set REG_R15, 17 * 8 // + +// 15 general-purpose registers +// user ip +// user sp +// user flags +.set SYSCALL_STATE_SIZE, 18 * 8 + +.set REG_IRET_RCX, -(SYSCALL_STATE_SIZE - REG_RCX) +.set REG_IRET_R11, -(SYSCALL_STATE_SIZE - REG_R11) +.set REG_IRET_USER_IP, -(SYSCALL_STATE_SIZE - REG_USER_IP) +.set REG_IRET_USER_SP, -(SYSCALL_STATE_SIZE - REG_USER_SP) +.set REG_IRET_USER_FLAGS, -(SYSCALL_STATE_SIZE - REG_USER_FLAGS) + +.macro SYSCALL_SAVE_STATE + subq $SYSCALL_STATE_SIZE, %rsp + + // Syscall-specific ordering for these + movq %rax, REG_RAX(%rsp) + movq %rdi, REG_RDI(%rsp) + movq %rsi, REG_RSI(%rsp) + movq %rdx, REG_RDX(%rsp) + movq %r10, REG_R10(%rsp) + movq %r8, REG_R8(%rsp) + movq %r9, REG_R9(%rsp) + + movq %rcx, REG_RCX(%rsp) + movq %r11, REG_R11(%rsp) + + movq %gs:(16), %rax + movq %rcx, REG_USER_IP(%rsp) + movq %rax, REG_USER_SP(%rsp) + movq %r11, REG_USER_FLAGS(%rsp) + + movq %rbx, REG_RBX(%rsp) + movq %rbp, REG_RBP(%rsp) + movq %r12, REG_R12(%rsp) + movq %r13, REG_R13(%rsp) + movq %r14, REG_R14(%rsp) + movq %r15, REG_R15(%rsp) +.endm + +.macro SYSCALL_RESTORE_STATE +.endm + +.section .text +__x86_64_syscall_vector: + // On entry: + // %rcx - userspace %rip + // %r11 - rflags + + // Store user RSP + // TODO: eliminate magic %gs-relative addresses + mov %rsp, %gs:(16) + // Load the task's RSP0 from TSS + mov %gs:(8), %rsp + mov 4(%rsp), %rsp + + SYSCALL_SAVE_STATE + + mov $0x10, %ax + mov %ax, %ss + mov %ax, %ds + mov %ax, %es + + mov %rsp, %rdi + call {syscall_handler} + +.restore_state: + // Restore non-clobbered state + movq REG_RAX(%rsp), %rax + movq REG_RDI(%rsp), %rdi + movq REG_RSI(%rsp), %rsi + movq REG_RDX(%rsp), %rdx + movq REG_R10(%rsp), %r10 + movq REG_R8(%rsp), %r8 + movq REG_R9(%rsp), %r9 + + movq REG_RBX(%rsp), %rbx + movq REG_RBP(%rsp), %rbp + movq REG_R12(%rsp), %r12 + movq REG_R13(%rsp), %r13 + movq REG_R14(%rsp), %r14 + movq REG_R15(%rsp), %r15 + + movq REG_RCX(%rsp), %rcx + movq REG_USER_IP(%rsp), %r11 + + // TODO do I also need to check if rflags changed? + // If user_ip != rcx (syscall user ip), then use iretq instead + // Most likely the frame was loaded from a signal entry/return + cmpq %rcx, %r11 + jne .return_via_iret + + // Still not restored: + // %rcx (user ip), %r11 (user flags), user sp + +.return_via_sysret: + // Regular syscall return + movq REG_USER_SP(%rsp), %rcx + movq %rcx, %gs:(16) + + movq REG_USER_IP(%rsp), %rcx + movq REG_USER_FLAGS(%rsp), %r11 + + addq $SYSCALL_STATE_SIZE, %rsp + + // %rcx and %r11 now contain the expected values + // Restore user RSP + mov %gs:(16), %rsp + + sysretq + +.return_via_iret: + .set IRET_FRAME_SIZE, 5 * 8 + .set IRET_SS, 4 * 8 + .set IRET_RSP, 3 * 8 + .set IRET_RFLAGS, 2 * 8 + .set IRET_CS, 1 * 8 + .set IRET_RIP, 0 * 8 + + // Need to restore %rcx, %r11, user ip, user sp, user flags + // Syscall frame is ordered in a way to prevent iret frame from + // overwriting any context that was not restored at this moment + + // r15, r14, r13, r12, and rbp will be overwritten, but they're + // already restored to their registers by now + + // Restore %r11 and only use %rcx + movq REG_R11(%rsp), %r11 + + addq $SYSCALL_STATE_SIZE, %rsp + + subq $IRET_FRAME_SIZE, %rsp + + // SS:RSP + movq (IRET_FRAME_SIZE + REG_IRET_USER_SP)(%rsp), %rcx + movq $0x1B, IRET_SS(%rsp) + movq %rcx, IRET_RSP(%rsp) + + // RFLAGS + movq (IRET_FRAME_SIZE + REG_IRET_USER_FLAGS)(%rsp), %rcx + movq %rcx, IRET_RFLAGS(%rsp) + + // CS:RIP + movq (IRET_FRAME_SIZE + REG_IRET_USER_IP)(%rsp), %rcx + movq $0x23, IRET_CS(%rsp) + movq %rcx, IRET_RIP(%rsp) + + // Restore %rcx + movq (IRET_FRAME_SIZE + REG_IRET_RCX)(%rsp), %rcx + + iretq diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs new file mode 100644 index 00000000..20405d20 --- /dev/null +++ b/src/arch/x86_64/syscall.rs @@ -0,0 +1,157 @@ +use core::arch::global_asm; + +use abi::{arch::SavedFrame, syscall::SyscallFunction}; +use tock_registers::interfaces::{ReadWriteable, Writeable}; + +use crate::{ + arch::x86_64::registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, + syscall::raw_syscall_handler, + task::process::{Process, TaskFrame}, +}; + +use super::exception::handle_signal_exit; + +/// Set of registers saved when taking a syscall instruction +#[derive(Debug)] +#[repr(C)] +pub struct SyscallFrame { + rax: u64, + args: [u64; 6], + + rcx: u64, + r11: u64, + + user_ip: u64, + user_sp: u64, + user_flags: u64, + + rbx: u64, + rbp: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, +} + +impl TaskFrame for SyscallFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.args[2], + rbx: self.rbx, + rsi: self.args[1], + rdi: self.args[0], + rbp: self.rbp, + r8: self.args[4], + r9: self.args[5], + r10: self.args[3], + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.user_ip, + user_sp: self.user_sp, + rflags: self.user_flags, + } + } + + fn restore(&mut self, saved: &SavedFrame) { + self.rax = saved.rax; + self.args[0] = saved.rdi; + self.args[1] = saved.rsi; + self.args[2] = saved.rdx; + self.args[3] = saved.r10; + self.args[4] = saved.r8; + self.args[5] = saved.r9; + + self.rcx = saved.rcx; + self.r11 = saved.r11; + + self.user_ip = saved.user_ip; + self.user_sp = saved.user_sp; + self.user_flags = saved.rflags; + + self.rbx = saved.rbx; + self.rbp = saved.rbp; + self.r12 = saved.r12; + self.r13 = saved.r13; + self.r14 = saved.r14; + self.r15 = saved.r15; + } + + fn argument(&self) -> u64 { + self.args[0] + } + + fn user_sp(&self) -> usize { + self.user_sp as _ + } + + fn user_ip(&self) -> usize { + self.user_ip as _ + } + + fn set_user_sp(&mut self, value: usize) { + self.user_sp = value as _; + } + + fn set_user_ip(&mut self, value: usize) { + self.user_ip = value as _; + } + + fn set_return_value(&mut self, value: u64) { + self.rax = value; + } + + fn set_argument(&mut self, value: u64) { + self.args[0] = value; + } +} + +fn syscall_inner(frame: &mut SyscallFrame) { + if frame.rax == usize::from(SyscallFunction::ExitSignal) as u64 { + unsafe { + handle_signal_exit(frame); + return; + } + } + + let result = raw_syscall_handler(frame.rax, &frame.args); + + frame.rax = result; +} + +extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { + let frame = unsafe { &mut *frame }; + let process = Process::current(); + syscall_inner(frame); + unsafe { + process.handle_signal(frame); + } +} + +pub unsafe fn init_syscall() { + extern "C" { + fn __x86_64_syscall_vector(); + } + + // Initialize syscall vector + MSR_IA32_LSTAR.set(__x86_64_syscall_vector as u64); + MSR_IA32_SFMASK.write(MSR_IA32_SFMASK::IF::Masked); + MSR_IA32_STAR.write( + // On sysret, CS = val + 16 (0x23), SS = val + 8 (0x1B) + MSR_IA32_STAR::SYSRET_CS_SS.val(0x1B - 8) + + // On syscall, CS = val (0x08), SS = val + 8 (0x10) + MSR_IA32_STAR::SYSCALL_CS_SS.val(0x08), + ); + + MSR_IA32_EFER.modify(MSR_IA32_EFER::SCE::Enable); +} + +global_asm!( + include_str!("syscall.S"), + syscall_handler = sym __x86_64_syscall_handler, + options(att_syntax) +); diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/table/mod.rs index 3265c756..e5eb864d 100644 --- a/src/arch/x86_64/table/mod.rs +++ b/src/arch/x86_64/table/mod.rs @@ -14,13 +14,15 @@ pub use fixed::{init_fixed_tables, KERNEL_TABLES}; use crate::mem::{ phys::{self, PageUsage}, - table::{EntryLevel, NonTerminalEntryLevel}, + table::{ + EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, + }, ConvertAddress, }; bitflags! { /// Describes how each page table entry is mapped - pub struct PageAttributes: u64 { + struct PageAttributes: u64 { /// When set, the mapping is considered valid and pointing somewhere const PRESENT = 1 << 0; /// For tables, allows writes to further translation levels, for pages/blocks, allows @@ -78,8 +80,8 @@ impl NonTerminalEntryLevel for L2 { } impl const EntryLevel for L0 { - fn index(_addr: usize) -> usize { - todo!() + fn index(addr: usize) -> usize { + (addr >> 39) & 0x1FF } fn page_offset(_addr: usize) -> usize { @@ -119,17 +121,27 @@ impl const EntryLevel for L3 { impl PageEntry { /// Constructs a mapping which points to a 4KiB page - pub fn page(phys: usize, attrs: PageAttributes) -> Self { + fn page(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::USER).bits(), PhantomData, ) } + + /// Returns the physical address of the page this entry refers to, returning None if it does + /// not + pub fn as_page(self) -> Option { + if self.0 & PageAttributes::PRESENT.bits() != 0 { + Some((self.0 & !0xFFF) as usize) + } else { + None + } + } } impl PageEntry { /// Constructs a mapping which points to a 2MiB block - pub fn block(phys: usize, attrs: PageAttributes) -> Self { + fn block(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK | PageAttributes::USER) @@ -141,7 +153,7 @@ impl PageEntry { impl PageEntry { /// Constructs a mapping which points to a next-level table - pub fn table(phys: usize, attrs: PageAttributes) -> Self { + fn table(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (attrs @@ -152,11 +164,27 @@ impl PageEntry { PhantomData, ) } + + /// Returns the physical address of the table this entry refers to, returning None if it + /// does not + pub fn as_table(self) -> Option { + if self.0 & PageAttributes::PRESENT.bits() != 0 + && self.0 & PageAttributes::BLOCK.bits() == 0 + { + Some((self.0 & !0xFFF) as usize) + } else { + None + } + } } impl PageEntry { /// An entry that is not mapped pub const INVALID: Self = Self(0, PhantomData); + + pub fn is_present(&self) -> bool { + self.0 & PageAttributes::PRESENT.bits() != 0 + } } impl PageTable { @@ -167,12 +195,49 @@ impl PageTable { } } + /// Allocates a new page table, filling it with non-preset entries + pub fn new_zeroed() -> Result<&'static mut Self, Error> { + let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() }; + let table = unsafe { &mut *(page as *mut Self) }; + for i in 0..512 { + table[i] = PageEntry::INVALID; + } + Ok(table) + } + /// Returns the physical address of this table pub fn physical_address(&self) -> usize { unsafe { (self.data.as_ptr() as usize).physicalize() } } } +impl NextPageTable for PageTable { + type NextLevel = PageTable; + + fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> { + let entry = self[index]; + + entry + .as_table() + .map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) }) + } + + fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) }) + } else { + let table = PageTable::new_zeroed()?; + self[index] = PageEntry::::table( + table.physical_address(), + PageAttributes::WRITABLE | PageAttributes::USER, + ); + Ok(table) + } + } +} + impl Index for PageTable { type Output = PageEntry; @@ -187,6 +252,74 @@ impl IndexMut for PageTable { } } +impl From for MapAttributes { + fn from(value: PageAttributes) -> Self { + todo!(); + } +} + +impl From for PageAttributes { + fn from(value: MapAttributes) -> Self { + let mut res = PageAttributes::WRITABLE; + if value.intersects(MapAttributes::USER_READ | MapAttributes::USER_WRITE) { + res |= PageAttributes::USER; + } + res + } +} + +impl VirtualMemoryManager for AddressSpace { + fn allocate( + &self, + hint: Option, + len: usize, + attrs: MapAttributes, + ) -> Result { + if hint.is_some() { + todo!(); + } + + const TRY_ALLOC_START: usize = 0x100000000; + const TRY_ALLOC_END: usize = 0xF00000000; + + 'l0: for base in (TRY_ALLOC_START..TRY_ALLOC_END - len * 0x1000).step_by(0x1000) { + for i in 0..len { + if self.translate(base + i * 0x1000).is_some() { + continue 'l0; + } + } + + for i in 0..len { + let page = phys::alloc_page(PageUsage::Used)?; + self.map_page(base + i * 0x1000, page, attrs)?; + } + + return Ok(base); + } + + Err(Error::OutOfMemory) + } + + fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> { + self.write_entry(virt, PageEntry::page(phys, attrs.into()), true) + } + + fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> { + for page in (addr..addr + len).step_by(0x1000) { + let Some(phys) = self.translate(page) else { + todo!(); + }; + + self.write_entry(page, PageEntry::INVALID, true)?; + unsafe { + phys::free_page(phys); + } + } + + Ok(()) + } +} + impl AddressSpace { /// Allocates an empty address space with all entries marked as non-present pub fn new_empty() -> Result { @@ -204,6 +337,55 @@ impl AddressSpace { Ok(Self { l0 }) } + pub unsafe fn from_cr3(cr3: usize) -> Self { + let cr3_virt = cr3.virtualize(); + let l0 = unsafe { cr3_virt as *mut PageTable }; + Self { l0 } + } + + unsafe fn as_mut(&self) -> &'static mut PageTable { + self.l0.as_mut().unwrap() + } + + // TODO return page size and attributes + /// Returns the physical address to which the `virt` address is mapped + pub fn translate(&self, virt: usize) -> Option { + let l0i = L0::index(virt); + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l1 = unsafe { self.as_mut().get_mut(l0i) }?; + let l2 = l1.get_mut(l1i)?; + let l3 = l2.get_mut(l2i)?; + + l3[l3i].as_page() + } + + // Write a single 4KiB entry + fn write_entry(&self, virt: usize, entry: PageEntry, overwrite: bool) -> Result<(), Error> { + let l0i = L0::index(virt); + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l1 = unsafe { self.as_mut().get_mut_or_alloc(l0i) }?; + let l2 = l1.get_mut_or_alloc(l1i)?; + let l3 = l2.get_mut_or_alloc(l2i)?; + + if l3[l3i].is_present() && !overwrite { + todo!(); + } + + l3[l3i] = entry; + + unsafe { + core::arch::asm!("invlpg ({0})", in(reg) virt, options(att_syntax)); + } + + Ok(()) + } + /// Returns the physical address of the root table pub fn physical_address(&self) -> usize { unsafe { (self.l0 as usize).physicalize() } diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index 1285a9f2..d7afb4ba 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -7,38 +7,38 @@ mov %rax, 0(%rsp) mov %rcx, 8(%rsp) mov %rdx, 16(%rsp) - mov %rbx, 32(%rsp) - mov %rsi, 36(%rsp) + mov %rbx, 24(%rsp) + mov %rsi, 32(%rsp) mov %rdi, 40(%rsp) mov %rbp, 48(%rsp) - mov %r8, 52(%rsp) - mov %r9, 56(%rsp) - mov %r10, 60(%rsp) - mov %r11, 64(%rsp) - mov %r12, 68(%rsp) - mov %r13, 72(%rsp) - mov %r14, 76(%rsp) - mov %r15, 80(%rsp) + mov %r8, 56(%rsp) + mov %r9, 64(%rsp) + mov %r10, 72(%rsp) + mov %r11, 80(%rsp) + mov %r12, 88(%rsp) + mov %r13, 96(%rsp) + mov %r14, 104(%rsp) + mov %r15, 112(%rsp) .endm .macro EXC_RESTORE_STATE mov 0(%rsp), %rax mov 8(%rsp), %rcx mov 16(%rsp), %rdx - mov 32(%rsp), %rbx - mov 36(%rsp), %rsi + mov 24(%rsp), %rbx + mov 32(%rsp), %rsi mov 40(%rsp), %rdi mov 48(%rsp), %rbp - mov 52(%rsp), %r8 - mov 56(%rsp), %r9 - mov 60(%rsp), %r10 - mov 64(%rsp), %r11 - mov 68(%rsp), %r12 - mov 72(%rsp), %r13 - mov 76(%rsp), %r14 - mov 80(%rsp), %r15 + mov 56(%rsp), %r8 + mov 64(%rsp), %r9 + mov 72(%rsp), %r10 + mov 80(%rsp), %r11 + mov 88(%rsp), %r12 + mov 96(%rsp), %r13 + mov 104(%rsp), %r14 + mov 112(%rsp), %r15 addq $PT_REGS_SIZE, %rsp .endm @@ -58,46 +58,93 @@ __x86_64_exc_\n: jmp __x86_64_exc_common .endm +.set IRQ_REG_RAX, 0 * 8 +.set IRQ_REG_RCX, 1 * 8 +.set IRQ_REG_RDX, 2 * 8 +.set IRQ_REG_RBX, 3 * 8 +.set IRQ_REG_RSI, 4 * 8 +.set IRQ_REG_RDI, 5 * 8 +.set IRQ_REG_RBP, 6 * 8 +.set IRQ_REG_R8, 7 * 8 +.set IRQ_REG_R9, 8 * 8 +.set IRQ_REG_R10, 9 * 8 +.set IRQ_REG_R11, 10 * 8 +.set IRQ_REG_R12, 11 * 8 +.set IRQ_REG_R13, 12 * 8 +.set IRQ_REG_R14, 13 * 8 +.set IRQ_REG_R15, 14 * 8 + +// 15 registers + 1 padding qword +.set IRQ_STATE_SIZE, 16 * 8 + .macro apic_vector, n __x86_64_apic_irq_\n: - cli + // Save state + subq $IRQ_STATE_SIZE, %rsp - // Push dummy error codes - // pushq $0 - // pushq $0 - - EXC_SAVE_STATE + movq %rax, IRQ_REG_RAX(%rsp) + movq %rcx, IRQ_REG_RCX(%rsp) + movq %rdx, IRQ_REG_RDX(%rsp) + movq %rbx, IRQ_REG_RBX(%rsp) + movq %rsi, IRQ_REG_RSI(%rsp) + movq %rdi, IRQ_REG_RDI(%rsp) + movq %rbp, IRQ_REG_RBP(%rsp) + movq %r8, IRQ_REG_R8(%rsp) + movq %r9, IRQ_REG_R9(%rsp) + movq %r10, IRQ_REG_R10(%rsp) + movq %r11, IRQ_REG_R11(%rsp) + movq %r12, IRQ_REG_R12(%rsp) + movq %r13, IRQ_REG_R13(%rsp) + movq %r14, IRQ_REG_R14(%rsp) + movq %r15, IRQ_REG_R15(%rsp) mov $0x10, %ax mov %ax, %ss + mov %ax, %ds + mov %ax, %es mov $\n, %rdi mov %rsp, %rsi call {apic_irq_handler} - EXC_RESTORE_STATE + // Restore state + movq IRQ_REG_RAX(%rsp), %rax + movq IRQ_REG_RCX(%rsp), %rcx + movq IRQ_REG_RDX(%rsp), %rdx + movq IRQ_REG_RBX(%rsp), %rbx + movq IRQ_REG_RSI(%rsp), %rsi + movq IRQ_REG_RDI(%rsp), %rdi + movq IRQ_REG_RBP(%rsp), %rbp + movq IRQ_REG_R8(%rsp), %r8 + movq IRQ_REG_R9(%rsp), %r9 + movq IRQ_REG_R10(%rsp), %r10 + movq IRQ_REG_R11(%rsp), %r11 + movq IRQ_REG_R12(%rsp), %r12 + movq IRQ_REG_R13(%rsp), %r13 + movq IRQ_REG_R14(%rsp), %r14 + movq IRQ_REG_R15(%rsp), %r15 - // Remove dummy error codes - // add $16, %rsp + addq $IRQ_STATE_SIZE, %rsp iretq .endm .global __x86_64_exception_vectors .global __x86_64_apic_vectors -.global __x86_64_syscall_vector .section .text __x86_64_exc_common: EXC_SAVE_STATE mov %rsp, %rdi + call {exception_handler} - // TODO -1: - cli - hlt - jmp 1b + EXC_RESTORE_STATE + + // Remove error code and number from the stack + addq $16, %rsp + + iretq ISR_NERR 0 ISR_NERR 1 @@ -149,39 +196,6 @@ apic_vector 13 apic_vector 14 apic_vector 15 -__x86_64_syscall_vector: - // On entry: - // %rcx - userspace %rip - // %r11 - rflags - - // Store user RSP - // TODO: eliminate magic %gs-relative addresses - mov %rsp, %gs:(16) - // Load the task's RSP0 from TSS - mov %gs:(8), %rsp - mov 4(%rsp), %rsp - - // Store the state - EXC_SAVE_STATE - // Store user stack pointer - mov %gs:(16), %rax - pushq %rax - - mov %rsp, %rdi - call {syscall_handler} - - // Restore user stack pointer - popq %rax - mov %rax, %gs:(16) - // Restore the state - EXC_RESTORE_STATE - - // %rcx and %r11 now contain the expected values - // Restore user RSP - mov %gs:(16), %rsp - - sysretq - .section .rodata .global __x86_64_exception_vectors .p2align 4 diff --git a/src/debug.rs b/src/debug.rs index 449656cd..9ee927b7 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -13,7 +13,7 @@ use crate::{arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, u // }; /// Defines the severity of the message -#[derive(Clone, Copy)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum LogLevel { /// Debugging and verbose information Debug, @@ -161,9 +161,13 @@ pub fn init() { // } #[doc(hidden)] -pub fn debug_internal(args: Arguments, _level: LogLevel) { +pub fn debug_internal(args: Arguments, level: LogLevel) { use fmt::Write; + if level <= LogLevel::Debug { + return; + } + if DEBUG_PRINTER.is_initialized() { let mut printer = DEBUG_PRINTER.get().lock(); diff --git a/src/device/input.rs b/src/device/input.rs new file mode 100644 index 00000000..b2f543c8 --- /dev/null +++ b/src/device/input.rs @@ -0,0 +1,5 @@ +use super::{tty::TtyDevice, Device}; + +pub trait KeyboardDevice: Device { + fn attach(&self, terminal: &'static dyn TtyDevice<16>); +} diff --git a/src/device/mod.rs b/src/device/mod.rs index 537f6a8a..c699c79d 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -2,11 +2,12 @@ use abi::error::Error; pub mod display; +pub mod input; pub mod interrupt; pub mod platform; -// pub mod serial; -// pub mod timer; -// pub mod tty; +pub mod serial; +pub mod timer; +pub mod tty; /// General device interface pub trait Device { diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index ab79c5c7..cc4765f4 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -3,6 +3,7 @@ use abi::error::Error; use super::Device; +#[cfg(target_arch = "aarch64")] pub mod pl011; /// Generic serial device interface diff --git a/src/device/tty.rs b/src/device/tty.rs index b4f526ce..8a7af3da 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -1,17 +1,88 @@ //! Terminal driver implementation use abi::{ error::Error, - io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, + io::{ + DeviceRequest, TerminalInputOptions, TerminalLineOptions, TerminalOptions, + TerminalOutputOptions, + }, process::Signal, }; +use vfs::CharDevice; use crate::{ + debug::DebugSink, + device::display::fb_console::FramebufferConsole, proc::wait::Wait, sync::IrqSafeSpinlock, task::{process::Process, ProcessId}, }; -use super::serial::SerialDevice; +use super::{input::KeyboardDevice, serial::SerialDevice, Device}; + +pub struct CombinedTerminal { + input: &'static dyn KeyboardDevice, + output: &'static FramebufferConsole, + // TODO output + input_ring: CharRing<16>, +} + +impl CombinedTerminal { + pub fn new(input: &'static dyn KeyboardDevice, output: &'static FramebufferConsole) -> Self { + Self { + input, + output, + input_ring: CharRing::new(), + } + } +} + +impl TtyDevice<16> for CombinedTerminal { + fn ring(&self) -> &CharRing<16> { + &self.input_ring + } +} + +impl SerialDevice for CombinedTerminal { + fn receive(&self, blocking: bool) -> Result { + todo!() + } + + fn send(&self, byte: u8) -> Result<(), Error> { + self.output.putc(byte) + } +} + +impl Device for CombinedTerminal { + unsafe fn init(&self) -> Result<(), Error> { + todo!() + } + + fn name(&self) -> &'static str { + "Combined terminal device" + } +} + +impl CharDevice for CombinedTerminal { + fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { + assert!(blocking); + self.line_read(data) + } + + fn write(&self, blocking: bool, data: &[u8]) -> Result { + assert!(blocking); + self.line_write(data) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(id as _); + Ok(()) + } + _ => Err(Error::InvalidArgument), + } + } +} struct CharRingInner { rd: usize, @@ -91,7 +162,8 @@ pub trait TtyDevice: SerialDevice { } } - if byte == config.chars.interrupt && config.line.contains(TerminalLineOptions::SIGNAL) { + // byte == config.chars.interrupt + if byte == b'=' && config.line.contains(TerminalLineOptions::SIGNAL) { drop(config); let pgrp = ring.inner.lock().process_group; if let Some(pgrp) = pgrp { diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index cd018a1b..a62c20c6 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -15,6 +15,8 @@ use crate::util::OneTimeInit; /// Describes the kind of a character device #[derive(Debug)] pub enum CharDeviceType { + /// Regular terminal + TtyRegular, /// Serial terminal TtySerial, } @@ -71,9 +73,11 @@ fn _add_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Er /// Adds a character device to the devfs pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Result<(), Error> { + static TTY_COUNT: AtomicUsize = AtomicUsize::new(0); static TTYS_COUNT: AtomicUsize = AtomicUsize::new(0); let (count, prefix) = match kind { + CharDeviceType::TtyRegular => (&TTY_COUNT, "tty"), CharDeviceType::TtySerial => (&TTYS_COUNT, "ttyS"), }; diff --git a/src/main.rs b/src/main.rs index 08986974..4f9d7053 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,13 +8,24 @@ maybe_uninit_slice, arbitrary_self_types, const_mut_refs, - let_chains + let_chains, + linked_list_cursors )] #![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] #![warn(missing_docs)] #![no_std] #![no_main] +use abi::{ + error::Error, + io::{FileMode, OpenOptions, RawFd}, + process::ExitCode, +}; +use fs::{devfs, FileBlockAllocator, INITRD_DATA}; +use memfs::MemoryFilesystem; +use task::process::Process; +use vfs::{Filesystem, IoContext, VnodeRef}; + extern crate yggdrasil_abi as abi; // // use abi::{ @@ -35,74 +46,74 @@ pub mod debug; pub mod arch; pub mod device; -// pub mod fs; +pub mod fs; pub mod mem; pub mod panic; -// pub mod proc; +pub mod proc; pub mod sync; -// pub mod syscall; +pub mod syscall; pub mod task; pub mod util; -// -// fn setup_root() -> Result { -// let initrd_data = INITRD_DATA.get(); -// let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); -// fs.root() -// } -// -// /// Entry point for common kernel code. -// /// -// /// # Note -// /// -// /// This function is meant to be used as a kernel-space process after all the platform-specific -// /// initialization has finished. -// pub fn kernel_main() { -// // use crate::{debug::LogLevel, mem::phys}; -// // use crate::task::tasklet::{self, TaskFlow}; -// // use core::time::Duration; -// -// // Schedule a tasklet to print memory usage stats every 10 seconds -// // tasklet::add_periodic("mem-stats", Duration::from_secs(3), || { -// // let phys_mem = phys::PHYSICAL_MEMORY.get().lock(); -// // let stats = phys_mem.stats(); -// // stats.dump(LogLevel::Debug); -// -// // TaskFlow::Continue -// // }); -// -// let root = match setup_root() { -// Ok(root) => root, -// Err(err) => { -// warnln!("Could not setup root from initrd: {:?}", err); -// return; -// } -// }; -// -// let ioctx = IoContext::new(root); -// let node = ioctx.find(None, "/init", true, true).unwrap(); -// let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); -// -// let devfs = devfs::root(); -// let console = ioctx -// .find(Some(devfs.clone()), "ttyS0", true, true) -// .unwrap(); -// let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); -// let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); -// let stderr = stdout.clone(); -// -// { -// let user_init = proc::exec::load_elf(file, &["/init", "xxx"]).unwrap(); -// let mut io = user_init.io.lock(); -// io.set_ioctx(ioctx); -// io.set_file(RawFd::STDIN, stdin).unwrap(); -// io.set_file(RawFd::STDOUT, stdout).unwrap(); -// io.set_file(RawFd::STDERR, stderr).unwrap(); -// drop(io); -// -// user_init.set_session_terminal(console); -// -// user_init.enqueue_somewhere(); -// } -// -// Process::current().exit(ExitCode::SUCCESS); -// } + +fn setup_root() -> Result { + let initrd_data = INITRD_DATA.get(); + let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); + fs.root() +} + +/// Entry point for common kernel code. +/// +/// # Note +/// +/// This function is meant to be used as a kernel-space process after all the platform-specific +/// initialization has finished. +pub fn kernel_main() { + // use crate::{debug::LogLevel, mem::phys}; + // use crate::task::tasklet::{self, TaskFlow}; + // use core::time::Duration; + + // Schedule a tasklet to print memory usage stats every 10 seconds + // tasklet::add_periodic("mem-stats", Duration::from_secs(3), || { + // let phys_mem = phys::PHYSICAL_MEMORY.get().lock(); + // let stats = phys_mem.stats(); + // stats.dump(LogLevel::Debug); + + // TaskFlow::Continue + // }); + + let root = match setup_root() { + Ok(root) => root, + Err(err) => { + warnln!("Could not setup root from initrd: {:?}", err); + return; + } + }; + + let ioctx = IoContext::new(root); + let node = ioctx.find(None, "/init", true, true).unwrap(); + let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + + let devfs = devfs::root(); + let console = ioctx.find(Some(devfs.clone()), "tty0", true, true).unwrap(); + let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); + let stderr = stdout.clone(); + + { + let user_init = proc::exec::load_elf("init", file, &["/init", "xxx"]).unwrap(); + let mut io = user_init.io.lock(); + io.set_ioctx(ioctx); + io.set_file(RawFd::STDIN, stdin).unwrap(); + io.set_file(RawFd::STDOUT, stdout).unwrap(); + io.set_file(RawFd::STDERR, stderr).unwrap(); + drop(io); + + user_init.set_session_terminal(console); + + user_init.enqueue_somewhere(); + } + + loop {} + // XXX + // Process::current().exit(ExitCode::SUCCESS); +} diff --git a/src/mem/mod.rs b/src/mem/mod.rs index d5cf0ad7..fcd6fab8 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,12 +1,18 @@ //! Memory management utilities and types // use core::{alloc::Layout, mem::size_of}; +use core::{alloc::Layout, mem::size_of}; + +use abi::error::Error; + // use abi::error::Error; // use crate::{ arch::{Architecture, ArchitectureImpl, PlatformImpl}, device::platform::Platform, }; + +use self::table::AddressSpace; // // use self::table::AddressSpace; @@ -49,74 +55,74 @@ pub unsafe trait ConvertAddress { unsafe fn physicalize(self) -> Self; } -// /// Helper trait to allow cross-address space access to pointers -// pub trait ForeignPointer: Sized { -// /// Perform a volatile pointer write without dropping the old value. -// /// -// /// # Panics -// /// -// /// The function panics if any of the following conditions is met: -// /// -// /// * The address of the pointer is not mapped in the `space`. -// /// * The pointer is not writable. -// /// * The pointer is misaligned. -// /// -// /// # Safety -// /// -// /// As this function allows direct memory writes, it is inherently unsafe. -// unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); -// -// /// Performs pointer validation for given address space: -// /// -// /// * Checks if the pointer has proper alignment for the type. -// /// * Checks if the pointer is mapped in the address space. -// /// * Checks if the pointer is above the userspace memory boundary. -// /// -// /// # Safety -// /// -// /// Even though this function does the necessary checks, it is still a raw pointer to reference -// /// conversion, and thus is unsafe. -// unsafe fn validate_user_ptr<'a>( -// self: *const Self, -// space: &AddressSpace, -// ) -> Result<&'a Self, Error>; -// -// /// [ForeignPointer::validate_user_ptr], with extra "writability" check. -// /// -// /// # Safety -// /// -// /// Even though this function does the necessary checks, it is still a raw pointer to reference -// /// conversion, and thus is unsafe. -// unsafe fn validate_user_mut<'a>( -// self: *mut Self, -// space: &AddressSpace, -// ) -> Result<&'a mut Self, Error>; -// -// /// [ForeignPointer::validate_user_ptr], but for slices -// /// -// /// # Safety -// /// -// /// Even though this function does the necessary checks, it is still a raw pointer to reference -// /// conversion, and thus is unsafe. -// unsafe fn validate_user_slice<'a>( -// self: *const Self, -// len: usize, -// space: &AddressSpace, -// ) -> Result<&'a [Self], Error>; -// -// /// [ForeignPointer::validate_user_slice], but for mutable slices -// /// -// /// # Safety -// /// -// /// Even though this function does the necessary checks, it is still a raw pointer to reference -// /// conversion, and thus is unsafe. -// unsafe fn validate_user_slice_mut<'a>( -// self: *mut Self, -// len: usize, -// space: &AddressSpace, -// ) -> Result<&'a mut [Self], Error>; -// } -// +/// Helper trait to allow cross-address space access to pointers +pub trait ForeignPointer: Sized { + /// Perform a volatile pointer write without dropping the old value. + /// + /// # Panics + /// + /// The function panics if any of the following conditions is met: + /// + /// * The address of the pointer is not mapped in the `space`. + /// * The pointer is not writable. + /// * The pointer is misaligned. + /// + /// # Safety + /// + /// As this function allows direct memory writes, it is inherently unsafe. + unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); + + /// Performs pointer validation for given address space: + /// + /// * Checks if the pointer has proper alignment for the type. + /// * Checks if the pointer is mapped in the address space. + /// * Checks if the pointer is above the userspace memory boundary. + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_ptr<'a>( + self: *const Self, + space: &AddressSpace, + ) -> Result<&'a Self, Error>; + + /// [ForeignPointer::validate_user_ptr], with extra "writability" check. + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_mut<'a>( + self: *mut Self, + space: &AddressSpace, + ) -> Result<&'a mut Self, Error>; + + /// [ForeignPointer::validate_user_ptr], but for slices + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_slice<'a>( + self: *const Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a [Self], Error>; + + /// [ForeignPointer::validate_user_slice], but for mutable slices + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_slice_mut<'a>( + self: *mut Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a mut [Self], Error>; +} + unsafe impl ConvertAddress for usize { #[inline(always)] unsafe fn virtualize(self) -> Self { @@ -163,120 +169,120 @@ unsafe impl ConvertAddress for *const T { } } -// impl ForeignPointer for T { -// unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { -// // TODO check align -// let addr = self as usize; -// let start_page = addr & !0xFFF; -// let end_page = (addr + size_of::() - 1) & !0xFFF; -// let page_offset = addr & 0xFFF; -// -// if start_page != end_page { -// todo!("Foreign pointer write crossed a page boundary"); -// } -// -// let phys_page = space -// .translate(start_page) -// .expect("Address is not mapped in the target address space"); -// -// let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; -// virt_ptr.write_volatile(value); -// } -// -// unsafe fn validate_user_slice_mut<'a>( -// self: *mut Self, -// len: usize, -// space: &AddressSpace, -// ) -> Result<&'a mut [Self], Error> { -// let base = self as usize; -// let layout = Layout::array::(len).unwrap(); -// -// validate_user_align_size(base, &layout)?; -// validate_user_region(space, base, layout.size(), true)?; -// -// Ok(core::slice::from_raw_parts_mut(self, len)) -// } -// -// unsafe fn validate_user_slice<'a>( -// self: *const Self, -// len: usize, -// space: &AddressSpace, -// ) -> Result<&'a [Self], Error> { -// let base = self as usize; -// let layout = Layout::array::(len).unwrap(); -// -// validate_user_align_size(base, &layout)?; -// validate_user_region(space, base, layout.size(), false)?; -// -// Ok(core::slice::from_raw_parts(self, len)) -// } -// -// unsafe fn validate_user_mut<'a>( -// self: *mut Self, -// space: &AddressSpace, -// ) -> Result<&'a mut Self, Error> { -// let addr = self as usize; -// let layout = Layout::new::(); -// -// // Common validation -// validate_user_align_size(addr, &layout)?; -// -// // Validate that the pages covered by this address are mapped as writable by the process -// // TODO for CoW this may differ -// validate_user_region(space, addr, layout.size(), true)?; -// -// Ok(&mut *self) -// } -// -// unsafe fn validate_user_ptr<'a>( -// self: *const Self, -// space: &AddressSpace, -// ) -> Result<&'a Self, Error> { -// let addr = self as usize; -// let layout = Layout::new::(); -// -// // Common validation -// validate_user_align_size(addr, &layout)?; -// validate_user_region(space, addr, layout.size(), false)?; -// -// Ok(&*self) -// } -// } -// -// fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { -// // Explicitly disallow NULL -// if addr == 0 { -// return Err(Error::InvalidArgument); -// } -// // Validate alignment -// if addr % layout.align() != 0 { -// return Err(Error::InvalidArgument); -// } -// if addr + layout.size() > KERNEL_VIRT_OFFSET { -// todo!(); -// } -// -// Ok(()) -// } -// -// /// Validates access to given userspace memory region with given constraints -// pub fn validate_user_region( -// space: &AddressSpace, -// base: usize, -// len: usize, -// _need_write: bool, -// ) -> Result<(), Error> { -// if base + len > crate::mem::KERNEL_VIRT_OFFSET { -// panic!("Invalid argument"); -// } -// -// let aligned_start = base & !0xFFF; -// let aligned_end = (base + len + 0xFFF) & !0xFFF; -// -// for page in (aligned_start..aligned_end).step_by(0x1000) { -// // TODO check writability -// space.translate(page).ok_or(Error::InvalidArgument)?; -// } -// -// Ok(()) -// } +impl ForeignPointer for T { + unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { + // TODO check align + let addr = self as usize; + let start_page = addr & !0xFFF; + let end_page = (addr + size_of::() - 1) & !0xFFF; + let page_offset = addr & 0xFFF; + + if start_page != end_page { + todo!("Foreign pointer write crossed a page boundary"); + } + + let phys_page = space + .translate(start_page) + .expect("Address is not mapped in the target address space"); + + let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; + virt_ptr.write_volatile(value); + } + + unsafe fn validate_user_slice_mut<'a>( + self: *mut Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a mut [Self], Error> { + let base = self as usize; + let layout = Layout::array::(len).unwrap(); + + validate_user_align_size(base, &layout)?; + validate_user_region(space, base, layout.size(), true)?; + + Ok(core::slice::from_raw_parts_mut(self, len)) + } + + unsafe fn validate_user_slice<'a>( + self: *const Self, + len: usize, + space: &AddressSpace, + ) -> Result<&'a [Self], Error> { + let base = self as usize; + let layout = Layout::array::(len).unwrap(); + + validate_user_align_size(base, &layout)?; + validate_user_region(space, base, layout.size(), false)?; + + Ok(core::slice::from_raw_parts(self, len)) + } + + unsafe fn validate_user_mut<'a>( + self: *mut Self, + space: &AddressSpace, + ) -> Result<&'a mut Self, Error> { + let addr = self as usize; + let layout = Layout::new::(); + + // Common validation + validate_user_align_size(addr, &layout)?; + + // Validate that the pages covered by this address are mapped as writable by the process + // TODO for CoW this may differ + validate_user_region(space, addr, layout.size(), true)?; + + Ok(&mut *self) + } + + unsafe fn validate_user_ptr<'a>( + self: *const Self, + space: &AddressSpace, + ) -> Result<&'a Self, Error> { + let addr = self as usize; + let layout = Layout::new::(); + + // Common validation + validate_user_align_size(addr, &layout)?; + validate_user_region(space, addr, layout.size(), false)?; + + Ok(&*self) + } +} + +fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { + // Explicitly disallow NULL + if addr == 0 { + return Err(Error::InvalidArgument); + } + // Validate alignment + if addr % layout.align() != 0 { + return Err(Error::InvalidArgument); + } + if addr + layout.size() > KERNEL_VIRT_OFFSET { + todo!(); + } + + Ok(()) +} + +/// Validates access to given userspace memory region with given constraints +pub fn validate_user_region( + space: &AddressSpace, + base: usize, + len: usize, + _need_write: bool, +) -> Result<(), Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + + let aligned_start = base & !0xFFF; + let aligned_end = (base + len + 0xFFF) & !0xFFF; + + for page in (aligned_start..aligned_end).step_by(0x1000) { + // TODO check writability + space.translate(page).ok_or(Error::InvalidArgument)?; + } + + Ok(()) +} diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 4971dddc..8110ac90 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -4,7 +4,7 @@ use crate::util::StaticVector; use super::PhysicalMemoryRegion; -static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); +static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); /// Marks a region of physical memory as reserved. /// diff --git a/src/mem/table.rs b/src/mem/table.rs index 4e57cbd3..97c205cf 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -1,12 +1,22 @@ //! Virtual memory table interface use abi::error::Error; +use bitflags::bitflags; use cfg_if::cfg_if; cfg_if! { if #[cfg(target_arch = "aarch64")] { pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; + pub use crate::arch::x86_64::table::{AddressSpace, PageEntry, PageTable}; + } +} + +bitflags! { + #[derive(Clone, Copy)] + pub struct MapAttributes: u64 { + const USER_READ = 1 << 0; + const USER_WRITE = 1 << 1; + const NON_GLOBAL = 1 << 2; } } @@ -18,9 +28,11 @@ pub trait VirtualMemoryManager { &self, hint: Option, len: usize, - attrs: PageAttributes, + attrs: MapAttributes, ) -> Result; + fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error>; + /// Releases the virtual memory region from the address space and the pages it refers to fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error>; } diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 01f2b3fa..44d09dea 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -9,7 +9,7 @@ use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::mem::{ phys::{self, PageUsage}, - table::{AddressSpace, PageAttributes}, + table::{AddressSpace, MapAttributes, VirtualMemoryManager}, ConvertAddress, }; @@ -69,11 +69,11 @@ where // TODO check for crazy addresses here let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { - (0, 0) => PageAttributes::AP_BOTH_READONLY, - (_, 0) => PageAttributes::AP_BOTH_READWRITE, - (0, _) => PageAttributes::AP_BOTH_READONLY, - (_, _) => PageAttributes::AP_BOTH_READWRITE, - } | PageAttributes::NON_GLOBAL; + (0, 0) => MapAttributes::USER_READ, + (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + (0, _) => MapAttributes::USER_READ, + (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + } | MapAttributes::NON_GLOBAL; let dst_page_off = addr & 0xFFF; let dst_page_aligned = addr & !0xFFF; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 7b506315..78f9e577 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -2,18 +2,18 @@ use core::mem::size_of; use abi::error::Error; -use alloc::rc::Rc; +use alloc::{rc::Rc, string::String}; use vfs::FileRef; use crate::{ - arch::aarch64::context::TaskContext, + // arch::aarch64::context::TaskContext, mem::{ phys::{self, PageUsage}, - table::{AddressSpace, PageAttributes}, - ConvertAddress, + table::{AddressSpace, MapAttributes, VirtualMemoryManager}, + ConvertAddress, ForeignPointer, }, proc, - task::process::Process, + task::{process::Process, TaskContext}, }; fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { @@ -35,7 +35,7 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er space.map_page( virt, phys_page, - PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + MapAttributes::USER_READ | MapAttributes::USER_WRITE, // PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, )?; let write = unsafe { phys_page.virtualize() }; @@ -71,7 +71,12 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er Ok(()) } -fn setup_binary(space: AddressSpace, entry: usize, args: &[&str]) -> Result, Error> { +fn setup_binary>( + name: S, + space: AddressSpace, + entry: usize, + args: &[&str], +) -> Result, Error> { const USER_STACK_PAGES: usize = 8; let virt_stack_base = 0x3000000; @@ -83,7 +88,7 @@ fn setup_binary(space: AddressSpace, entry: usize, args: &[&str]) -> Result Result Result, Error> { +pub fn load_elf>( + name: S, + file: FileRef, + args: &[&str], +) -> Result, Error> { let space = AddressSpace::new_empty()?; let elf_entry = proc::elf::load_elf_from_file(&space, file)?; - setup_binary(space, elf_entry, args) + setup_binary(name, space, elf_entry, args) } diff --git a/src/proc/io.rs b/src/proc/io.rs index e91a5915..ecf03f6a 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -21,7 +21,10 @@ impl ProcessIo { /// Returns a file given descriptor refers to pub fn file(&self, fd: RawFd) -> Result { - self.files.get(&fd).cloned().ok_or(Error::InvalidFile) + debugln!("fd = {:?}", fd); + let res = self.files.get(&fd).cloned().unwrap(); + // .ok_or(Error::InvalidFile) + Ok(res) } /// Iterates over the file descriptors in [ProcessIo] and removes them if predicate is `false` diff --git a/src/proc/wait.rs b/src/proc/wait.rs index ad6cf382..0b95f3dd 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -123,22 +123,22 @@ impl Wait { queue_lock = self.queue.lock(); if let Some(deadline) = deadline { - let now = PLATFORM.timestamp_source().timestamp()?; + // let now = PLATFORM.timestamp_source().timestamp()?; - if now > deadline { - let mut cursor = queue_lock.cursor_front_mut(); + // if now > deadline { + // let mut cursor = queue_lock.cursor_front_mut(); - while let Some(item) = cursor.current() { - if item.id() == process.id() { - cursor.remove_current(); - return Err(Error::TimedOut); - } else { - cursor.move_next(); - } - } + // while let Some(item) = cursor.current() { + // if item.id() == process.id() { + // cursor.remove_current(); + // return Err(Error::TimedOut); + // } else { + // cursor.move_next(); + // } + // } - // Most likely the process was killed by a signal - } + // // Most likely the process was killed by a signal + // } } } } @@ -148,34 +148,36 @@ static TICK_LIST: IrqSafeSpinlock> = IrqSafeSpinlock::new(Li /// Suspends current task until given deadline pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { - static SLEEP_NOTIFY: Wait = Wait::new("sleep"); - let now = PLATFORM.timestamp_source().timestamp()?; - let deadline = now + timeout; + todo!(); + // static SLEEP_NOTIFY: Wait = Wait::new("sleep"); + // let now = PLATFORM.timestamp_source().timestamp()?; + // let deadline = now + timeout; - match SLEEP_NOTIFY.wait(Some(deadline)) { - // Just what we expected - Err(Error::TimedOut) => { - *remaining = Duration::ZERO; - Ok(()) - } + // match SLEEP_NOTIFY.wait(Some(deadline)) { + // // Just what we expected + // Err(Error::TimedOut) => { + // *remaining = Duration::ZERO; + // Ok(()) + // } - Ok(_) => panic!("This should not happen"), - Err(e) => Err(e), - } + // Ok(_) => panic!("This should not happen"), + // Err(e) => Err(e), + // } } /// Updates all pending timeouts and wakes up the tasks that have reached theirs pub fn tick(now: Duration) { - let mut list = TICK_LIST.lock(); - let mut cursor = list.cursor_front_mut(); + todo!(); + // let mut list = TICK_LIST.lock(); + // let mut cursor = list.cursor_front_mut(); - while let Some(item) = cursor.current() { - if now > item.deadline { - let t = cursor.remove_current().unwrap(); + // while let Some(item) = cursor.current() { + // if now > item.deadline { + // let t = cursor.remove_current().unwrap(); - t.process.enqueue_somewhere(); - } else { - cursor.move_next(); - } - } + // t.process.enqueue_somewhere(); + // } else { + // cursor.move_next(); + // } + // } } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 49b6c939..5d2219af 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -16,11 +16,12 @@ use yggdrasil_abi::{ use crate::{ fs, - mem::table::{PageAttributes, VirtualMemoryManager}, + mem::table::{MapAttributes, VirtualMemoryManager}, proc::{ self, io::ProcessIo, - wait::{self, PROCESS_EXIT_WAIT}, + wait::PROCESS_EXIT_WAIT, + // wait::{self, PROCESS_EXIT_WAIT}, }, sync::IrqSafeSpinlockGuard, task::{process::Process, ProcessId}, @@ -62,14 +63,15 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } SyscallFunction::Nanosleep => { - let seconds = args[0]; - let nanos = args[1] as u32; - let duration = Duration::new(seconds, nanos); - let mut remaining = Duration::ZERO; + todo!(); + // let seconds = args[0]; + // let nanos = args[1] as u32; + // let duration = Duration::new(seconds, nanos); + // let mut remaining = Duration::ZERO; - wait::sleep(duration, &mut remaining).unwrap(); + // wait::sleep(duration, &mut remaining).unwrap(); - Ok(0) + // Ok(0) } SyscallFunction::Exit => { let code = ExitCode::from(args[0] as i32); @@ -89,7 +91,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result space.allocate( None, len / 0x1000, - PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, ) } SyscallFunction::UnmapMemory => { @@ -140,11 +142,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result debugln!("run_with_io_at {:?}", at); let file = io.ioctx().open(at, path, opts, mode)?; - if proc.session_terminal().is_none() && - let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { - debugln!("Session terminal set for #{}: {}", proc.id(), path); - proc.set_session_terminal(node); - } + // if proc.session_terminal().is_none() && + // let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { + // debugln!("Session terminal set for #{}: {}", proc.id(), path); + // proc.set_session_terminal(node); + // } let fd = io.place_file(file)?; Ok(fd.0 as usize) @@ -177,16 +179,17 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result }) } SyscallFunction::Unmount => { - let options = arg_user_ref::(args[0] as usize)?; + todo!(); + // let options = arg_user_ref::(args[0] as usize)?; - run_with_io(|mut io| { - let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; - mountpoint.unmount_target()?; + // run_with_io(|mut io| { + // let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; + // mountpoint.unmount_target()?; - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); + // debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - Ok(0) - }) + // Ok(0) + // }) } SyscallFunction::OpenDirectory => { let at = arg_option_fd(args[0] as u32); @@ -215,71 +218,75 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result }) } SyscallFunction::CreateDirectory => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let _mode = FileMode::from(args[3] as u32); + todo!(); + // let at = arg_option_fd(args[0] as u32); + // let path = arg_user_str(args[1] as usize, args[2] as usize)?; + // let _mode = FileMode::from(args[3] as u32); - run_with_io_at(at, |at, mut io| { - let (parent, name) = abi::path::split_right(path); - let parent_node = io.ioctx().find(at, parent, true, true)?; - parent_node.create(name, VnodeKind::Directory)?; + // run_with_io_at(at, |at, mut io| { + // let (parent, name) = abi::path::split_right(path); + // let parent_node = io.ioctx().find(at, parent, true, true)?; + // parent_node.create(name, VnodeKind::Directory)?; - Ok(0) - }) + // Ok(0) + // }) } SyscallFunction::Remove => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let recurse = args[3] != 0; + todo!(); + // let at = arg_option_fd(args[0] as u32); + // let path = arg_user_str(args[1] as usize, args[2] as usize)?; + // let recurse = args[3] != 0; - run_with_io_at(at, |at, mut io| { - let node = io.ioctx().find(at, path, false, false)?; + // run_with_io_at(at, |at, mut io| { + // let node = io.ioctx().find(at, path, false, false)?; - if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { - todo!(); - } + // if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { + // todo!(); + // } - let parent = node.parent(); + // let parent = node.parent(); - parent.remove(node, recurse)?; + // parent.remove(node, recurse)?; - Ok(0) - }) + // Ok(0) + // }) } SyscallFunction::GetMetadata => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let buffer = arg_user_mut::>(args[3] as usize)?; - let follow = args[4] != 0; + todo!(); + // let at = arg_option_fd(args[0] as u32); + // let path = arg_user_str(args[1] as usize, args[2] as usize)?; + // let buffer = arg_user_mut::>(args[3] as usize)?; + // let follow = args[4] != 0; - run_with_io_at(at, |at, mut io| { - let node = if path.is_empty() { - at.ok_or(Error::InvalidArgument)? - } else { - io.ioctx().find(None, path, follow, true)? - }; + // run_with_io_at(at, |at, mut io| { + // let node = if path.is_empty() { + // at.ok_or(Error::InvalidArgument)? + // } else { + // io.ioctx().find(None, path, follow, true)? + // }; - let metadata = node.metadata()?; - buffer.write(metadata); + // let metadata = node.metadata()?; + // buffer.write(metadata); - Ok(0) - }) + // Ok(0) + // }) } SyscallFunction::Seek => { - let fd = RawFd(args[0] as u32); - let pos = SeekFrom::from(args[1]); + todo!(); + // let fd = RawFd(args[0] as u32); + // let pos = SeekFrom::from(args[1]); - run_with_io(|io| { - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); + // run_with_io(|io| { + // let file = io.file(fd)?; + // let mut file_borrow = file.borrow_mut(); - file_borrow.seek(pos).map(|v| v as usize) - }) + // file_borrow.seek(pos).map(|v| v as usize) + // }) } SyscallFunction::Spawn => { let options = arg_user_ref::(args[0] as usize)?; - // debugln!("Spawn {:#?}", options); + debugln!("Spawn {:#?}", options); let proc = Process::current(); run_with_io(|mut io| { @@ -287,7 +294,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Setup a new process from the file let file = node.open(OpenOptions::READ, FileMode::empty())?; - let child = proc::exec::load_elf(file, options.arguments)?; + let child = proc::exec::load_elf(options.program, file, options.arguments)?; let pid = child.id() as u32; // Inherit group and session from the creator @@ -335,17 +342,20 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } } SyscallFunction::SendSignal => { - let pid = args[0] as u32; - let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + todo!(); + // let pid = args[0] as u32; + // let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; - target.try_set_signal(signal)?; + // let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; + // target.try_set_signal(signal)?; - Ok(0) + // Ok(0) } SyscallFunction::SetSignalEntry => { let entry = args[0] as usize; let sp = args[1] as usize; + let proc = Process::current(); + warnln!("SetSignalEntry({}, {:#x}, {:#x})", proc.id(), entry, sp); Process::current().set_signal_entry(entry, sp); diff --git a/src/task/mod.rs b/src/task/mod.rs index 0756e545..3cd1122e 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -3,7 +3,7 @@ #![allow(dead_code)] use abi::error::Error; -use alloc::{rc::Rc, vec::Vec}; +use alloc::{rc::Rc, string::String, vec::Vec}; use cfg_if::cfg_if; cfg_if! { @@ -15,6 +15,7 @@ cfg_if! { } use crate::{ + kernel_main, mem::{ phys::{self, PageUsage}, table::AddressSpace, @@ -75,46 +76,16 @@ impl ProcessList { pub static PROCESSES: IrqSafeSpinlock = IrqSafeSpinlock::new(ProcessList::new()); /// Creates a new kernel-space process to execute a closure and queues it to some CPU -pub fn spawn_kernel_closure(f: F) -> Result<(), Error> { - let proc = Process::new_with_context(None, TaskContext::kernel_closure(f)?); +pub fn spawn_kernel_closure, F: Fn() + Send + 'static>( + name: S, + f: F, +) -> Result<(), Error> { + let proc = Process::new_with_context(name, None, TaskContext::kernel_closure(f)?); proc.enqueue_somewhere(); Ok(()) } -unsafe fn spawn_user_in_kernel(entry: usize, arg: usize) -> Result<(), Error> { - let stack = phys::alloc_pages_contiguous(4, PageUsage::Used)?; - let stack_pointer = stack + 4 * 0x1000; - - let space = AddressSpace::new_empty()?; - - let user_ctx = TaskContext::user( - entry, - arg, - space.physical_address(), - stack_pointer.virtualize(), - )?; - let proc = Process::new_with_context(Some(space), user_ctx); - - proc.enqueue_somewhere(); - - Ok(()) -} - -extern "C" fn f1(arg: usize) -> ! { - loop { - unsafe { - core::arch::asm!("syscall", in("rax") arg, options(att_syntax), clobber_abi("C")); - } - - for _ in 0..100000 { - unsafe { - core::arch::asm!("nop"); - } - } - } -} - /// Sets up CPU queues and gives them some processes to run pub fn init() -> Result<(), Error> { // XXX @@ -124,21 +95,7 @@ pub fn init() -> Result<(), Error> { // Create a queue for each CPU sched::init_queues(Vec::from_iter((0..cpu_count).map(|_| CpuQueue::new()))); - // Spawn kernel main task - unsafe { - spawn_user_in_kernel(f1 as usize, 0).unwrap(); - } - unsafe { - spawn_user_in_kernel(f1 as usize, 1).unwrap(); - } - - // spawn_kernel_closure(move || loop { - // debugln!("A"); - // for _ in 0..100000 { - // core::hint::spin_loop(); - // } - // })?; - + spawn_kernel_closure("[kmain]", kernel_main)?; // spawn_kernel_closure(move || loop { // debugln!("B"); // for _ in 0..100000 { diff --git a/src/task/process.rs b/src/task/process.rs index 95a53ed8..2a07d1fa 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,27 +1,45 @@ //! Process data structures use core::{ + mem::size_of, ops::Deref, sync::atomic::{AtomicU32, Ordering}, }; -// use aarch64_cpu::registers::DAIF; -use alloc::rc::Rc; +use abi::{ + arch::SavedFrame, + error::Error, + process::{ExitCode, Signal, SignalEntryData}, +}; +use alloc::{collections::VecDeque, rc::Rc, string::String}; use atomic_enum::atomic_enum; +use vfs::VnodeRef; use crate::{ - mem::table::AddressSpace, - // arch::aarch64::{context::TaskContext, cpu::Cpu, exception::ExceptionFrame}, - // mem::{table::AddressSpace, ForeignPointer}, - // proc::{ - // io::ProcessIo, - // wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, - // }, + mem::{table::AddressSpace, ForeignPointer}, + proc::{ + io::ProcessIo, + wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, + }, sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; use super::{sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; +pub trait TaskFrame { + fn store(&self) -> SavedFrame; + fn restore(&mut self, saved: &SavedFrame); + + fn set_return_value(&mut self, value: u64); + fn set_user_sp(&mut self, value: usize); + fn set_user_ip(&mut self, value: usize); + fn set_argument(&mut self, value: u64); + + fn argument(&self) -> u64; + fn user_sp(&self) -> usize; + fn user_ip(&self) -> usize; +} + /// Represents the states a process can be at some point in time #[atomic_enum] #[derive(PartialEq)] @@ -45,21 +63,21 @@ pub struct SignalEntry { struct ProcessInner { // XXX - // pending_wait: Option<&'static Wait>, - // wait_status: WaitStatus, - // exit_status: i32, + pending_wait: Option<&'static Wait>, + wait_status: WaitStatus, + exit_status: i32, - // session_id: ProcessId, - // group_id: ProcessId, - // session_terminal: Option, - - // signal_entry: Option, - // signal_stack: VecDeque, + session_id: ProcessId, + group_id: ProcessId, + session_terminal: Option, + signal_entry: Option, + signal_stack: VecDeque, } /// Process data and state structure pub struct Process { normal_context: TaskContext, + name: String, // Process state info id: OneTimeInit, @@ -67,8 +85,8 @@ pub struct Process { cpu_id: AtomicU32, space: Option, inner: IrqSafeSpinlock, - // /// I/O state of the task - // pub io: IrqSafeSpinlock, + /// I/O state of the task + pub io: IrqSafeSpinlock, } /// Guard type that provides [Process] operations only available for current processes @@ -80,40 +98,43 @@ impl Process { /// # Note /// /// Has side-effect of allocating a new PID for itself. - pub fn new_with_context(space: Option, normal_context: TaskContext) -> Rc { + pub fn new_with_context>( + name: S, + space: Option, + normal_context: TaskContext, + ) -> Rc { let this = Rc::new(Self { normal_context, id: OneTimeInit::new(), + name: name.into(), state: AtomicProcessState::new(ProcessState::Suspended), cpu_id: AtomicU32::new(0), space, inner: IrqSafeSpinlock::new(ProcessInner { // XXX - // pending_wait: None, - // wait_status: WaitStatus::Done, - // exit_status: 0, + pending_wait: None, + wait_status: WaitStatus::Done, + exit_status: 0, - // session_id: 0, - // group_id: 0, - // session_terminal: None, - - // signal_entry: None, - // signal_stack: VecDeque::new(), + session_id: 0, + group_id: 0, + session_terminal: None, + signal_entry: None, + signal_stack: VecDeque::new(), }), - // XXX - // io: IrqSafeSpinlock::new(ProcessIo::new()), + io: IrqSafeSpinlock::new(ProcessIo::new()), }); let id = unsafe { PROCESSES.lock().push(this.clone()) }; this.id.init(id); - // { - // let mut inner = this.inner.lock(); - // inner.session_id = id; - // inner.group_id = id; - // } + { + let mut inner = this.inner.lock(); + inner.session_id = id; + inner.group_id = id; + } this } @@ -128,6 +149,10 @@ impl Process { *self.id.get() } + pub fn name(&self) -> &str { + self.name.as_str() + } + /// Returns the state of the process. /// /// # Note @@ -163,35 +188,35 @@ impl Process { } // XXX - // /// Replaces the task's session terminal device with another one - // pub fn set_session_terminal(&self, terminal: VnodeRef) { - // self.inner.lock().session_terminal.replace(terminal); - // } + /// Replaces the task's session terminal device with another one + pub fn set_session_terminal(&self, terminal: VnodeRef) { + self.inner.lock().session_terminal.replace(terminal); + } - // /// Removes the task's current terminal - // pub fn clear_session_terminal(&self) -> Option { - // self.inner.lock().session_terminal.take() - // } + /// Removes the task's current terminal + pub fn clear_session_terminal(&self) -> Option { + self.inner.lock().session_terminal.take() + } - // /// Returns the current terminal of the task - // pub fn session_terminal(&self) -> Option { - // self.inner.lock().session_terminal.clone() - // } + /// Returns the current terminal of the task + pub fn session_terminal(&self) -> Option { + self.inner.lock().session_terminal.clone() + } - // /// Sets the session ID of the task - // pub fn set_session_id(&self, sid: ProcessId) { - // self.inner.lock().session_id = sid; - // } + /// Sets the session ID of the task + pub fn set_session_id(&self, sid: ProcessId) { + self.inner.lock().session_id = sid; + } - // /// Sets the process group ID of the task - // pub fn set_group_id(&self, gid: ProcessId) { - // self.inner.lock().group_id = gid; - // } + /// Sets the process group ID of the task + pub fn set_group_id(&self, gid: ProcessId) { + self.inner.lock().group_id = gid; + } - // /// Returns the process group ID of the task - // pub fn group_id(&self) -> ProcessId { - // self.inner.lock().group_id - // } + /// Returns the process group ID of the task + pub fn group_id(&self) -> ProcessId { + self.inner.lock().group_id + } /// Selects a suitable CPU queue and submits the process for execution. /// @@ -244,44 +269,43 @@ impl Process { ProcessState::Suspended => (), ProcessState::Terminated => panic!("Process is terminated"), ProcessState::Running => { - todo!(); - // let cpu_id = self.cpu_id.load(Ordering::Acquire); - // let local_cpu_id = Cpu::local_id(); - // let queue = Cpu::local().queue(); + let cpu_id = self.cpu_id.load(Ordering::Acquire); + let local_cpu_id = Cpu::local_id(); + let queue = Cpu::local().queue(); - // if cpu_id == local_cpu_id { - // // Suspending a process running on local CPU - // unsafe { queue.yield_cpu() } - // } else { - // todo!(); - // } + if cpu_id == local_cpu_id { + // Suspending a process running on local CPU + unsafe { queue.yield_cpu() } + } else { + todo!(); + } } } } - // /// Returns current wait status of the task - // pub fn wait_status(&self) -> WaitStatus { - // self.inner.lock().wait_status - // } + /// Returns current wait status of the task + pub fn wait_status(&self) -> WaitStatus { + self.inner.lock().wait_status + } - // /// Updates the wait status for the task. - // /// - // /// # Safety - // /// - // /// This function is only meant to be called on waiting tasks, otherwise atomicity is not - // /// guaranteed. - // pub unsafe fn set_wait_status(&self, status: WaitStatus) { - // self.inner.lock().wait_status = status; - // } + /// Updates the wait status for the task. + /// + /// # Safety + /// + /// This function is only meant to be called on waiting tasks, otherwise atomicity is not + /// guaranteed. + pub unsafe fn set_wait_status(&self, status: WaitStatus) { + self.inner.lock().wait_status = status; + } - // /// Returns an exit code if the process exited, [None] if it didn't - // pub fn get_exit_status(&self) -> Option { - // if self.state() == ProcessState::Terminated { - // Some(ExitCode::from(self.inner.lock().exit_status)) - // } else { - // None - // } - // } + /// Returns an exit code if the process exited, [None] if it didn't + pub fn get_exit_status(&self) -> Option { + if self.state() == ProcessState::Terminated { + Some(ExitCode::from(self.inner.lock().exit_status)) + } else { + None + } + } /// Returns the [Process] currently executing on local CPU, None if idling. pub fn get_current() -> Option { @@ -301,70 +325,68 @@ impl Process { } // /// Handles the cleanup of an exited process - // pub fn handle_exit(&self) { - // // Queue lock is still held - // assert_eq!(self.state(), ProcessState::Terminated); + pub fn handle_exit(&self) { + // Queue lock is still held + assert_eq!(self.state(), ProcessState::Terminated); - // // TODO cancel Wait if a process was killed while suspended? - // { - // let inner = self.inner.lock(); - // let exit_status = ExitCode::from(inner.exit_status); - // debugln!( - // "Handling exit of #{} with status {:?}", - // self.id(), - // exit_status - // ); + // TODO cancel Wait if a process was killed while suspended? + { + let inner = self.inner.lock(); + let exit_status = ExitCode::from(inner.exit_status); + debugln!( + "Handling exit of #{} with status {:?}", + self.id(), + exit_status + ); - // // TODO cleanup address space - // // if let Some(space) = self.get_address_space() { - // // } + // TODO cleanup address space + // if let Some(space) = self.get_address_space() { + // } - // self.io.lock().handle_exit(); - // } + self.io.lock().handle_exit(); + } - // // Notify any waiters we're done - // PROCESS_EXIT_WAIT.wakeup_all(); - // } + // Notify any waiters we're done + PROCESS_EXIT_WAIT.wakeup_all(); + } - // /// Raises an asynchronous signal for the target process - // pub fn try_set_signal(self: &Rc, signal: Signal) -> Result<(), Error> { - // { - // let mut inner = self.inner.lock(); - // inner.wait_status = WaitStatus::Interrupted; - // inner.signal_stack.push_back(signal); - // } + /// Raises a signal for the currentprocess + pub fn raise_signal(self: &Rc, signal: Signal) { + { + let mut inner = self.inner.lock(); + inner.wait_status = WaitStatus::Interrupted; + inner.signal_stack.push_back(signal); + } - // if self.state() == ProcessState::Suspended { - // self.clone().enqueue_somewhere(); - // } + if self.state() == ProcessState::Suspended { + self.clone().enqueue_somewhere(); + } + } - // Ok(()) - // } + /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. + pub fn inherit(&self, parent: &Rc) -> Result<(), Error> { + let mut our_inner = self.inner.lock(); + let their_inner = parent.inner.lock(); - // /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. - // pub fn inherit(&self, parent: &Rc) -> Result<(), Error> { - // let mut our_inner = self.inner.lock(); - // let their_inner = parent.inner.lock(); + our_inner.session_id = their_inner.session_id; + our_inner.group_id = their_inner.group_id; + our_inner.session_terminal = their_inner.session_terminal.clone(); - // our_inner.session_id = their_inner.session_id; - // our_inner.group_id = their_inner.group_id; - // our_inner.session_terminal = their_inner.session_terminal.clone(); + Ok(()) + } - // Ok(()) - // } - - // /// Raises a signal for the specified process group - // pub fn signal_group(group_id: ProcessId, signal: Signal) { - // let processes = PROCESSES.lock(); - // for (_, proc) in processes.data.iter() { - // let inner = proc.inner.lock(); - // if proc.state() != ProcessState::Terminated && inner.group_id == group_id { - // debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); - // drop(inner); - // proc.try_set_signal(signal).unwrap(); - // } - // } - // } + /// Raises a signal for the specified process group + pub fn signal_group(group_id: ProcessId, signal: Signal) { + let processes = PROCESSES.lock(); + for (_, proc) in processes.data.iter() { + let inner = proc.inner.lock(); + if proc.state() != ProcessState::Terminated && inner.group_id == group_id { + debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); + drop(inner); + proc.raise_signal(signal); + } + } + } } impl Drop for Process { @@ -385,94 +407,96 @@ impl CurrentProcess { Self(inner) } - // /// Sets up a pending wait for the process. - // /// - // /// # Safety - // /// - // /// This function is only meant to be called in no-IRQ context and when caller can guarantee - // /// the task won't get scheduled to a CPU in such state. - // pub unsafe fn setup_wait(&self, wait: &'static Wait) { - // let mut inner = self.inner.lock(); - // inner.pending_wait.replace(wait); - // inner.wait_status = WaitStatus::Pending; - // } + /// Sets up a pending wait for the process. + /// + /// # Safety + /// + /// This function is only meant to be called in no-IRQ context and when caller can guarantee + /// the task won't get scheduled to a CPU in such state. + pub unsafe fn setup_wait(&self, wait: &'static Wait) { + let mut inner = self.inner.lock(); + inner.pending_wait.replace(wait); + inner.wait_status = WaitStatus::Pending; + } - // /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. - // /// - // /// # Safety - // /// - // /// This function is only meant to be called right before returning from an userspace - // /// exception handler. - // pub unsafe fn handle_signal(&self, frame: &mut ExceptionFrame) { - // let mut inner = self.inner.lock(); - // if let Some(signal) = inner.signal_stack.pop_front() { - // let Some(entry) = inner.signal_entry.clone() else { - // todo!(); - // }; + /// Configures signal entry information for the process + pub fn set_signal_entry(&self, entry: usize, stack: usize) { + let mut inner = self.inner.lock(); + inner.signal_entry.replace(SignalEntry { entry, stack }); + } - // debugln!( - // "Enter signal handler from: pc={:#x}, sp={:#x}", - // frame.elr_el1, - // frame.sp_el0 - // ); + /// Terminate the current process + pub fn exit(&self, status: ExitCode) { + self.inner.lock().exit_status = status.into(); + let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); + assert_eq!(current_state, ProcessState::Running); + infoln!("Process {} exited with code {:?}", self.id(), status); - // // TODO check if really in a syscall, lol - // let syscall_return = -(u32::from(Error::Interrupted) as isize); - // frame.r[0] = syscall_return as u64; + match current_state { + ProcessState::Suspended => { + todo!(); + } + ProcessState::Ready => todo!(), + ProcessState::Running => { + self.handle_exit(); + unsafe { Cpu::local().queue().yield_cpu() } + } + ProcessState::Terminated => todo!(), + } + } - // // Setup signal frame - // let usp = (entry.stack - size_of::()) & !0xF; - // let frame_ptr = usp as *mut SignalEntryData; + /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. + /// + /// # Safety + /// + /// This function is only meant to be called right before returning from an userspace + /// exception handler. + pub unsafe fn handle_signal(&self, frame: &mut F) { + let mut inner = self.inner.lock(); + if let Some(signal) = inner.signal_stack.pop_front() { + let Some(entry) = inner.signal_entry.clone() else { + todo!(); + }; - // let saved_frame = frame.to_saved_frame(); - // frame_ptr.write_foreign_volatile( - // self.address_space(), - // SignalEntryData { - // signal, - // frame: saved_frame, - // }, - // ); + debugln!( + "Enter signal handler from: pc={:#x}, sp={:#x}", + frame.user_ip(), + frame.user_sp() + ); - // // Setup return to signal handler - // debugln!( - // "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", - // entry.entry, - // usp, - // entry.stack - // ); - // frame.sp_el0 = usp as _; - // frame.elr_el1 = entry.entry as _; + // TODO check if really in a syscall, lol + let syscall_return = -(u32::from(Error::Interrupted) as isize); + frame.set_return_value(syscall_return as u64); - // // Pass the frame pointer as an argument to signal handler entry - // frame.r[0] = usp as _; - // } - // } + // Setup signal frame + let usp = ((entry.stack - size_of::()) & !0xF) + - TaskContext::SIGNAL_STACK_EXTRA_ALIGN; + let frame_ptr = usp as *mut SignalEntryData; - // /// Configures signal entry information for the process - // pub fn set_signal_entry(&self, entry: usize, stack: usize) { - // let mut inner = self.inner.lock(); - // inner.signal_entry.replace(SignalEntry { entry, stack }); - // } + let saved_frame = frame.store(); + frame_ptr.write_foreign_volatile( + self.address_space(), + SignalEntryData { + signal, + frame: saved_frame, + }, + ); - // /// Terminate the current process - // pub fn exit(&self, status: ExitCode) { - // self.inner.lock().exit_status = status.into(); - // let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); - // assert_eq!(current_state, ProcessState::Running); - // infoln!("Process {} exited with code {:?}", self.id(), status); + // Setup return to signal handler + debugln!( + "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", + entry.entry, + usp, + entry.stack + ); - // match current_state { - // ProcessState::Suspended => { - // todo!(); - // } - // ProcessState::Ready => todo!(), - // ProcessState::Running => { - // self.handle_exit(); - // unsafe { Cpu::local().queue().yield_cpu() } - // } - // ProcessState::Terminated => todo!(), - // } - // } + frame.set_user_sp(usp); + frame.set_user_ip(entry.entry); + + // Pass the frame pointer as an argument to signal handler entry + frame.set_argument(usp as _); + } + } } impl Deref for CurrentProcess { diff --git a/src/task/sched.rs b/src/task/sched.rs index fa277024..4c326bcd 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -6,6 +6,7 @@ use cfg_if::cfg_if; use crate::{ // arch::aarch64::{context::TaskContext, cpu::Cpu}, + arch::{Architecture, ArchitectureImpl}, sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, util::OneTimeInit, }; @@ -124,6 +125,7 @@ impl CpuQueue { /// /// Only meant to be called from [crate::task::enter()] function. pub unsafe fn enter(&self) -> ! { + assert!(ArchitectureImpl::interrupt_mask()); // Start from idle thread to avoid having a Rc stuck here without getting dropped // let t = CNTPCT_EL0.get(); // self.lock().stats.measure_time = t; @@ -137,6 +139,7 @@ impl CpuQueue { /// The function is only meant to be called from kernel threads (e.g. if they want to yield /// CPU execution to wait for something) or interrupt handlers. pub unsafe fn yield_cpu(&self) { + assert!(ArchitectureImpl::interrupt_mask()); let mut inner = self.inner.lock(); // let t = CNTPCT_EL0.get(); @@ -178,12 +181,7 @@ impl CpuQueue { // log_print_raw!(crate::debug::LogLevel::Info, "{}: ", Cpu::local_id()); // if let Some(from) = current.as_ref() { - // log_print_raw!( - // crate::debug::LogLevel::Info, - // "{} ({:?})", - // from.id(), - // from.pending_signal() - // ); + // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id(),); // } else { // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); // } @@ -191,12 +189,7 @@ impl CpuQueue { // log_print_raw!(crate::debug::LogLevel::Info, " -> "); // if let Some(to) = next.as_ref() { - // log_print_raw!( - // crate::debug::LogLevel::Info, - // "{} ({:?})", - // to.id(), - // to.pending_signal() - // ); + // log_print_raw!(crate::debug::LogLevel::Info, "{}", to.id(),); // } else { // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); // } From da911262939d5ad558574b84d4d188ed04e6c27c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 2 Aug 2023 20:43:21 +0300 Subject: [PATCH 029/211] ***: fix all warnings and clippy stuff --- Cargo.toml | 2 +- src/arch/aarch64/context.rs | 19 +- src/arch/aarch64/exception.rs | 59 +++-- src/arch/aarch64/gic/gicd.rs | 1 + src/arch/aarch64/gic/mod.rs | 104 +++++---- src/arch/aarch64/intrinsics.rs | 13 -- src/arch/aarch64/mod.rs | 4 +- src/arch/aarch64/plat_qemu/mod.rs | 43 ++-- src/arch/aarch64/table.rs | 30 ++- src/arch/aarch64/timer.rs | 4 +- src/arch/mod.rs | 22 +- src/arch/x86_64/apic/mod.rs | 16 +- src/arch/x86_64/context.rs | 8 +- src/arch/x86_64/exception.rs | 287 ++++++++++--------------- src/arch/x86_64/intrinsics.rs | 12 -- src/arch/x86_64/peripherals/ps2/mod.rs | 4 +- src/arch/x86_64/syscall.rs | 25 ++- src/arch/x86_64/table/mod.rs | 13 +- src/debug.rs | 13 +- src/device/input.rs | 5 + src/device/serial/pl011.rs | 4 +- src/device/tty.rs | 8 +- src/main.rs | 23 +- src/mem/table.rs | 5 + src/proc/exec.rs | 2 + src/proc/wait.rs | 10 +- src/syscall/mod.rs | 139 ++++++------ src/task/mod.rs | 7 - src/task/process.rs | 20 +- src/task/sched.rs | 4 +- 30 files changed, 467 insertions(+), 439 deletions(-) delete mode 100644 src/arch/aarch64/intrinsics.rs diff --git a/Cargo.toml b/Cargo.toml index b67152c5..805002a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ cfg-if = "1.0.0" bitmap-font = { version = "0.3.0" } embedded-graphics = "0.8.0" git-version = "0.3.5" -acpi = "4.1.1" [dependencies.elf] version = "0.7.2" @@ -35,3 +34,4 @@ aarch64-cpu = "9.3.1" [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } aml = "0.16.4" +acpi = "4.1.1" diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 1bcc0e55..ab0e2370 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -1,12 +1,15 @@ //! AArch64-specific task context implementation use core::{arch::global_asm, cell::UnsafeCell}; -use abi::error::Error; +use abi::{error::Error, process::ExitCode}; use alloc::boxed::Box; -use crate::mem::{ - phys::{self, PageUsage}, - ConvertAddress, +use crate::{ + mem::{ + phys::{self, PageUsage}, + ConvertAddress, + }, + task::process::Process, }; struct StackBuilder { @@ -82,6 +85,11 @@ impl StackBuilder { } impl TaskContext { + /// Extra number of bytes to offset the user stack pointer + pub const USER_STACK_EXTRA_ALIGN: usize = 0; + /// Extra number of bytes to offset the signal stack pointer + pub const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; + /// Constructs a kernel thread context. For a more convenient way of constructing kernel /// processes, see [TaskContext::kernel_closure()]. pub fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { @@ -114,7 +122,8 @@ impl TaskContext { extern "C" fn closure_wrapper(closure_addr: usize) -> ! { let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; closure(); - todo!("Process termination"); + Process::current().exit(ExitCode::SUCCESS); + unreachable!(); } let closure = Box::new(f); diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 546e4145..5ebf8e0b 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -3,7 +3,8 @@ use core::{arch::global_asm, fmt}; use aarch64_cpu::registers::{ELR_EL1, ESR_EL1, FAR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; use abi::{ - process::{SavedFrame, Signal, SignalEntryData}, + arch::SavedFrame, + process::{Signal, SignalEntryData}, syscall::SyscallFunction, }; use tock_registers::interfaces::{Readable, Writeable}; @@ -11,10 +12,10 @@ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ arch::{aarch64::cpu::Cpu, CpuMessage, PLATFORM}, debug::LogLevel, - device::{interrupt::IrqContext, platform::Platform}, + device::interrupt::IrqContext, panic::panic_secondary, syscall::raw_syscall_handler, - task::process::Process, + task::process::{Process, TaskFrame}, }; /// Struct for register values saved when taking an exception @@ -32,9 +33,8 @@ pub struct ExceptionFrame { // ... } -impl ExceptionFrame { - /// Saves an userspace execution context into a frame "snapshot" - pub fn to_saved_frame(&self) -> SavedFrame { +impl TaskFrame for ExceptionFrame { + fn store(&self) -> SavedFrame { SavedFrame { gp_regs: self.r, spsr_el1: self.spsr_el1, @@ -43,12 +43,39 @@ impl ExceptionFrame { } } - /// Restores an userspace execution context from its frame "snapshot" - pub fn restore_from_signal(&mut self, frame: &SavedFrame) { - self.r = frame.gp_regs; - self.spsr_el1 = frame.spsr_el1; - self.elr_el1 = frame.elr_el1; - self.sp_el0 = frame.sp_el0; + fn restore(&mut self, saved: &SavedFrame) { + self.r = saved.gp_regs; + self.spsr_el1 = saved.spsr_el1; + self.elr_el1 = saved.elr_el1; + self.sp_el0 = saved.sp_el0; + } + + fn argument(&self) -> u64 { + self.r[0] + } + + fn user_ip(&self) -> usize { + self.elr_el1 as _ + } + + fn user_sp(&self) -> usize { + self.sp_el0 as _ + } + + fn set_argument(&mut self, value: u64) { + self.r[0] = value; + } + + fn set_return_value(&mut self, value: u64) { + self.r[0] = value; + } + + fn set_user_ip(&mut self, value: usize) { + self.elr_el1 = value as _; + } + + fn set_user_sp(&mut self, value: usize) { + self.sp_el0 = value as _; } } @@ -217,7 +244,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { 0b111100 => { let proc = Process::current(); warnln!("Process #{} hit a breakpoint", proc.id()); - proc.try_set_signal(Signal::Aborted).ok(); + proc.raise_signal(Signal::Aborted); } _ => { let iss = esr_el1 & 0x1FFFFFF; @@ -231,7 +258,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { FAR_EL1.get() ); - proc.try_set_signal(Signal::MemoryAccessViolation).ok(); + proc.raise_signal(Signal::MemoryAccessViolation); return; } @@ -245,7 +272,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { fn irq_common() { unsafe { let ic = IrqContext::new(); - PLATFORM.interrupt_controller().handle_pending_irqs(&ic); + PLATFORM.gic.handle_pending_irqs(&ic); } } @@ -258,7 +285,7 @@ unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) { saved_data.frame.sp_el0 ); - frame.restore_from_signal(&saved_data.frame); + frame.restore(&saved_data.frame); } pub(super) fn ipi_handler(msg: Option) { diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index e428f7ae..5ed010cd 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -96,6 +96,7 @@ impl Gicd { } } + #[allow(dead_code)] pub unsafe fn set_sgir(&self, target: IpiDeliveryTarget, interrupt_id: u64) { assert_eq!(interrupt_id & !0xF, 0); let value = match target { diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index d6461f96..23aa6d36 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -1,14 +1,11 @@ //! ARM Generic Interrupt Controller v2 driver -use core::sync::atomic::Ordering; -use aarch64_cpu::asm::barrier; use abi::error::Error; use spinning_top::Spinlock; use crate::{ - arch::CpuMessage, device::{ - interrupt::{InterruptController, InterruptSource, IpiDeliveryTarget}, + interrupt::{ExternalInterruptController, InterruptSource, IrqContext}, Device, }, mem::device::{DeviceMemory, DeviceMemoryIo}, @@ -17,7 +14,7 @@ use crate::{ use self::{gicc::Gicc, gicd::Gicd}; -use super::{cpu::Cpu, exception::ipi_handler, smp::CPU_COUNT}; +use super::{cpu::Cpu, exception::ipi_handler}; const MAX_IRQ: usize = 300; const IPI_VECTOR: u64 = 1; @@ -82,7 +79,7 @@ impl Device for Gic { } } -impl InterruptController for Gic { +impl ExternalInterruptController for Gic { type IrqNumber = IrqNumber; fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { @@ -90,34 +87,6 @@ impl InterruptController for Gic { Ok(()) } - fn handle_pending_irqs<'irq>(&'irq self, ic: &crate::device::interrupt::IrqContext<'irq>) { - let gicc = self.gicc.get(); - let irq_number = gicc.pending_irq_number(ic); - if irq_number >= MAX_IRQ { - return; - } - - gicc.clear_irq(irq_number, ic); - - if irq_number as u64 == IPI_VECTOR { - // IPI received - let msg = Cpu::local().get_ipi(); - ipi_handler(msg); - return; - } - - { - let table = self.irq_table.lock(); - match table[irq_number] { - None => panic!("No IRQ handler registered for irq{}", irq_number), - Some(handler) => { - drop(table); - handler.handle_irq().expect("IRQ handler failed"); - } - } - } - } - fn register_handler( &self, irq: Self::IrqNumber, @@ -135,28 +104,28 @@ impl InterruptController for Gic { Ok(()) } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { - // TODO message queue insertion should be moved - match target { - IpiDeliveryTarget::AllExceptLocal => { - let local = Cpu::local_id(); - for i in 0..CPU_COUNT.load(Ordering::Acquire) { - if i != local as usize { - Cpu::push_ipi_queue(i as u32, msg); - } - } - } - IpiDeliveryTarget::Specified(_) => todo!(), - } + // unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { + // // TODO message queue insertion should be moved + // match target { + // IpiDeliveryTarget::AllExceptLocal => { + // let local = Cpu::local_id(); + // for i in 0..CPU_COUNT.load(Ordering::Acquire) { + // if i != local as usize { + // Cpu::push_ipi_queue(i as u32, msg); + // } + // } + // } + // IpiDeliveryTarget::Specified(_) => todo!(), + // } - // Issue a memory barrier - barrier::dsb(barrier::ISH); - barrier::isb(barrier::SY); + // // Issue a memory barrier + // barrier::dsb(barrier::ISH); + // barrier::isb(barrier::SY); - self.gicd.get().set_sgir(target, IPI_VECTOR); + // self.gicd.get().set_sgir(target, IPI_VECTOR); - Ok(()) - } + // Ok(()) + // } } impl Gic { @@ -184,4 +153,33 @@ impl Gic { self.gicc.get().init(); Ok(()) } + + /// Handles a single pending IRQ (if there're many, the CPU will just get here again) + pub fn handle_pending_irqs<'irq>(&'irq self, ic: &IrqContext<'irq>) { + let gicc = self.gicc.get(); + let irq_number = gicc.pending_irq_number(ic); + if irq_number >= MAX_IRQ { + return; + } + + gicc.clear_irq(irq_number, ic); + + if irq_number as u64 == IPI_VECTOR { + // IPI received + let msg = Cpu::local().get_ipi(); + ipi_handler(msg); + return; + } + + { + let table = self.irq_table.lock(); + match table[irq_number] { + None => panic!("No IRQ handler registered for irq{}", irq_number), + Some(handler) => { + drop(table); + handler.handle_irq().expect("IRQ handler failed"); + } + } + } + } } diff --git a/src/arch/aarch64/intrinsics.rs b/src/arch/aarch64/intrinsics.rs deleted file mode 100644 index 66d1b8cd..00000000 --- a/src/arch/aarch64/intrinsics.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Intrinsic helper functions for AArch64 platforms - -/// Returns an absolute address to the given symbol -#[macro_export] -macro_rules! absolute_address { - ($sym:expr) => {{ - let mut _x: usize; - unsafe { - core::arch::asm!("ldr {0}, ={1}", out(reg) _x, sym $sym); - } - _x - }}; -} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index a8ca6e3c..0ddc263e 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -30,8 +30,6 @@ use self::{ table::{init_fixed_tables, KERNEL_TABLES}, }; -pub mod intrinsics; - pub mod plat_qemu; pub mod boot; @@ -205,7 +203,7 @@ pub fn kernel_main(dtb_phys: usize) -> ! { AArch64::set_interrupt_mask(true); ARCHITECTURE.init_device_tree(dtb_phys); - PLATFORM.init_primary_serial(); + PLATFORM.init_primary_debug_sink(); } // Setup debugging functions debug::init(); diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs index a6420b97..0750633c 100644 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -4,11 +4,11 @@ use abi::error::Error; use tock_registers::interfaces::Writeable; use crate::{ + debug::DebugSink, device::{ - interrupt::{InterruptController, InterruptSource}, + interrupt::{ExternalInterruptController, InterruptSource}, platform::Platform, - serial::{pl011::Pl011, SerialDevice}, - timer::TimestampSource, + serial::pl011::Pl011, Device, }, fs::devfs::{self, CharDeviceType}, @@ -21,7 +21,8 @@ use super::{ /// AArch64 "virt" platform implementation pub struct QemuPlatform { - gic: Gic, + /// ... + pub gic: Gic, pl011: Pl011, local_timer: ArmTimer, } @@ -52,25 +53,39 @@ impl Platform for QemuPlatform { Ok(()) } - unsafe fn init_primary_serial(&self) { + unsafe fn init_primary_debug_sink(&self) { self.pl011.init().ok(); } + fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { + Some(&self.pl011) + } + + fn interrupt_controller( + &self, + ) -> &dyn ExternalInterruptController { + &self.gic + } + + // unsafe fn init_primary_serial(&self) { + // self.pl011.init().ok(); + // } + fn name(&self) -> &'static str { "qemu" } - fn primary_serial(&self) -> Option<&dyn SerialDevice> { - Some(&self.pl011) - } + // fn primary_serial(&self) -> Option<&dyn SerialDevice> { + // Some(&self.pl011) + // } - fn interrupt_controller(&self) -> &dyn InterruptController { - &self.gic - } + // fn interrupt_controller(&self) -> &dyn InterruptController { + // &self.gic + // } - fn timestamp_source(&self) -> &dyn TimestampSource { - &self.local_timer - } + // fn timestamp_source(&self) -> &dyn TimestampSource { + // &self.local_timer + // } } /// AArch64 "virt" platform diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index 57dcc78a..01e6156a 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -12,7 +12,9 @@ use tock_registers::interfaces::Readable; use crate::mem::{ phys::{self, PageUsage}, - table::{EntryLevel, NextPageTable, VirtualMemoryManager}, + table::{ + EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, + }, ConvertAddress, KERNEL_VIRT_OFFSET, }; @@ -306,12 +308,27 @@ impl FixedTables { } } +impl From for PageAttributes { + fn from(value: MapAttributes) -> Self { + let mut res = Self::empty(); + if value.contains(MapAttributes::USER_WRITE) { + res |= PageAttributes::AP_BOTH_READWRITE; + } else { + res |= PageAttributes::AP_BOTH_READONLY; + } + if value.contains(MapAttributes::NON_GLOBAL) { + res |= PageAttributes::NON_GLOBAL; + } + res + } +} + impl VirtualMemoryManager for AddressSpace { fn allocate( &self, hint: Option, len: usize, - attrs: PageAttributes, + attrs: MapAttributes, ) -> Result { assert_eq!(DAIF.read(DAIF::I), 1); @@ -356,6 +373,10 @@ impl VirtualMemoryManager for AddressSpace { Ok(()) } + + fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> { + self.write_entry(virt, PageEntry::page(phys, attrs.into()), true) + } } impl AddressSpace { @@ -412,11 +433,6 @@ impl AddressSpace { Ok(()) } - /// Inserts a single 4KiB virt -> phys mapping into the address apce - pub fn map_page(&self, virt: usize, phys: usize, attrs: PageAttributes) -> Result<(), Error> { - self.write_entry(virt, PageEntry::page(phys, attrs), true) - } - /// Returns the physical address of the address space (to be used in a TTBRn_ELx) pub fn physical_address(&self) -> usize { unsafe { (self.l1 as usize).physicalize() | ((self.asid as usize) << 48) } diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 2d8622fd..2f010428 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -44,7 +44,7 @@ impl TimestampSource for ArmTimer { } impl InterruptSource for ArmTimer { - fn handle_irq(&self) -> Result<(), Error> { + fn handle_irq(&self) -> Result { CNTP_TVAL_EL0.set(TICK_INTERVAL); let t = self.timestamp()?; @@ -55,7 +55,7 @@ impl InterruptSource for ArmTimer { Cpu::local().queue().yield_cpu(); } - Ok(()) + Ok(true) } unsafe fn init_irq(&'static self) -> Result<(), Error> { diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 122a746d..cdf04700 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -3,18 +3,32 @@ use abi::error::Error; use cfg_if::cfg_if; +/// Returns an absolute address to the given symbol +#[macro_export] +macro_rules! absolute_address { + ($sym:expr) => {{ + let mut _x: usize; + #[cfg(target_arch = "aarch64")] + unsafe { + core::arch::asm!("ldr {0}, ={1}", out(reg) _x, sym $sym); + } + #[cfg(target_arch = "x86_64")] + unsafe { + core::arch::asm!("movabsq ${1}, {0}", out(reg) _x, sym $sym, options(att_syntax)); + } + _x + }}; +} + cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod aarch64; - pub use aarch64::intrinsics::absolute_address; pub use aarch64::plat_qemu::{QemuPlatform as PlatformImpl, PLATFORM}; pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; - use abi::error::Error; - use cfg_if::cfg_if; + } else if #[cfg(target_arch = "x86_64")] { - #[macro_use] pub mod x86_64; pub use x86_64::{ diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 5608cded..4ac72820 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -8,7 +8,7 @@ use crate::{ task::process::Process, }; -use super::exception::{self, ExceptionFrame, IrqFrame}; +use super::exception::{self, IrqFrame}; pub mod ioapic; pub mod local; @@ -82,15 +82,17 @@ fn apic_irq_inner(vector: u32) { } } -/// Main handler for APIC interrupts -pub extern "C" fn __x86_64_apic_irq_handler(vector: u32, frame: *mut IrqFrame) { - let frame = unsafe { &mut *frame }; +/// Main handler for APIC interrupts. +/// +/// # Safety +/// +/// Only meant to be called from the assembly irq vector. +pub unsafe extern "C" fn __x86_64_apic_irq_handler(vector: u32, frame: *mut IrqFrame) { + let frame = &mut *frame; apic_irq_inner(vector); if let Some(process) = Process::get_current() { - unsafe { - process.handle_signal(frame); - } + process.handle_signal(frame); } } diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index f25f17ad..3af8be5c 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -1,7 +1,7 @@ //! x86-64 task context setup and switching use core::{arch::global_asm, cell::UnsafeCell}; -use abi::error::Error; +use abi::{error::Error, process::ExitCode}; use alloc::boxed::Box; use crate::{ @@ -10,6 +10,7 @@ use crate::{ phys::{self, PageUsage}, ConvertAddress, }, + task::process::Process, }; struct StackBuilder { @@ -84,7 +85,9 @@ impl StackBuilder { } impl TaskContext { + /// Number of bytes to offset the signal stack pointer by pub const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; + /// Number of bytes to offset the user stack pointer by pub const USER_STACK_EXTRA_ALIGN: usize = 8; /// Constructs a kernel-space task context @@ -147,7 +150,8 @@ impl TaskContext { extern "C" fn closure_wrapper(closure_addr: usize) -> ! { let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; closure(); - todo!("Process termination"); + Process::current().exit(ExitCode::SUCCESS); + unreachable!(); } let closure = Box::new(f); diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 39c529ce..97e3cf31 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -1,32 +1,15 @@ //! x86-64 exception and interrupt handling use core::{arch::global_asm, mem::size_of_val}; -use abi::{ - arch::SavedFrame, - primitive_enum, - process::{Signal, SignalEntryData}, - syscall::SyscallFunction, -}; -use tock_registers::interfaces::{ReadWriteable, Writeable}; +use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ - arch::{ - x86_64::{ - apic::{self, __x86_64_apic_irq_handler}, - registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, - }, - Architecture, ArchitectureImpl, - }, - mem::table::AddressSpace, - syscall::raw_syscall_handler, - task::{ - process::{Process, TaskFrame}, - Cpu, - }, + arch::x86_64::apic::{self, __x86_64_apic_irq_handler}, + task::process::{Process, TaskFrame}, }; primitive_enum! { - pub enum ExceptionKind: u64 { + enum ExceptionKind: u64 { DivisionError = 0, Debug = 1, NonMaskableInterrupt = 2, @@ -53,119 +36,25 @@ primitive_enum! { } impl ExceptionKind { - pub fn ring3_possible(&self) -> bool { - match self { + fn ring3_possible(&self) -> bool { + matches!( + self, Self::DivisionError - | Self::Debug - | Self::Breakpoint - | Self::Overflow - | Self::BoundRangeExceeded - | Self::InvalidOpcode - | Self::GeneralProtectionFault - | Self::PageFault - | Self::FpuException - | Self::AlignmentCheck - | Self::SimdFpuException => true, - _ => false, - } - } -} - -impl TaskFrame for IrqFrame { - fn store(&self) -> SavedFrame { - todo!() - } - - fn restore(&mut self, saved: &SavedFrame) { - todo!() - } - - fn argument(&self) -> u64 { - todo!(); - } - - fn user_ip(&self) -> usize { - todo!() - } - - fn user_sp(&self) -> usize { - todo!() - } - - fn set_argument(&mut self, value: u64) { - self.rdi = value; - } - - fn set_return_value(&mut self, value: u64) { - todo!() - } - - fn set_user_ip(&mut self, value: usize) { - todo!() - } - - fn set_user_sp(&mut self, value: usize) { - todo!() - } -} - -impl TaskFrame for ExceptionFrame { - fn store(&self) -> SavedFrame { - SavedFrame { - rax: self.rax, - rcx: self.rcx, - rdx: self.rdx, - rbx: self.rbx, - rsi: self.rsi, - rdi: self.rdi, - rbp: self.rbp, - r8: self.r8, - r9: self.r9, - r10: self.r10, - r11: self.r11, - r12: self.r12, - r13: self.r13, - r14: self.r14, - r15: self.r15, - user_ip: self.rip, - user_sp: self.rsp, - rflags: self.rflags, - } - } - - fn restore(&mut self, saved: &SavedFrame) { - todo!() - } - - fn argument(&self) -> u64 { - 0 - } - - fn user_sp(&self) -> usize { - self.rsp as _ - } - - fn user_ip(&self) -> usize { - self.rip as _ - } - - fn set_user_sp(&mut self, value: usize) { - self.rsp = value as _; - } - - fn set_user_ip(&mut self, value: usize) { - self.rip = value as _; - } - - fn set_return_value(&mut self, _value: u64) { - // Not in syscall, do not overwrite - } - - fn set_argument(&mut self, value: u64) { - self.rdi = value; + | Self::Debug + | Self::Breakpoint + | Self::Overflow + | Self::BoundRangeExceeded + | Self::InvalidOpcode + | Self::GeneralProtectionFault + | Self::PageFault + | Self::FpuException + | Self::AlignmentCheck + | Self::SimdFpuException + ) } } +/// Frame saved onto the stack when taking an IRQ #[derive(Debug)] #[repr(C)] pub struct IrqFrame { @@ -243,6 +132,101 @@ struct Pointer { offset: usize, } +impl TaskFrame for IrqFrame { + fn store(&self) -> SavedFrame { + todo!() + } + + fn restore(&mut self, _saved: &SavedFrame) { + todo!() + } + + fn argument(&self) -> u64 { + todo!(); + } + + fn user_ip(&self) -> usize { + todo!() + } + + fn user_sp(&self) -> usize { + todo!() + } + + fn set_argument(&mut self, value: u64) { + self.rdi = value; + } + + fn set_return_value(&mut self, _value: u64) { + todo!() + } + + fn set_user_ip(&mut self, _value: usize) { + todo!() + } + + fn set_user_sp(&mut self, _value: usize) { + todo!() + } +} + +impl TaskFrame for ExceptionFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.rdx, + rbx: self.rbx, + rsi: self.rsi, + rdi: self.rdi, + rbp: self.rbp, + r8: self.r8, + r9: self.r9, + r10: self.r10, + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.rip, + user_sp: self.rsp, + rflags: self.rflags, + } + } + + fn restore(&mut self, _saved: &SavedFrame) { + todo!() + } + + fn argument(&self) -> u64 { + 0 + } + + fn user_sp(&self) -> usize { + self.rsp as _ + } + + fn user_ip(&self) -> usize { + self.rip as _ + } + + fn set_user_sp(&mut self, value: usize) { + self.rsp = value as _; + } + + fn set_user_ip(&mut self, value: usize) { + self.rip = value as _; + } + + fn set_return_value(&mut self, _value: u64) { + // Not in syscall, do not overwrite + } + + fn set_argument(&mut self, value: u64) { + self.rdi = value; + } +} + const SIZE: usize = 256; impl Entry { @@ -278,7 +262,7 @@ impl Entry { static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; -fn user_exception_inner(kind: ExceptionKind, frame: &mut ExceptionFrame) { +fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { let process = Process::current(); warnln!("{:?} in #{} {:?}", kind, process.id(), process.name()); @@ -295,42 +279,15 @@ fn user_exception_inner(kind: ExceptionKind, frame: &mut ExceptionFrame) { warnln!("CR2 = {:#x}", cr2); process.raise_signal(Signal::MemoryAccessViolation); - return; } ExceptionKind::InvalidOpcode => { process.raise_signal(Signal::Aborted); - return; } _ => todo!("No handler for exception: {:?}", kind), } - - // errorln!("Exception {}", frame.exc_number); - // errorln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); - // errorln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); - - // if let Some(cpu) = Cpu::get_local() { - // let current = cpu.queue().current_process(); - // if let Some(current) = current { - // errorln!("In process #{} {:?}", current.id(), current.name()); - // } - // } - - // debugln!("Registers:\n{:#x?}", &frame); - - // if frame.exc_number == 6 { - // // Invalid opcode - // warnln!("Invalid opcode"); - // } - - // if frame.exc_number == 14 { - // } - - // loop { - // ArchitectureImpl::wait_for_interrupt(); - // } } -fn kernel_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) -> ! { +fn kernel_exception_inner(_kind: ExceptionKind, _frame: &ExceptionFrame) -> ! { todo!() } @@ -349,18 +306,6 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { } } -pub unsafe fn handle_signal_exit(frame: &mut F) { - // TODO validate the argument - let saved_data = &*(frame.argument() as *const SignalEntryData); - infoln!( - "Handling signal exit to ip={:#x}, sp={:#x}", - saved_data.frame.user_ip, - saved_data.frame.user_sp - ); - - frame.restore(&saved_data.frame); -} - /// Initializes the interrupt descriptor table for the given CPU. /// /// # Safety diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index 71375ac7..c359d00e 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -2,18 +2,6 @@ use core::marker::PhantomData; -/// Returns an absolute address to the given symbol -#[macro_export] -macro_rules! absolute_address { - ($sym:expr) => {{ - let mut _x: usize; - unsafe { - core::arch::asm!("movabsq ${1}, {0}", out(reg) _x, sym $sym, options(att_syntax)); - } - _x - }}; -} - /// Wrapper struct providing access to an x86 I/O port #[derive(Clone, Debug)] #[repr(transparent)] diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index f194ed25..297f7177 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -76,10 +76,10 @@ impl InterruptSource for PS2Controller { return Ok(false); } - let mut key = self.data.read(); + let key = self.data.read(); if key == 0xE0 { - key = self.data.read(); + self.data.read(); infoln!("TODO: handle 0xE0"); return Ok(true); } diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 20405d20..792e0e53 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -1,6 +1,8 @@ +//! x86-64 implementation of system call interface + use core::arch::global_asm; -use abi::{arch::SavedFrame, syscall::SyscallFunction}; +use abi::{arch::SavedFrame, process::SignalEntryData, syscall::SyscallFunction}; use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ @@ -9,12 +11,10 @@ use crate::{ task::process::{Process, TaskFrame}, }; -use super::exception::handle_signal_exit; - /// Set of registers saved when taking a syscall instruction #[derive(Debug)] #[repr(C)] -pub struct SyscallFrame { +struct SyscallFrame { rax: u64, args: [u64; 6], @@ -132,6 +132,23 @@ extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { } } +unsafe fn handle_signal_exit(frame: &mut F) { + // TODO validate the argument + let saved_data = &*(frame.argument() as *const SignalEntryData); + infoln!( + "Handling signal exit to ip={:#x}, sp={:#x}", + saved_data.frame.user_ip, + saved_data.frame.user_sp + ); + + frame.restore(&saved_data.frame); +} + +/// Initializes system call instruction support for the current CPU. +/// +/// # Safety +/// +/// Only meant to be called once per each CPU during their init. pub unsafe fn init_syscall() { extern "C" { fn __x86_64_syscall_vector(); diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/table/mod.rs index e5eb864d..1c9327f4 100644 --- a/src/arch/x86_64/table/mod.rs +++ b/src/arch/x86_64/table/mod.rs @@ -182,6 +182,7 @@ impl PageEntry { /// An entry that is not mapped pub const INVALID: Self = Self(0, PhantomData); + /// Returns `true` if the entry contains a valid mapping to either a table or to a page/block pub fn is_present(&self) -> bool { self.0 & PageAttributes::PRESENT.bits() != 0 } @@ -252,12 +253,6 @@ impl IndexMut for PageTable { } } -impl From for MapAttributes { - fn from(value: PageAttributes) -> Self { - todo!(); - } -} - impl From for PageAttributes { fn from(value: MapAttributes) -> Self { let mut res = PageAttributes::WRITABLE; @@ -337,12 +332,6 @@ impl AddressSpace { Ok(Self { l0 }) } - pub unsafe fn from_cr3(cr3: usize) -> Self { - let cr3_virt = cr3.virtualize(); - let l0 = unsafe { cr3_virt as *mut PageTable }; - Self { l0 } - } - unsafe fn as_mut(&self) -> &'static mut PageTable { self.l0.as_mut().unwrap() } diff --git a/src/debug.rs b/src/debug.rs index 9ee927b7..155d4e81 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -3,7 +3,12 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use crate::{arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, util::OneTimeInit}; +use crate::{ + arch::PLATFORM, + device::{platform::Platform, serial::SerialDevice}, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; // use crate::{ // arch::PLATFORM, @@ -46,6 +51,12 @@ pub trait DebugSink { } } +impl DebugSink for T { + fn putc(&self, c: u8) -> Result<(), Error> { + self.send(c) + } +} + struct DebugPrinter { sink: &'static dyn DebugSink, } diff --git a/src/device/input.rs b/src/device/input.rs index b2f543c8..7e462dc2 100644 --- a/src/device/input.rs +++ b/src/device/input.rs @@ -1,5 +1,10 @@ +//! Input device interfaces + use super::{tty::TtyDevice, Device}; +/// Generic keyboard-like input device interface pub trait KeyboardDevice: Device { + /// Indicates to the keyboard driver that it was attached to a terminal so it knows where to + /// send its input data to fn attach(&self, terminal: &'static dyn TtyDevice<16>); } diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 051ae0e9..7d567e1d 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -151,7 +151,7 @@ impl InterruptSource for Pl011 { Ok(()) } - fn handle_irq(&self) -> Result<(), Error> { + fn handle_irq(&self) -> Result { let inner = self.inner.get().lock(); inner.regs.ICR.write(ICR::ALL::CLEAR); @@ -160,7 +160,7 @@ impl InterruptSource for Pl011 { self.recv_byte(byte as u8); - Ok(()) + Ok(true) } } diff --git a/src/device/tty.rs b/src/device/tty.rs index 8a7af3da..6677cd93 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -19,14 +19,18 @@ use crate::{ use super::{input::KeyboardDevice, serial::SerialDevice, Device}; +// TODO rewrite this +/// Helper device to combine a display and a keyboard input into a single terminal pub struct CombinedTerminal { + #[allow(dead_code)] input: &'static dyn KeyboardDevice, output: &'static FramebufferConsole, - // TODO output + input_ring: CharRing<16>, } impl CombinedTerminal { + /// Create a combined terminal device from a keyboard and console output devices pub fn new(input: &'static dyn KeyboardDevice, output: &'static FramebufferConsole) -> Self { Self { input, @@ -43,7 +47,7 @@ impl TtyDevice<16> for CombinedTerminal { } impl SerialDevice for CombinedTerminal { - fn receive(&self, blocking: bool) -> Result { + fn receive(&self, _blocking: bool) -> Result { todo!() } diff --git a/src/main.rs b/src/main.rs index 4f9d7053..3eb7c487 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,25 +19,13 @@ use abi::{ error::Error, io::{FileMode, OpenOptions, RawFd}, - process::ExitCode, }; use fs::{devfs, FileBlockAllocator, INITRD_DATA}; use memfs::MemoryFilesystem; -use task::process::Process; use vfs::{Filesystem, IoContext, VnodeRef}; extern crate yggdrasil_abi as abi; -// -// use abi::{ -// error::Error, -// io::{FileMode, OpenOptions, RawFd}, -// process::ExitCode, -// }; -// use fs::{devfs, FileBlockAllocator, INITRD_DATA}; -// use memfs::MemoryFilesystem; -// use task::process::Process; -// use vfs::{Filesystem, IoContext, VnodeRef}; -// + extern crate alloc; #[macro_use] @@ -94,7 +82,12 @@ pub fn kernel_main() { let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); let devfs = devfs::root(); + #[cfg(target_arch = "x86_64")] let console = ioctx.find(Some(devfs.clone()), "tty0", true, true).unwrap(); + #[cfg(target_arch = "aarch64")] + let console = ioctx + .find(Some(devfs.clone()), "ttyS0", true, true) + .unwrap(); let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); let stderr = stdout.clone(); @@ -112,8 +105,4 @@ pub fn kernel_main() { user_init.enqueue_somewhere(); } - - loop {} - // XXX - // Process::current().exit(ExitCode::SUCCESS); } diff --git a/src/mem/table.rs b/src/mem/table.rs index 97c205cf..fd40e306 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -12,10 +12,14 @@ cfg_if! { } bitflags! { + /// Describes how a page translation mapping should behave #[derive(Clone, Copy)] pub struct MapAttributes: u64 { + /// The data mapped can be read by the user process const USER_READ = 1 << 0; + /// The data mapped can be written to by the user process const USER_WRITE = 1 << 1; + /// The mapping is not global across the address spaces const NON_GLOBAL = 1 << 2; } } @@ -31,6 +35,7 @@ pub trait VirtualMemoryManager { attrs: MapAttributes, ) -> Result; + /// Insert a single 4KiB-page translation mapping into the table fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error>; /// Releases the virtual memory region from the address space and the pages it refers to diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 78f9e577..52cb3140 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -106,6 +106,8 @@ fn setup_binary>( // Fill with some sentinel value to detect stack underflows let ptr = user_sp as *mut u64; + + #[allow(clippy::reversed_empty_ranges)] for i in 0..TaskContext::USER_STACK_EXTRA_ALIGN / 8 { unsafe { ptr.add(i).write_foreign_volatile(&space, 0xDEADC0DE); diff --git a/src/proc/wait.rs b/src/proc/wait.rs index 0b95f3dd..ad1cbc5f 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -5,8 +5,6 @@ use abi::error::Error; use alloc::{collections::LinkedList, rc::Rc}; use crate::{ - arch::PLATFORM, - device::platform::Platform, sync::IrqSafeSpinlock, task::process::{Process, ProcessState}, }; @@ -32,6 +30,7 @@ pub struct Wait { struct Timeout { process: Rc, + #[allow(dead_code)] deadline: Duration, } @@ -122,7 +121,7 @@ impl Wait { queue_lock = self.queue.lock(); - if let Some(deadline) = deadline { + if let Some(_deadline) = deadline { // let now = PLATFORM.timestamp_source().timestamp()?; // if now > deadline { @@ -147,7 +146,7 @@ impl Wait { static TICK_LIST: IrqSafeSpinlock> = IrqSafeSpinlock::new(LinkedList::new()); /// Suspends current task until given deadline -pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { +pub fn sleep(_timeout: Duration, _remaining: &mut Duration) -> Result<(), Error> { todo!(); // static SLEEP_NOTIFY: Wait = Wait::new("sleep"); // let now = PLATFORM.timestamp_source().timestamp()?; @@ -166,8 +165,7 @@ pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { } /// Updates all pending timeouts and wakes up the tasks that have reached theirs -pub fn tick(now: Duration) { - todo!(); +pub fn tick(_now: Duration) { // let mut list = TICK_LIST.lock(); // let mut cursor = list.cursor_front_mut(); diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 5d2219af..11cf35d6 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -20,7 +20,7 @@ use crate::{ proc::{ self, io::ProcessIo, - wait::PROCESS_EXIT_WAIT, + wait::{self, PROCESS_EXIT_WAIT}, // wait::{self, PROCESS_EXIT_WAIT}, }, sync::IrqSafeSpinlockGuard, @@ -63,15 +63,14 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } SyscallFunction::Nanosleep => { - todo!(); - // let seconds = args[0]; - // let nanos = args[1] as u32; - // let duration = Duration::new(seconds, nanos); - // let mut remaining = Duration::ZERO; + let seconds = args[0]; + let nanos = args[1] as u32; + let duration = Duration::new(seconds, nanos); + let mut remaining = Duration::ZERO; - // wait::sleep(duration, &mut remaining).unwrap(); + wait::sleep(duration, &mut remaining).unwrap(); - // Ok(0) + Ok(0) } SyscallFunction::Exit => { let code = ExitCode::from(args[0] as i32); @@ -142,11 +141,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result debugln!("run_with_io_at {:?}", at); let file = io.ioctx().open(at, path, opts, mode)?; - // if proc.session_terminal().is_none() && - // let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { - // debugln!("Session terminal set for #{}: {}", proc.id(), path); - // proc.set_session_terminal(node); - // } + if proc.session_terminal().is_none() && + let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { + debugln!("Session terminal set for #{}: {}", proc.id(), path); + proc.set_session_terminal(node); + } let fd = io.place_file(file)?; Ok(fd.0 as usize) @@ -179,17 +178,16 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result }) } SyscallFunction::Unmount => { - todo!(); - // let options = arg_user_ref::(args[0] as usize)?; + let options = arg_user_ref::(args[0] as usize)?; - // run_with_io(|mut io| { - // let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; - // mountpoint.unmount_target()?; + run_with_io(|mut io| { + let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; + mountpoint.unmount_target()?; - // debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - // Ok(0) - // }) + Ok(0) + }) } SyscallFunction::OpenDirectory => { let at = arg_option_fd(args[0] as u32); @@ -218,70 +216,66 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result }) } SyscallFunction::CreateDirectory => { - todo!(); - // let at = arg_option_fd(args[0] as u32); - // let path = arg_user_str(args[1] as usize, args[2] as usize)?; - // let _mode = FileMode::from(args[3] as u32); + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let _mode = FileMode::from(args[3] as u32); - // run_with_io_at(at, |at, mut io| { - // let (parent, name) = abi::path::split_right(path); - // let parent_node = io.ioctx().find(at, parent, true, true)?; - // parent_node.create(name, VnodeKind::Directory)?; + run_with_io_at(at, |at, mut io| { + let (parent, name) = abi::path::split_right(path); + let parent_node = io.ioctx().find(at, parent, true, true)?; + parent_node.create(name, VnodeKind::Directory)?; - // Ok(0) - // }) + Ok(0) + }) } SyscallFunction::Remove => { - todo!(); - // let at = arg_option_fd(args[0] as u32); - // let path = arg_user_str(args[1] as usize, args[2] as usize)?; - // let recurse = args[3] != 0; + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let recurse = args[3] != 0; - // run_with_io_at(at, |at, mut io| { - // let node = io.ioctx().find(at, path, false, false)?; + run_with_io_at(at, |at, mut io| { + let node = io.ioctx().find(at, path, false, false)?; - // if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { - // todo!(); - // } + if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { + todo!(); + } - // let parent = node.parent(); + let parent = node.parent(); - // parent.remove(node, recurse)?; + parent.remove(node, recurse)?; - // Ok(0) - // }) + Ok(0) + }) } SyscallFunction::GetMetadata => { - todo!(); - // let at = arg_option_fd(args[0] as u32); - // let path = arg_user_str(args[1] as usize, args[2] as usize)?; - // let buffer = arg_user_mut::>(args[3] as usize)?; - // let follow = args[4] != 0; + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let buffer = arg_user_mut::>(args[3] as usize)?; + let follow = args[4] != 0; - // run_with_io_at(at, |at, mut io| { - // let node = if path.is_empty() { - // at.ok_or(Error::InvalidArgument)? - // } else { - // io.ioctx().find(None, path, follow, true)? - // }; + run_with_io_at(at, |at, mut io| { + let node = if path.is_empty() { + at.ok_or(Error::InvalidArgument)? + } else { + io.ioctx().find(None, path, follow, true)? + }; - // let metadata = node.metadata()?; - // buffer.write(metadata); + let metadata = node.metadata()?; + buffer.write(metadata); - // Ok(0) - // }) + Ok(0) + }) } SyscallFunction::Seek => { - todo!(); - // let fd = RawFd(args[0] as u32); - // let pos = SeekFrom::from(args[1]); + let fd = RawFd(args[0] as u32); + let pos = SeekFrom::from(args[1]); - // run_with_io(|io| { - // let file = io.file(fd)?; - // let mut file_borrow = file.borrow_mut(); + run_with_io(|io| { + let file = io.file(fd)?; + let mut file_borrow = file.borrow_mut(); - // file_borrow.seek(pos).map(|v| v as usize) - // }) + file_borrow.seek(pos).map(|v| v as usize) + }) } SyscallFunction::Spawn => { let options = arg_user_ref::(args[0] as usize)?; @@ -342,14 +336,13 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } } SyscallFunction::SendSignal => { - todo!(); - // let pid = args[0] as u32; - // let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + let pid = args[0] as u32; + let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - // let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; - // target.try_set_signal(signal)?; + let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; + target.raise_signal(signal); - // Ok(0) + Ok(0) } SyscallFunction::SetSignalEntry => { let entry = args[0] as usize; diff --git a/src/task/mod.rs b/src/task/mod.rs index 3cd1122e..0da9ebb8 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -16,13 +16,6 @@ cfg_if! { use crate::{ kernel_main, - mem::{ - phys::{self, PageUsage}, - table::AddressSpace, - ConvertAddress, - }, - //arch::aarch64::{context::TaskContext, cpu::Cpu, smp::CPU_COUNT}, - // kernel_main, sync::{IrqSafeSpinlock, SpinFence}, task::sched::CpuQueue, }; diff --git a/src/task/process.rs b/src/task/process.rs index 2a07d1fa..6bedad48 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -26,17 +26,33 @@ use crate::{ use super::{sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; +/// Interface for task state save/restore mechanisms pub trait TaskFrame { + /// Creates a "snapshot" of a exception/syscall frame fn store(&self) -> SavedFrame; + + /// Restores the exception/syscall frame from its saved state fn restore(&mut self, saved: &SavedFrame); + /// Replaces the return value in the frame (or does nothing, if the frame is not a part of a + /// syscall signal handler) fn set_return_value(&mut self, value: u64); + + /// Replaces the userspace stack pointer in the frame fn set_user_sp(&mut self, value: usize); + + /// Replaces the userspace instruction pointer in the frame fn set_user_ip(&mut self, value: usize); + + /// Replaces the argument in the frame fn set_argument(&mut self, value: u64); + /// Returns the argument (if any) of the frame being processed fn argument(&self) -> u64; + + /// Returns the userspace stack pointer fn user_sp(&self) -> usize; + /// Returns the userspace instruction pointer fn user_ip(&self) -> usize; } @@ -149,6 +165,7 @@ impl Process { *self.id.get() } + /// Returns the binary name of the process pub fn name(&self) -> &str { self.name.as_str() } @@ -187,7 +204,6 @@ impl Process { self.space.as_ref() } - // XXX /// Replaces the task's session terminal device with another one pub fn set_session_terminal(&self, terminal: VnodeRef) { self.inner.lock().session_terminal.replace(terminal); @@ -324,7 +340,7 @@ impl Process { Self::get_current().unwrap() } - // /// Handles the cleanup of an exited process + /// Handles the cleanup of an exited process pub fn handle_exit(&self) { // Queue lock is still held assert_eq!(self.state(), ProcessState::Terminated); diff --git a/src/task/sched.rs b/src/task/sched.rs index 4c326bcd..5797adc1 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -139,7 +139,9 @@ impl CpuQueue { /// The function is only meant to be called from kernel threads (e.g. if they want to yield /// CPU execution to wait for something) or interrupt handlers. pub unsafe fn yield_cpu(&self) { - assert!(ArchitectureImpl::interrupt_mask()); + // Make sure the scheduling process doesn't get interrupted + ArchitectureImpl::set_interrupt_mask(true); + let mut inner = self.inner.lock(); // let t = CNTPCT_EL0.get(); From 67d8f269ad855466ddf22838628f3f12cddaead3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 2 Aug 2023 20:53:14 +0300 Subject: [PATCH 030/211] proc: merge TaskContextImpl trait --- src/arch/aarch64/context.rs | 53 ++++----------------- src/arch/aarch64/exception.rs | 2 +- src/arch/x86_64/context.rs | 52 +++++---------------- src/arch/x86_64/exception.rs | 2 +- src/arch/x86_64/syscall.rs | 2 +- src/proc/exec.rs | 3 +- src/task/context.rs | 88 +++++++++++++++++++++++++++++++++++ src/task/mod.rs | 14 ++---- src/task/process.rs | 34 +------------- src/task/sched.rs | 1 + 10 files changed, 119 insertions(+), 132 deletions(-) create mode 100644 src/task/context.rs diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index ab0e2370..fdd9c612 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -1,15 +1,14 @@ //! AArch64-specific task context implementation use core::{arch::global_asm, cell::UnsafeCell}; -use abi::{error::Error, process::ExitCode}; -use alloc::boxed::Box; +use abi::error::Error; use crate::{ mem::{ phys::{self, PageUsage}, ConvertAddress, }, - task::process::Process, + task::context::TaskContextImpl, }; struct StackBuilder { @@ -84,15 +83,11 @@ impl StackBuilder { } } -impl TaskContext { - /// Extra number of bytes to offset the user stack pointer - pub const USER_STACK_EXTRA_ALIGN: usize = 0; - /// Extra number of bytes to offset the signal stack pointer - pub const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; +impl TaskContextImpl for TaskContext { + const USER_STACK_EXTRA_ALIGN: usize = 0; + const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; - /// Constructs a kernel thread context. For a more convenient way of constructing kernel - /// processes, see [TaskContext::kernel_closure()]. - pub fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { + fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 4; let stack_base = unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() @@ -117,27 +112,7 @@ impl TaskContext { }) } - /// Constructs a safe wrapper process to execute a kernel-space closure - pub fn kernel_closure(f: F) -> Result { - extern "C" fn closure_wrapper(closure_addr: usize) -> ! { - let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; - closure(); - Process::current().exit(ExitCode::SUCCESS); - unreachable!(); - } - - let closure = Box::new(f); - Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) - } - - /// Constructs a user thread context. The caller is responsible for allocating the userspace - /// stack and setting up a valid address space for the context. - pub fn user( - entry: usize, - arg: usize, - ttbr0: usize, - user_stack_sp: usize, - ) -> Result { + fn user(entry: usize, arg: usize, ttbr0: usize, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 8; let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; @@ -160,21 +135,11 @@ impl TaskContext { }) } - /// Starts execution of `self` task on local CPU. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code. - pub unsafe fn enter(&self) -> ! { + unsafe fn enter(&self) -> ! { __aarch64_enter_task(self.inner.get()) } - /// Switches from `from` task to `self` task. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code. - pub unsafe fn switch(&self, from: &Self) { + unsafe fn switch(&self, from: &Self) { __aarch64_switch_task(self.inner.get(), from.inner.get()) } } diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 5ebf8e0b..eb5c8417 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -15,7 +15,7 @@ use crate::{ device::interrupt::IrqContext, panic::panic_secondary, syscall::raw_syscall_handler, - task::process::{Process, TaskFrame}, + task::{context::TaskFrame, process::Process}, }; /// Struct for register values saved when taking an exception diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 3af8be5c..3a64daa1 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -1,8 +1,7 @@ //! x86-64 task context setup and switching use core::{arch::global_asm, cell::UnsafeCell}; -use abi::{error::Error, process::ExitCode}; -use alloc::boxed::Box; +use abi::error::Error; use crate::{ arch::x86_64::table::KERNEL_TABLES, @@ -10,7 +9,7 @@ use crate::{ phys::{self, PageUsage}, ConvertAddress, }, - task::process::Process, + task::context::TaskContextImpl, }; struct StackBuilder { @@ -34,8 +33,6 @@ pub struct TaskContext { stack_size: usize, } -unsafe impl Sync for TaskContext {} - // 8 registers + return address (which is not included) const COMMON_CONTEXT_SIZE: usize = 8 * 8; @@ -84,14 +81,13 @@ impl StackBuilder { } } -impl TaskContext { - /// Number of bytes to offset the signal stack pointer by - pub const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; - /// Number of bytes to offset the user stack pointer by - pub const USER_STACK_EXTRA_ALIGN: usize = 8; +unsafe impl Sync for TaskContext {} - /// Constructs a kernel-space task context - pub fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { +impl TaskContextImpl for TaskContext { + const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; + const USER_STACK_EXTRA_ALIGN: usize = 8; + + fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 4; let stack_base = unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() @@ -119,9 +115,7 @@ impl TaskContext { }) } - /// Constructs a user thread context. The caller is responsible for allocating the userspace - /// stack and setting up a valid address space for the context. - pub fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result { + fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 8; let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; @@ -145,35 +139,11 @@ impl TaskContext { }) } - /// Constructs a safe wrapper process to execute a kernel-space closure - pub fn kernel_closure(f: F) -> Result { - extern "C" fn closure_wrapper(closure_addr: usize) -> ! { - let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; - closure(); - Process::current().exit(ExitCode::SUCCESS); - unreachable!(); - } - - let closure = Box::new(f); - debugln!("closure: {:p}", closure); - Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) - } - - /// Performs an entry into a context. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code. - pub unsafe fn enter(&self) -> ! { + unsafe fn enter(&self) -> ! { __x86_64_enter_task(self.inner.get()) } - /// Performs a context switch between two contexts. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code. - pub unsafe fn switch(&self, from: &Self) { + unsafe fn switch(&self, from: &Self) { let to = self.inner.get(); let from = from.inner.get(); diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 97e3cf31..1b170176 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -5,7 +5,7 @@ use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ arch::x86_64::apic::{self, __x86_64_apic_irq_handler}, - task::process::{Process, TaskFrame}, + task::{context::TaskFrame, process::Process}, }; primitive_enum! { diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 792e0e53..71aebdc9 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -8,7 +8,7 @@ use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ arch::x86_64::registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, syscall::raw_syscall_handler, - task::process::{Process, TaskFrame}, + task::{context::TaskFrame, process::Process}, }; /// Set of registers saved when taking a syscall instruction diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 52cb3140..4803d636 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -6,14 +6,13 @@ use alloc::{rc::Rc, string::String}; use vfs::FileRef; use crate::{ - // arch::aarch64::context::TaskContext, mem::{ phys::{self, PageUsage}, table::{AddressSpace, MapAttributes, VirtualMemoryManager}, ConvertAddress, ForeignPointer, }, proc, - task::{process::Process, TaskContext}, + task::{context::TaskContextImpl, process::Process, TaskContext}, }; fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { diff --git a/src/task/context.rs b/src/task/context.rs new file mode 100644 index 00000000..6808d302 --- /dev/null +++ b/src/task/context.rs @@ -0,0 +1,88 @@ +//! Platform-specific task context manipulation interfaces + +use abi::{arch::SavedFrame, error::Error, process::ExitCode}; +use alloc::boxed::Box; +use cfg_if::cfg_if; + +use crate::task::process::Process; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub use crate::arch::aarch64::{context::TaskContext, cpu::Cpu}; + } else if #[cfg(target_arch = "x86_64")] { + pub use crate::arch::x86_64::{context::TaskContext, cpu::Cpu}; + } +} + +/// Interface for task state save/restore mechanisms +pub trait TaskFrame { + /// Creates a "snapshot" of a exception/syscall frame + fn store(&self) -> SavedFrame; + + /// Restores the exception/syscall frame from its saved state + fn restore(&mut self, saved: &SavedFrame); + + /// Replaces the return value in the frame (or does nothing, if the frame is not a part of a + /// syscall signal handler) + fn set_return_value(&mut self, value: u64); + + /// Replaces the userspace stack pointer in the frame + fn set_user_sp(&mut self, value: usize); + + /// Replaces the userspace instruction pointer in the frame + fn set_user_ip(&mut self, value: usize); + + /// Replaces the argument in the frame + fn set_argument(&mut self, value: u64); + + /// Returns the argument (if any) of the frame being processed + fn argument(&self) -> u64; + + /// Returns the userspace stack pointer + fn user_sp(&self) -> usize; + /// Returns the userspace instruction pointer + fn user_ip(&self) -> usize; +} + +/// Platform-specific task context implementation +pub trait TaskContextImpl: Sized { + /// Number of bytes to offset the signal stack pointer by + const SIGNAL_STACK_EXTRA_ALIGN: usize; + /// Number of bytes to offset the user stack pointer by + const USER_STACK_EXTRA_ALIGN: usize; + + /// Constructs a kernel-space task context + fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result; + + /// Constructs a user thread context. The caller is responsible for allocating the userspace + /// stack and setting up a valid address space for the context. + fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result; + + /// Performs an entry into a context. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + unsafe fn enter(&self) -> !; + + /// Performs a context switch between two contexts. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + unsafe fn switch(&self, from: &Self); + + /// Constructs a safe wrapper process to execute a kernel-space closure + fn kernel_closure(f: F) -> Result { + extern "C" fn closure_wrapper(closure_addr: usize) -> ! { + let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; + closure(); + Process::current().exit(ExitCode::SUCCESS); + unreachable!(); + } + + let closure = Box::new(f); + debugln!("closure: {:p}", closure); + Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) + } +} diff --git a/src/task/mod.rs b/src/task/mod.rs index 0da9ebb8..f852a62b 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -4,15 +4,6 @@ use abi::error::Error; use alloc::{rc::Rc, string::String, vec::Vec}; -use cfg_if::cfg_if; - -cfg_if! { - if #[cfg(target_arch = "aarch64")] { - pub use crate::arch::aarch64::{context::TaskContext, cpu::Cpu}; - } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::{context::TaskContext, cpu::Cpu}; - } -} use crate::{ kernel_main, @@ -20,12 +11,15 @@ use crate::{ task::sched::CpuQueue, }; -use self::process::Process; +use self::{context::TaskContextImpl, process::Process}; +pub mod context; pub mod process; pub mod sched; pub mod tasklet; +pub use context::{Cpu, TaskContext}; + /// Process identifier alias for clarity pub type ProcessId = usize; diff --git a/src/task/process.rs b/src/task/process.rs index 6bedad48..785654e9 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -6,7 +6,6 @@ use core::{ }; use abi::{ - arch::SavedFrame, error::Error, process::{ExitCode, Signal, SignalEntryData}, }; @@ -21,40 +20,11 @@ use crate::{ wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, }, sync::{IrqGuard, IrqSafeSpinlock}, + task::context::TaskContextImpl, util::OneTimeInit, }; -use super::{sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; - -/// Interface for task state save/restore mechanisms -pub trait TaskFrame { - /// Creates a "snapshot" of a exception/syscall frame - fn store(&self) -> SavedFrame; - - /// Restores the exception/syscall frame from its saved state - fn restore(&mut self, saved: &SavedFrame); - - /// Replaces the return value in the frame (or does nothing, if the frame is not a part of a - /// syscall signal handler) - fn set_return_value(&mut self, value: u64); - - /// Replaces the userspace stack pointer in the frame - fn set_user_sp(&mut self, value: usize); - - /// Replaces the userspace instruction pointer in the frame - fn set_user_ip(&mut self, value: usize); - - /// Replaces the argument in the frame - fn set_argument(&mut self, value: u64); - - /// Returns the argument (if any) of the frame being processed - fn argument(&self) -> u64; - - /// Returns the userspace stack pointer - fn user_sp(&self) -> usize; - /// Returns the userspace instruction pointer - fn user_ip(&self) -> usize; -} +use super::{context::TaskFrame, sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; /// Represents the states a process can be at some point in time #[atomic_enum] diff --git a/src/task/sched.rs b/src/task/sched.rs index 5797adc1..b6c68b9e 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -12,6 +12,7 @@ use crate::{ }; use super::{ + context::TaskContextImpl, process::{CurrentProcess, Process, ProcessState}, Cpu, ProcessId, TaskContext, }; From b8e7783da325e70c19453f98a8c3343ac73617e6 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 3 Aug 2023 09:59:02 +0300 Subject: [PATCH 031/211] x86-64: fix %rcx getting clobbered in gdt init --- src/arch/x86_64/gdt.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 7eab6b35..8bcb32b7 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -190,6 +190,7 @@ pub unsafe fn init(_id: u32) -> usize { "#, in(reg) &gdtr, out("rax") _, + out("rcx") _, options(att_syntax) ); From 2b4f6bbdc372f4c30c05c97e803ec4b01db2d252 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 3 Aug 2023 10:05:41 +0300 Subject: [PATCH 032/211] x86-64: force correct %rsp alignment on exc entry --- src/arch/x86_64/syscall.S | 10 ++++++++++ src/arch/x86_64/vectors.S | 29 +++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/arch/x86_64/syscall.S b/src/arch/x86_64/syscall.S index 1bb7b1d0..9de00d5f 100644 --- a/src/arch/x86_64/syscall.S +++ b/src/arch/x86_64/syscall.S @@ -85,9 +85,19 @@ __x86_64_syscall_vector: mov %ax, %ds mov %ax, %es + // Save %rsp to %rdi (arg 1) and %rbp mov %rsp, %rdi + mov %rsp, %rbp + + // Force correct stack alignment + orq $0xF, %rsp + xorq $0xF, %rsp + call {syscall_handler} + // Restore the stack + mov %rbp, %rsp + .restore_state: // Restore non-clobbered state movq REG_RAX(%rsp), %rax diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index d7afb4ba..b3cf82b4 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -74,8 +74,8 @@ __x86_64_exc_\n: .set IRQ_REG_R14, 13 * 8 .set IRQ_REG_R15, 14 * 8 -// 15 registers + 1 padding qword -.set IRQ_STATE_SIZE, 16 * 8 +// 15 registers + stack align word if needed +.set IRQ_STATE_SIZE, 15 * 8 .macro apic_vector, n __x86_64_apic_irq_\n: @@ -98,15 +98,26 @@ __x86_64_apic_irq_\n: movq %r14, IRQ_REG_R14(%rsp) movq %r15, IRQ_REG_R15(%rsp) + // Save current stack into %rsi (arg 2) + %rbp + movq %rsp, %rsi + movq %rsp, %rbp + + // Force correct stack alignment + orq $0xF, %rsp + xorq $0xF, %rsp + + // Force correct segment registers mov $0x10, %ax mov %ax, %ss mov %ax, %ds mov %ax, %es mov $\n, %rdi - mov %rsp, %rsi call {apic_irq_handler} + // Restore the stack pointer + movq %rbp, %rsp + // Restore state movq IRQ_REG_RAX(%rsp), %rax movq IRQ_REG_RCX(%rsp), %rcx @@ -135,10 +146,20 @@ __x86_64_apic_irq_\n: .section .text __x86_64_exc_common: EXC_SAVE_STATE - mov %rsp, %rdi + + // Save current stack into %rdi (arg 1) + %rbp + movq %rsp, %rdi + movq %rsp, %rbp + + // Force correct stack alignment + orq $0xF, %rsp + xorq $0xF, %rsp call {exception_handler} + // Restore the stack + movq %rbp, %rsp + EXC_RESTORE_STATE // Remove error code and number from the stack From 474d20c690bbc75568c9e73535de9b5663aac1ea Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 3 Aug 2023 10:14:42 +0300 Subject: [PATCH 033/211] x86-64: differentiate kernel gs and user gs --- src/arch/x86_64/context.S | 2 ++ src/arch/x86_64/cpu.rs | 4 +++- src/arch/x86_64/syscall.S | 5 +++++ src/arch/x86_64/vectors.S | 21 +++++++++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/context.S b/src/arch/x86_64/context.S index ffcbe2af..1892e63e 100644 --- a/src/arch/x86_64/context.S +++ b/src/arch/x86_64/context.S @@ -57,6 +57,8 @@ __x86_64_task_enter_user: pushq $0x23 pushq %rax + swapgs + iretq __x86_64_task_enter_kernel: diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 52689d83..210e8f58 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -51,7 +51,9 @@ impl Cpu { MSR_IA32_KERNEL_GS_BASE.set(this as u64); core::arch::asm!("swapgs"); - MSR_IA32_KERNEL_GS_BASE.set(this as u64); + + // Force some incorrect value into the register for debugging + MSR_IA32_KERNEL_GS_BASE.set(0xDEADB0D1 as u64); } /// Returns this CPU's local data structure diff --git a/src/arch/x86_64/syscall.S b/src/arch/x86_64/syscall.S index 9de00d5f..e84f7f17 100644 --- a/src/arch/x86_64/syscall.S +++ b/src/arch/x86_64/syscall.S @@ -71,6 +71,9 @@ __x86_64_syscall_vector: // %rcx - userspace %rip // %r11 - rflags + // syscall can only be issued from the userspace, so swapgs + swapgs + // Store user RSP // TODO: eliminate magic %gs-relative addresses mov %rsp, %gs:(16) @@ -141,6 +144,7 @@ __x86_64_syscall_vector: // Restore user RSP mov %gs:(16), %rsp + swapgs sysretq .return_via_iret: @@ -182,4 +186,5 @@ __x86_64_syscall_vector: // Restore %rcx movq (IRET_FRAME_SIZE + REG_IRET_RCX)(%rsp), %rcx + swapgs iretq diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index b3cf82b4..4487b279 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -58,6 +58,13 @@ __x86_64_exc_\n: jmp __x86_64_exc_common .endm +.macro SWAPGS_IF_NEEDED, cs_off + cmpq $0x08, \cs_off(%rsp) + je 1f + swapgs +1: +.endm + .set IRQ_REG_RAX, 0 * 8 .set IRQ_REG_RCX, 1 * 8 .set IRQ_REG_RDX, 2 * 8 @@ -79,6 +86,10 @@ __x86_64_exc_\n: .macro apic_vector, n __x86_64_apic_irq_\n: + // %rsp + 0: %rip + // %rsp + 8: %cs + SWAPGS_IF_NEEDED 8 + // Save state subq $IRQ_STATE_SIZE, %rsp @@ -137,6 +148,8 @@ __x86_64_apic_irq_\n: addq $IRQ_STATE_SIZE, %rsp + SWAPGS_IF_NEEDED 8 + iretq .endm @@ -145,6 +158,12 @@ __x86_64_apic_irq_\n: .section .text __x86_64_exc_common: + // %rsp + 0: error number + // %rsp + 8: error code + // %rsp + 16: %rip + // %rsp + 24: %cs + SWAPGS_IF_NEEDED 24 + EXC_SAVE_STATE // Save current stack into %rdi (arg 1) + %rbp @@ -165,6 +184,8 @@ __x86_64_exc_common: // Remove error code and number from the stack addq $16, %rsp + SWAPGS_IF_NEEDED 8 + iretq ISR_NERR 0 From b5fcc1eaa9aa9142455eb4a3c0d17df435fe4759 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 3 Aug 2023 13:01:30 +0300 Subject: [PATCH 034/211] display: basic buffered console --- src/arch/x86_64/mod.rs | 17 ++- src/debug.rs | 6 - src/device/display/console.rs | 250 +++++++++++++++++++++++++++++++ src/device/display/fb_console.rs | 135 +++++++++-------- src/device/display/mod.rs | 1 + src/device/serial/mod.rs | 2 + src/device/serial/pl011.rs | 6 + src/device/tty.rs | 9 +- 8 files changed, 355 insertions(+), 71 deletions(-) create mode 100644 src/device/display/console.rs diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5dad6708..e919a781 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -9,7 +9,11 @@ use crate::{ arch::x86_64::table::{init_fixed_tables, KERNEL_TABLES}, debug::DebugSink, device::{ - display::{fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, + display::{ + console::{ConsoleBuffer, ConsoleRow, DisplayConsole}, + fb_console::FramebufferConsole, + linear_fb::LinearFramebuffer, + }, input::KeyboardDevice, interrupt::{ExternalInterruptController, InterruptSource}, platform::Platform, @@ -238,6 +242,7 @@ impl Platform for X86_64 { FB_CONSOLE.init(FramebufferConsole::from_framebuffer( LINEAR_FB.get(), &bitmap_font::tamzen::FONT_6x12, + ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 255, 32), )); } @@ -288,6 +293,12 @@ impl X86_64 { size: r.page_count() * 0x1000, })) .expect("Failed to initialize the physical memory manager"); + + // Reallocate the console buffer + FB_CONSOLE + .get() + .reallocate_buffer() + .expect("Failed to reallocate console buffer"); } fn init_rsdp(&self, phys: usize) { @@ -330,3 +341,7 @@ pub static ARCHITECTURE: X86_64 = X86_64 { static LINEAR_FB: OneTimeInit = OneTimeInit::new(); static FB_CONSOLE: OneTimeInit = OneTimeInit::new(); + +const EARLY_BUFFER_LINES: usize = 32; +static mut EARLY_CONSOLE_BUFFER: [ConsoleRow; EARLY_BUFFER_LINES] = + [ConsoleRow::zeroed(); EARLY_BUFFER_LINES]; diff --git a/src/debug.rs b/src/debug.rs index 155d4e81..79437e12 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -51,12 +51,6 @@ pub trait DebugSink { } } -impl DebugSink for T { - fn putc(&self, c: u8) -> Result<(), Error> { - self.send(c) - } -} - struct DebugPrinter { sink: &'static dyn DebugSink, } diff --git a/src/device/display/console.rs b/src/device/display/console.rs new file mode 100644 index 00000000..3614107e --- /dev/null +++ b/src/device/display/console.rs @@ -0,0 +1,250 @@ +use core::mem::size_of; + +use abi::{error::Error, primitive_enum}; + +use crate::{ + debug::DebugSink, + mem::{ + phys::{self, PageUsage}, + ConvertAddress, + }, + sync::IrqSafeSpinlock, +}; + +const CONSOLE_ROW_LEN: usize = 128; + +primitive_enum! { + pub enum ColorAttribute: u8 { + Black = 0, + Red = 1, + Green = 2, + Blue = 3, + White = 7, + } +} + +impl ColorAttribute { + pub fn as_rgba(&self) -> u32 { + match self { + Self::Black => 0xFF000000, + Self::Red => 0xFFFF0000, + Self::Green => 0xFF00FF00, + Self::Blue => 0xFF0000FF, + Self::White => 0xFFFFFFFF, + } + } +} + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct ConsoleChar(u16); + +#[derive(Clone, Copy)] +pub struct ConsoleRow { + dirty: u8, + chars: [ConsoleChar; CONSOLE_ROW_LEN], +} + +pub struct ConsoleBuffer { + rows: &'static mut [ConsoleRow], + width: u32, + height: u32, +} + +pub struct ConsoleState { + cursor_row: u32, + cursor_col: u32, + fg_color: ColorAttribute, + + pub buffer: ConsoleBuffer, +} + +pub struct RowIter<'a> { + buffer: &'a mut ConsoleBuffer, + index: u32, +} + +pub trait DisplayConsole { + fn state(&self) -> &IrqSafeSpinlock; + + fn flush(&self, state: &mut ConsoleState); + + fn reallocate_buffer(&self) -> Result<(), Error>; + + fn write_char(&self, c: u8) { + let mut state = self.state().lock(); + if state.putc(c) { + self.flush(&mut *state); + } + } +} + +impl ConsoleChar { + pub const BLANK: Self = Self(0); + + #[inline] + pub fn from_parts(c: u8, fg: ColorAttribute) -> Self { + let fg = (u8::from(fg) as u16) << 8; + Self(fg | (c as u16)) + } + + #[inline] + pub fn fg_color(self) -> ColorAttribute { + ColorAttribute::try_from((self.0 >> 8) as u8).unwrap() + } + + #[inline] + pub const fn character(self) -> u8 { + self.0 as u8 + } +} + +impl<'a> RowIter<'a> { + pub fn next<'b>(&'b mut self) -> Option<(u32, &'b [ConsoleChar])> { + loop { + if self.index == self.buffer.height { + return None; + } + + let row_index = self.index; + let row = &self.buffer.rows[self.index as usize]; + + self.index += 1; + + if row.is_dirty() { + return Some((row_index, &row.chars)); + } + } + } +} + +impl ConsoleRow { + pub const fn zeroed() -> Self { + Self { + dirty: 0, + chars: [ConsoleChar::BLANK; CONSOLE_ROW_LEN], + } + } + + #[inline] + pub const fn is_dirty(&self) -> bool { + self.dirty != 0 + } + + pub fn clear(&mut self) { + self.dirty = 1; + self.chars.fill(ConsoleChar::BLANK); + } +} + +impl ConsoleBuffer { + pub const fn fixed(rows: &'static mut [ConsoleRow], width: u32, height: u32) -> Self { + Self { + rows, + width, + height, + } + } + + pub fn reallocate(&mut self, new_height: u32) -> Result<(), Error> { + // TODO suppress debugging output here + if new_height <= self.height { + // Keep using the old buffer + return Ok(()); + } + + let size = size_of::() * (new_height as usize); + let page_count = (size + 0xFFF) / 0x1000; + let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?; + + let data = unsafe { + core::slice::from_raw_parts_mut( + pages.virtualize() as *mut ConsoleRow, + new_height as usize, + ) + }; + + // Copy rows from the old buffer + data[0..self.height as usize].copy_from_slice(&self.rows); + data[self.height as usize..].fill(ConsoleRow::zeroed()); + + self.rows = data; + self.height = new_height; + + Ok(()) + } + + fn set_char(&mut self, row: u32, col: u32, c: ConsoleChar) { + self.rows[row as usize].dirty = 1; + self.rows[row as usize].chars[col as usize] = c; + } + + pub fn flush_rows<'a>(&'a mut self) -> RowIter<'a> { + RowIter { + buffer: self, + index: 0, + } + } + + fn scroll_once(&mut self) { + self.rows.copy_within(1.., 0); + self.rows[(self.height - 1) as usize].clear(); + } +} + +impl ConsoleState { + pub fn new(buffer: ConsoleBuffer) -> Self { + Self { + cursor_row: 0, + cursor_col: 0, + fg_color: ColorAttribute::White, + + buffer, + } + } + + fn putc(&mut self, c: u8) -> bool { + let mut flush = false; + + match c { + b'\n' => { + self.cursor_row += 1; + self.cursor_col = 0; + flush = true; + } + _ => { + self.buffer.set_char( + self.cursor_row, + self.cursor_col, + ConsoleChar::from_parts(c, self.fg_color), + ); + + self.cursor_col += 1; + } + } + + if self.cursor_col == self.buffer.width { + self.cursor_col = 0; + self.cursor_row += 1; + } + + if self.cursor_row == self.buffer.height { + self.buffer.scroll_once(); + self.cursor_row = self.buffer.height - 1; + flush = true; + } + + flush + } +} + +impl DebugSink for dyn DisplayConsole { + fn putc(&self, c: u8) -> Result<(), Error> { + self.write_char(c); + Ok(()) + } + + fn supports_control_sequences(&self) -> bool { + true + } +} diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index d9d271db..1ba02fd3 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -11,22 +11,80 @@ use embedded_graphics::{ use crate::{debug::DebugSink, sync::IrqSafeSpinlock}; -use super::{linear_fb::LinearFramebuffer, DisplayDevice, DisplayDimensions}; +use super::{ + console::{ConsoleBuffer, ConsoleState, DisplayConsole}, + linear_fb::LinearFramebuffer, + DisplayDevice, DisplayDimensions, +}; struct Inner { framebuffer: &'static LinearFramebuffer, font: &'static BitmapFont<'static>, - row: u32, - col: u32, - char_height: u32, char_width: u32, - width_chars: u32, - height_chars: u32, + char_height: u32, + width: u32, + height: u32, + + fg_color: u32, } /// Framebuffer console device wrapper pub struct FramebufferConsole { inner: IrqSafeSpinlock, + state: IrqSafeSpinlock, +} + +impl DebugSink for FramebufferConsole { + fn putc(&self, c: u8) -> Result<(), Error> { + self.write_char(c); + Ok(()) + } + + fn supports_control_sequences(&self) -> bool { + loop {} + } +} + +impl DisplayConsole for FramebufferConsole { + fn state(&self) -> &IrqSafeSpinlock { + &self.state + } + + fn reallocate_buffer(&self) -> Result<(), Error> { + let mut state = self.state.lock(); + let mut inner = self.inner.lock(); + + let new_height = inner.height; + state.buffer.reallocate(new_height) + } + + fn flush(&self, state: &mut ConsoleState) { + let mut inner = self.inner.lock(); + let font = inner.font; + let cw = inner.char_width; + let ch = inner.char_height; + + let mut iter = state.buffer.flush_rows(); + + while let Some((row_idx, row)) = iter.next() { + if row_idx >= inner.height { + break; + } + + inner.fill_row(row_idx * ch, ch, 0xFF000000); + + for (col_idx, &chr) in row.into_iter().take(inner.width as _).enumerate() { + let glyph = chr.character(); + + if glyph == 0 { + continue; + } + + inner.fg_color = chr.fg_color().as_rgba(); + inner.draw_glyph(font, (col_idx as u32) * cw, row_idx * ch, glyph); + } + } + } } impl FramebufferConsole { @@ -34,6 +92,7 @@ impl FramebufferConsole { pub fn from_framebuffer( framebuffer: &'static LinearFramebuffer, font: &'static BitmapFont<'static>, + buffer: ConsoleBuffer, ) -> Self { let char_width = 6; let char_height = 12; @@ -42,63 +101,22 @@ impl FramebufferConsole { let inner = Inner { framebuffer, font, - row: 0, - col: 0, - width_chars: dim.width / char_width, - height_chars: dim.height / char_height, + width: dim.width / char_width, + height: dim.height / char_height, char_width, char_height, + + fg_color: 0, }; Self { inner: IrqSafeSpinlock::new(inner), + state: IrqSafeSpinlock::new(ConsoleState::new(buffer)), } } } impl Inner { - fn putc(&mut self, c: u8) { - self.draw_glyph( - self.font, - self.char_width * self.col, - self.char_height * self.row, - c, - ); - - if c == b'\n' { - self.col = 0; - self.row += 1; - } else { - self.col += 1; - } - - if self.col == self.width_chars { - self.col = 0; - self.row += 1; - } - - self.scroll(); - // if self.row == self.height_chars { - // self.row = self.height_chars - 1; - // } - } - - fn scroll(&mut self) { - // Should only happen once - while self.row >= self.height_chars { - let mut fb = unsafe { self.framebuffer.lock() }; - - fb.copy_rows(self.char_height, 0, self.char_height * self.height_chars); - fb.fill_rows( - (self.height_chars - 1) * self.char_height, - self.char_height, - 0x00000000, - ); - - self.row -= 1; - } - } - fn draw_glyph(&mut self, font: &BitmapFont, x: u32, y: u32, c: u8) { let text_data = [c]; let text_str = unsafe { core::str::from_utf8_unchecked(&text_data) }; @@ -110,16 +128,11 @@ impl Inner { text.draw(self).ok(); } -} -impl DebugSink for FramebufferConsole { - fn putc(&self, c: u8) -> Result<(), Error> { - self.inner.lock().putc(c); - Ok(()) - } + fn fill_row(&mut self, y: u32, h: u32, val: u32) { + let mut fb = unsafe { self.framebuffer.lock() }; - fn supports_control_sequences(&self) -> bool { - false + fb.fill_rows(y, h, val); } } @@ -141,7 +154,7 @@ impl DrawTarget for Inner { for Pixel(coord, color) in pixels { let row = &mut fb[coord.y as u32]; let color = if color.is_on() { - 0xFFFFFFFF + self.fg_color } else { 0xFF000000 }; diff --git a/src/device/display/mod.rs b/src/device/display/mod.rs index 26db455f..9d6a6fea 100644 --- a/src/device/display/mod.rs +++ b/src/device/display/mod.rs @@ -2,6 +2,7 @@ use super::Device; +pub mod console; pub mod fb_console; pub mod linear_fb; diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index cc4765f4..44e2a1e2 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -1,6 +1,8 @@ //! Serial device interfaces use abi::error::Error; +use crate::debug::DebugSink; + use super::Device; #[cfg(target_arch = "aarch64")] diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 7d567e1d..2c011fe0 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -102,6 +102,12 @@ impl Pl011Inner { } } +impl DebugSink for Pl011 { + fn putc(&self, byte: u8) -> Result<(), Error> { + self.send(byte) + } +} + impl TtyDevice<16> for Pl011 { fn ring(&self) -> &CharRing<16> { &self.ring diff --git a/src/device/tty.rs b/src/device/tty.rs index 6677cd93..4908168d 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -17,14 +17,16 @@ use crate::{ task::{process::Process, ProcessId}, }; -use super::{input::KeyboardDevice, serial::SerialDevice, Device}; +use super::{ + display::console::DisplayConsole, input::KeyboardDevice, serial::SerialDevice, Device, +}; // TODO rewrite this /// Helper device to combine a display and a keyboard input into a single terminal pub struct CombinedTerminal { #[allow(dead_code)] input: &'static dyn KeyboardDevice, - output: &'static FramebufferConsole, + output: &'static dyn DisplayConsole, input_ring: CharRing<16>, } @@ -52,7 +54,8 @@ impl SerialDevice for CombinedTerminal { } fn send(&self, byte: u8) -> Result<(), Error> { - self.output.putc(byte) + self.output.write_char(byte); + Ok(()) } } From d19cb4dc26478e46004429c42b5a9c70585dae36 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 3 Aug 2023 18:49:29 +0300 Subject: [PATCH 035/211] x86-64: implement HPET --- src/arch/x86_64/apic/ioapic.rs | 54 ++++- src/arch/x86_64/apic/local.rs | 20 +- src/arch/x86_64/apic/mod.rs | 5 +- src/arch/x86_64/boot/mod.rs | 7 +- src/arch/x86_64/cpu.rs | 2 +- src/arch/x86_64/exception.rs | 8 +- src/arch/x86_64/mod.rs | 20 +- src/arch/x86_64/peripherals/hpet.rs | 263 +++++++++++++++++++++++++ src/arch/x86_64/peripherals/mod.rs | 1 + src/arch/x86_64/peripherals/ps2/mod.rs | 4 - src/debug.rs | 7 +- src/device/display/console.rs | 110 +++++++++-- src/device/display/fb_console.rs | 34 +++- src/device/display/linear_fb.rs | 4 - src/device/interrupt.rs | 8 + src/device/mod.rs | 13 +- src/device/platform.rs | 14 +- src/device/serial/mod.rs | 2 - src/device/tty.rs | 5 - src/main.rs | 21 +- src/proc/wait.rs | 79 ++++---- 21 files changed, 550 insertions(+), 131 deletions(-) create mode 100644 src/arch/x86_64/peripherals/hpet.rs diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 181c7eff..2517b19c 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -22,8 +22,8 @@ const REG_IOAPIC_VERSION: u32 = 0x01; const REG_REDIRECTION_BASE: u32 = 0x10; const ENTRY_LOW_MASK: u32 = 1 << 16; -// const ENTRY_LOW_TRIGGER_LEVEL: u32 = 1 << 15; -// const ENTRY_LOW_POLARITY_LOW: u32 = 1 << 13; +const ENTRY_LOW_TRIGGER_LEVEL: u32 = 1 << 15; +const ENTRY_LOW_POLARITY_LOW: u32 = 1 << 13; const ENTRY_LOW_DESTINATION_LOGICAL: u32 = 1 << 11; const ENTRY_HIGH_APIC_ID_SHIFT: u32 = 24; @@ -87,7 +87,7 @@ impl Inner { assert!(gsi < self.max_gsi); assert!(vector < 0x100); - debugln!("map_irq gsi{}, vec{}, apic{}", gsi, vector, apic_id); + infoln!("map_irq gsi{}, vec{}, apic{}", gsi, vector, apic_id); let mut low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2); let mut high = self.regs.read(REG_REDIRECTION_BASE + gsi * 2 + 1); @@ -108,6 +108,26 @@ impl Inner { Ok(()) } + fn configure_gsi(&mut self, gsi: u32, active_low: bool, level_triggered: bool) { + assert!(gsi < self.max_gsi); + + let mut low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2); + + if active_low { + low |= ENTRY_LOW_POLARITY_LOW; + } else { + low &= !ENTRY_LOW_POLARITY_LOW; + } + + if level_triggered { + low |= ENTRY_LOW_TRIGGER_LEVEL; + } else { + low &= !ENTRY_LOW_TRIGGER_LEVEL; + } + + self.regs.write(REG_REDIRECTION_BASE + gsi * 2, low); + } + fn set_gsi_enabled(&mut self, gsi: u32, enabled: bool) { assert!(gsi < self.max_gsi); @@ -124,10 +144,6 @@ impl Inner { } impl Device for IoApic { - unsafe fn init(&self) -> Result<(), Error> { - Ok(()) - } - fn name(&self) -> &'static str { "I/O APIC" } @@ -156,7 +172,13 @@ impl ExternalInterruptController for IoApic { let gsi_target_vector = (table_vector as u32) + IO_APIC_VECTOR_OFFSET; let bsp_apic = *BSP_APIC_ID.get(); - debugln!("Binding {:?} to {}:{}", irq, bsp_apic, table_vector); + infoln!( + "Binding {:?} ({}) to {}:{}", + irq, + handler.name(), + bsp_apic, + table_vector + ); table.add_handler(table_vector, handler)?; let gsi = self.translate_irq(irq); @@ -164,6 +186,20 @@ impl ExternalInterruptController for IoApic { Ok(()) } + + fn configure_irq( + &self, + irq: Self::IrqNumber, + active_low: bool, + level_triggered: bool, + ) -> Result<(), Error> { + let mut inner = self.inner.get().lock(); + let gsi = self.translate_irq(irq); + + inner.configure_gsi(gsi, active_low, level_triggered); + + Ok(()) + } } impl IoApic { @@ -226,6 +262,8 @@ impl IoApic { let max_gsi = (regs.read(REG_IOAPIC_VERSION) >> 16) & 0xFF; + infoln!("Maximum GSI number: {}", max_gsi); + let inner = Inner { regs, max_gsi }; self.inner.init(IrqSafeSpinlock::new(inner)); diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 07c6f07f..4e7399fe 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -32,6 +32,16 @@ register_bitfields! { Periodic = 1, OneShot = 0 ] + ], + LocalVectorEntry [ + Vector OFFSET(0) NUMBITS(8) [], + Mask OFFSET(16) NUMBITS(1) [ + Masked = 1, + Unmasked = 0, + ], + Nmi OFFSET(8) NUMBITS(3) [ + IsNmi = 4, + ], ] } @@ -47,12 +57,16 @@ register_structs! { (0xF4 => _3), (0x320 => TimerLocalVectorEntry: ReadWrite), (0x324 => _4), + (0x350 => LInt0: ReadWrite), + (0x354 => _5), + (0x360 => LInt1: ReadWrite), + (0x364 => _6), (0x380 => TimerInitCount: ReadWrite), - (0x384 => _5), + (0x384 => _7), (0x390 => TimerCurrentCount: ReadOnly), - (0x394 => _6), + (0x394 => _8), (0x3E0 => TimerDivideConfig: ReadWrite), - (0x3E4 => _7), + (0x3E4 => _9), (0x530 => @END), } } diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 4ac72820..a2f21c1b 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -18,7 +18,7 @@ pub mod local; /// Fixed IRQ vector for Local APIC timer pub const APIC_TIMER_VECTOR: u32 = 0; /// Start of the I/O APIC IRQ range -pub const APIC_EXTERNAL_OFFSET: u32 = 2; +pub const APIC_EXTERNAL_OFFSET: u32 = 4; /// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC pub const MAX_EXTERNAL_VECTORS: u32 = 16; @@ -70,13 +70,14 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) { fn apic_irq_inner(vector: u32) { let cpu = Cpu::local(); - cpu.local_apic().clear_interrupt(); if vector == APIC_TIMER_VECTOR { + cpu.local_apic().clear_interrupt(); unsafe { Cpu::local().queue().yield_cpu() } } else if (APIC_EXTERNAL_OFFSET..APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS).contains(&vector) { PLATFORM.ioapic.handle_irq(vector - APIC_EXTERNAL_OFFSET); + cpu.local_apic().clear_interrupt(); } else { panic!("Got an interrupt on undefined vector: {}", vector); } diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 8d2d6e7a..378ecdc1 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -9,11 +9,11 @@ use yboot_proto::{ use crate::{ arch::{ - x86_64::{apic::local::LocalApic, cpu::Cpu, cpuid, exception, syscall}, + x86_64::{apic::local::LocalApic, cpu::Cpu, cpuid, exception, syscall, FB_CONSOLE}, Architecture, ArchitectureImpl, PLATFORM, }, debug, - device::platform::Platform, + device::{display::console::add_console_autoflush, platform::Platform}, fs::{devfs, Initrd, INITRD_DATA}, mem::{ heap, @@ -128,6 +128,9 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { .expect("Could not allocate a block for heap"); heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + // Add console to flush list + add_console_autoflush(FB_CONSOLE.get()); + // Also initializes local APIC Cpu::init_local(LocalApic::new(), 0); syscall::init_syscall(); diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 210e8f58..8566db6c 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -53,7 +53,7 @@ impl Cpu { core::arch::asm!("swapgs"); // Force some incorrect value into the register for debugging - MSR_IA32_KERNEL_GS_BASE.set(0xDEADB0D1 as u64); + MSR_IA32_KERNEL_GS_BASE.set(0xDEADB0D1u64); } /// Returns this CPU's local data structure diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 1b170176..b81af7a5 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -287,8 +287,12 @@ fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { } } -fn kernel_exception_inner(_kind: ExceptionKind, _frame: &ExceptionFrame) -> ! { - todo!() +fn kernel_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) -> ! { + fatalln!("{:?} in KERNEL", kind); + fatalln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); + fatalln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); + + panic!("Irrecoverable exception"); } extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index e919a781..5da895a6 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -2,7 +2,7 @@ use core::ptr::NonNull; use abi::error::Error; -use acpi::{AcpiHandler, AcpiTables, InterruptModel, PhysicalMapping}; +use acpi::{AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap}; use crate::{ @@ -17,8 +17,8 @@ use crate::{ input::KeyboardDevice, interrupt::{ExternalInterruptController, InterruptSource}, platform::Platform, + timer::TimestampSource, tty::CombinedTerminal, - Device, }, fs::{ devfs::{self, CharDeviceType}, @@ -34,7 +34,7 @@ use crate::{ use self::{ apic::{ioapic::IoApic, IrqNumber}, intrinsics::IoPort, - peripherals::ps2::PS2Controller, + peripherals::{hpet::Hpet, ps2::PS2Controller}, }; use super::Architecture; @@ -138,6 +138,7 @@ pub struct X86_64 { // Static devices with dynamic addresses ioapic: IoApic, + timer: Hpet, // Static devices ps2: PS2Controller, @@ -197,6 +198,8 @@ impl Platform for X86_64 { // Initialize I/O APIC from ACPI let rsdp = *self.rsdp_phys_base.get(); let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap(); + + let hpet = HpetInfo::new(&acpi_tables).unwrap(); let platform_info = acpi_tables.platform_info().unwrap(); let InterruptModel::Apic(apic_info) = &platform_info.interrupt_model else { @@ -204,12 +207,11 @@ impl Platform for X86_64 { }; self.ioapic.init_from_acpi(apic_info)?; - - // Initialize static devices - self.ps2.init()?; + self.timer.init(&hpet).unwrap(); // Enable IRQs for the devices self.ps2.init_irq()?; + self.timer.init_irq()?; // Add a terminal to the devfs // TODO this is ugly @@ -263,6 +265,10 @@ impl Platform for X86_64 { ) -> &dyn ExternalInterruptController { &self.ioapic } + + fn timestamp_source(&self) -> &dyn TimestampSource { + &self.timer + } } impl X86_64 { @@ -306,6 +312,7 @@ impl X86_64 { } unsafe fn disable_8259() { + infoln!("Disabling i8259 PIC"); // TODO should I make a module for 8259 if I don't even use it? let pic_master_cmd = IoPort::new(0x20); let pic_master_data = IoPort::new(0x21); @@ -335,6 +342,7 @@ pub static ARCHITECTURE: X86_64 = X86_64 { combined_terminal: OneTimeInit::new(), ioapic: IoApic::new(), + timer: Hpet::uninit(), ps2: PS2Controller::new(IrqNumber::Isa(1), IrqNumber::Isa(12), 0x64, 0x60), }; diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs new file mode 100644 index 00000000..0b9e9071 --- /dev/null +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -0,0 +1,263 @@ +//! x86-64 implementation of a HPET +use core::time::Duration; + +use abi::error::Error; +use acpi::hpet::HpetInfo as AcpiHpet; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite}, +}; + +use crate::{ + arch::{x86_64::apic::IrqNumber, PLATFORM}, + device::{interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device}, + mem::device::DeviceMemoryIo, + proc::wait, + sync::IrqSafeSpinlock, + task::tasklet, + util::OneTimeInit, +}; + +register_bitfields! { + u64, + GeneralCapabilities [ + COUNTER_CLK_PERIOD OFFSET(32) NUMBITS(32) [], + COUNT_SIZE_CAP OFFSET(13) NUMBITS(1) [ + Supports64Bit = 1 + ], + NUM_TIM_CAP OFFSET(8) NUMBITS(5) [], + ], + GeneralConfiguration [ + LEG_RT_CNF OFFSET(1) NUMBITS(1) [], + ENABLE_CNF OFFSET(0) NUMBITS(1) [], + ], + + TimerConfigurationCapablities [ + TM_INT_TYPE_CNF OFFSET(1) NUMBITS(1) [ + EdgeTriggered = 0, + LevelTriggered = 1, + ], + TM_INT_ENB_CNF OFFSET(2) NUMBITS(1) [ + InterruptEnabled = 1, + InterruptDisabled = 0 + ], + TM_TYPE_CNF OFFSET(3) NUMBITS(1) [ + Periodic = 1, + OneShot = 0 + ], + TM_PER_INT_CAP OFFSET(4) NUMBITS(1) [ + SupportsPeriodic = 1, + ], + TM_SIZE_CAP OFFSET(5) NUMBITS(1) [ + Timer64Bit = 1, + Timer32Bit = 0, + ], + TM_VAL_SET_CNF OFFSET(6) NUMBITS(1) [], + TM_32MODE_CNF OFFSET(8) NUMBITS(1) [], + TM_INT_ROUTE_CNF OFFSET(9) NUMBITS(5) [], + TM_FSB_EN_CNF OFFSET(14) NUMBITS(1) [], + TM_FSB_INT_DEL_CAP OFFSET(15) NUMBITS(1) [], + TM_INT_ROUTE_CAP OFFSET(32) NUMBITS(32) [], + ] +} + +register_structs! { + #[allow(non_snake_case)] + TimerRegs { + (0x000 => ConfigurationCapability: ReadWrite), + (0x008 => Comparator: ReadWrite), + (0x010 => FsbInterruptRoute: ReadWrite), + (0x018 => _0), + (0x020 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x000 => GeneralCapabilities: ReadOnly), + (0x008 => _0), + (0x010 => GeneralConfiguration: ReadWrite), + (0x018 => _1), + (0x020 => GeneralInterruptStatus: ReadWrite), + (0x028 => _2), + + (0x0F0 => MainCounter: ReadWrite), + (0x0F8 => _3), + (0x100 => Timers: [TimerRegs; 3]), + (0x160 => _4), + (0x400 => @END), + } +} + +struct Inner { + regs: DeviceMemoryIo, + tim0_period: u64, + tim0_counter: u64, +} + +/// HPET timer group interface +pub struct Hpet { + inner: OneTimeInit>, +} + +impl Inner { + unsafe fn new(base: usize) -> Result { + let regs = DeviceMemoryIo::::map("hpet", base)?; + + // Disable the counter first + regs.GeneralConfiguration + .modify(GeneralConfiguration::ENABLE_CNF::CLEAR); + + // Disable legacy routing + regs.GeneralConfiguration + .modify(GeneralConfiguration::LEG_RT_CNF::CLEAR); + + // Reset the counter + regs.MainCounter.set(0); + + // Disable all comparator interrupts + for i in 0..3 { + regs.Timers[i] + .ConfigurationCapability + .modify(TimerConfigurationCapablities::TM_INT_ENB_CNF::InterruptDisabled); + } + + Ok(Self { + regs, + tim0_period: 0, + tim0_counter: 0, + }) + } +} + +impl TimestampSource for Hpet { + fn timestamp(&self) -> Result { + let inner = self.inner.get().lock(); + Ok(Duration::from_millis(inner.tim0_counter)) + } +} + +impl InterruptSource for Hpet { + fn handle_irq(&self) -> Result { + let now = { + let mut inner = self.inner.get().lock(); + inner.regs.GeneralInterruptStatus.set(1); + + inner.tim0_counter = inner.tim0_counter.wrapping_add(1); + Duration::from_millis(inner.tim0_counter) + }; + + wait::tick(now); + tasklet::tick(now); + + Ok(true) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + // Femtoseconds in a millisecond + const FS_IN_MS: u64 = 1000000000000; + + // Configure timer 0 + let intc = PLATFORM.interrupt_controller(); + let mut inner = self.inner.get().lock(); + + // Calculate the interrupt period + let clk_period = inner + .regs + .GeneralCapabilities + .read(GeneralCapabilities::COUNTER_CLK_PERIOD); + + // Will give an interrupt interval of 1ms + let period = FS_IN_MS / clk_period; + + inner.tim0_period = period; + + let tim0 = &inner.regs.Timers[0]; + + assert!(tim0.ConfigurationCapability.matches_all( + TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit + + TimerConfigurationCapablities::TM_PER_INT_CAP::SupportsPeriodic + )); + + let mut tim0_irq = None; + let tim0_irqs = tim0 + .ConfigurationCapability + .read(TimerConfigurationCapablities::TM_INT_ROUTE_CAP); + + // Skip most likely legacy routes: IRQ2, IRQ0 + for i in 3..32 { + if tim0_irqs & (1 << i) != 0 { + tim0_irq = Some(i); + break; + } + } + let tim0_irq = tim0_irq.expect("Could not pick an IRQ for HPET TIM0"); + + // Bind and enable the IRQ + let irq = IrqNumber::Gsi(tim0_irq as _); + intc.register_handler(irq, self)?; + intc.configure_irq(irq, true, true)?; + intc.enable_irq(irq)?; + + // Disable FSB interrupt route and 32 bit mode + tim0.ConfigurationCapability.modify( + TimerConfigurationCapablities::TM_FSB_EN_CNF::CLEAR + + TimerConfigurationCapablities::TM_32MODE_CNF::CLEAR + + TimerConfigurationCapablities::TM_VAL_SET_CNF::SET, + ); + + // Setup interrupt route + edge-triggered + tim0.ConfigurationCapability.modify( + TimerConfigurationCapablities::TM_INT_ROUTE_CNF.val(tim0_irq as u64) + + TimerConfigurationCapablities::TM_INT_TYPE_CNF::LevelTriggered, + ); + + // Setup periodic mode + tim0.ConfigurationCapability + .modify(TimerConfigurationCapablities::TM_TYPE_CNF::Periodic); + + // Set the comparator + tim0.Comparator.set(inner.tim0_period); + + // Enable the timer + tim0.ConfigurationCapability + .modify(TimerConfigurationCapablities::TM_INT_ENB_CNF::InterruptEnabled); + + // Enable the main counter + inner + .regs + .GeneralConfiguration + .modify(GeneralConfiguration::ENABLE_CNF::SET); + + Ok(()) + } +} + +impl Device for Hpet { + fn name(&self) -> &'static str { + "HPET" + } +} + +impl Hpet { + /// Constructs an uninitialized HPET instance + pub const fn uninit() -> Self { + Self { + inner: OneTimeInit::new(), + } + } + + /// Initialize HPET from ACPI definition + pub fn init(&self, hpet: &AcpiHpet) -> Result<(), Error> { + infoln!("Initializing HPET:"); + infoln!("Address: {:#x}", hpet.base_address); + + let inner = unsafe { Inner::new(hpet.base_address) }?; + + self.inner.init(IrqSafeSpinlock::new(inner)); + + Ok(()) + } +} diff --git a/src/arch/x86_64/peripherals/mod.rs b/src/arch/x86_64/peripherals/mod.rs index 8bb8ca17..a6229c1f 100644 --- a/src/arch/x86_64/peripherals/mod.rs +++ b/src/arch/x86_64/peripherals/mod.rs @@ -1,3 +1,4 @@ //! x86-64 platform peripheral drivers +pub mod hpet; pub mod ps2; diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index 297f7177..dbb355b1 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -102,10 +102,6 @@ impl InterruptSource for PS2Controller { } impl Device for PS2Controller { - unsafe fn init(&self) -> Result<(), Error> { - Ok(()) - } - fn name(&self) -> &'static str { "PS/2 Controller" } diff --git a/src/debug.rs b/src/debug.rs index 79437e12..9ee927b7 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -3,12 +3,7 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use crate::{ - arch::PLATFORM, - device::{platform::Platform, serial::SerialDevice}, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use crate::{arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, util::OneTimeInit}; // use crate::{ // arch::PLATFORM, diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 3614107e..4aab22bc 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -1,6 +1,8 @@ +//! Console device interfaces use core::mem::size_of; use abi::{error::Error, primitive_enum}; +use alloc::vec::Vec; use crate::{ debug::DebugSink, @@ -9,21 +11,29 @@ use crate::{ ConvertAddress, }, sync::IrqSafeSpinlock, + task::tasklet::TaskFlow, }; const CONSOLE_ROW_LEN: usize = 128; primitive_enum! { + #[doc = "Color attribute of a console character"] pub enum ColorAttribute: u8 { + #[doc = "..."] Black = 0, + #[doc = "..."] Red = 1, + #[doc = "..."] Green = 2, + #[doc = "..."] Blue = 3, + #[doc = "..."] White = 7, } } impl ColorAttribute { + /// Converts the attribute to RGBA representation pub fn as_rgba(&self) -> u32 { match self { Self::Black => 0xFF000000, @@ -35,64 +45,89 @@ impl ColorAttribute { } } +/// Represents a single character with its attributes #[derive(Clone, Copy)] #[repr(transparent)] pub struct ConsoleChar(u16); +/// Represents a single line in the console buffer #[derive(Clone, Copy)] pub struct ConsoleRow { dirty: u8, chars: [ConsoleChar; CONSOLE_ROW_LEN], } +/// Buffer that contains text rows of the console with their attributes + tracks dirty rows which +/// need to be flushed to the display pub struct ConsoleBuffer { rows: &'static mut [ConsoleRow], width: u32, height: u32, } +/// Common state for console output devices pub struct ConsoleState { - cursor_row: u32, - cursor_col: u32, - fg_color: ColorAttribute, + /// Current cursor row + pub cursor_row: u32, + /// Current cursor column + pub cursor_col: u32, + /// Current foreground color + pub fg_color: ColorAttribute, + /// Row buffer pub buffer: ConsoleBuffer, } +/// Helper type to iterate over dirty rows in the buffer pub struct RowIter<'a> { buffer: &'a mut ConsoleBuffer, index: u32, } +/// Interface to implement buffered console semantics on an abstract console output device pub trait DisplayConsole { + /// Returns the state lock fn state(&self) -> &IrqSafeSpinlock; - - fn flush(&self, state: &mut ConsoleState); - + /// Allocates a new buffer for the console according to the display's size fn reallocate_buffer(&self) -> Result<(), Error>; + /// Flushes the data from console buffer to the display + fn flush(&self, state: &mut ConsoleState); + + /// Writes characters to the backing buffer + handles special control sequences fn write_char(&self, c: u8) { let mut state = self.state().lock(); if state.putc(c) { - self.flush(&mut *state); + self.flush(&mut state); } } } impl ConsoleChar { + /// Empty character pub const BLANK: Self = Self(0); + /// Constructs a console character from a char and its attributes #[inline] - pub fn from_parts(c: u8, fg: ColorAttribute) -> Self { + pub fn from_parts(c: u8, fg: ColorAttribute, bg: ColorAttribute) -> Self { let fg = (u8::from(fg) as u16) << 8; - Self(fg | (c as u16)) + let bg = (u8::from(bg) as u16) << 12; + Self(fg | bg | (c as u16)) } + /// Returns the foreground color attribute #[inline] pub fn fg_color(self) -> ColorAttribute { - ColorAttribute::try_from((self.0 >> 8) as u8).unwrap() + ColorAttribute::try_from((self.0 >> 8) as u8).unwrap_or(ColorAttribute::White) } + /// Returns the background color attribute + #[inline] + pub fn bg_color(self) -> ColorAttribute { + ColorAttribute::try_from((self.0 >> 12) as u8).unwrap_or(ColorAttribute::Black) + } + + /// Returns the character data of this [ConsoleChar] #[inline] pub const fn character(self) -> u8 { self.0 as u8 @@ -100,25 +135,30 @@ impl ConsoleChar { } impl<'a> RowIter<'a> { - pub fn next<'b>(&'b mut self) -> Option<(u32, &'b [ConsoleChar])> { + /// Returns the next dirty row + pub fn next_dirty(&mut self) -> Option<(u32, &[ConsoleChar])> { loop { if self.index == self.buffer.height { return None; } + if !self.buffer.rows[self.index as usize].clear_dirty() { + self.index += 1; + continue; + } + let row_index = self.index; let row = &self.buffer.rows[self.index as usize]; self.index += 1; - if row.is_dirty() { - return Some((row_index, &row.chars)); - } + return Some((row_index, &row.chars)); } } } impl ConsoleRow { + /// Constructs a row filled with blank characters pub const fn zeroed() -> Self { Self { dirty: 0, @@ -126,11 +166,21 @@ impl ConsoleRow { } } + /// Returns `true` if the row's dirty flag is set #[inline] pub const fn is_dirty(&self) -> bool { self.dirty != 0 } + /// Clears "dirty" flag for the row + #[inline] + pub fn clear_dirty(&mut self) -> bool { + let old = self.dirty; + self.dirty = 0; + old == 1 + } + + /// Clears the console row with blank characters pub fn clear(&mut self) { self.dirty = 1; self.chars.fill(ConsoleChar::BLANK); @@ -138,6 +188,7 @@ impl ConsoleRow { } impl ConsoleBuffer { + /// Constructs a fixed-size console buffer pub const fn fixed(rows: &'static mut [ConsoleRow], width: u32, height: u32) -> Self { Self { rows, @@ -146,6 +197,7 @@ impl ConsoleBuffer { } } + /// Reallocates the internal buffer with a new size pub fn reallocate(&mut self, new_height: u32) -> Result<(), Error> { // TODO suppress debugging output here if new_height <= self.height { @@ -165,7 +217,7 @@ impl ConsoleBuffer { }; // Copy rows from the old buffer - data[0..self.height as usize].copy_from_slice(&self.rows); + data[0..self.height as usize].copy_from_slice(self.rows); data[self.height as usize..].fill(ConsoleRow::zeroed()); self.rows = data; @@ -179,7 +231,8 @@ impl ConsoleBuffer { self.rows[row as usize].chars[col as usize] = c; } - pub fn flush_rows<'a>(&'a mut self) -> RowIter<'a> { + /// Returns an iterator over dirty rows, while clearing dirty flag for them + pub fn flush_rows(&mut self) -> RowIter { RowIter { buffer: self, index: 0, @@ -189,10 +242,16 @@ impl ConsoleBuffer { fn scroll_once(&mut self) { self.rows.copy_within(1.., 0); self.rows[(self.height - 1) as usize].clear(); + + // Mark everything dirty + self.rows.iter_mut().for_each(|row| { + row.dirty = 1; + }); } } impl ConsoleState { + /// Constructs a new console state with given buffer pub fn new(buffer: ConsoleBuffer) -> Self { Self { cursor_row: 0, @@ -216,7 +275,7 @@ impl ConsoleState { self.buffer.set_char( self.cursor_row, self.cursor_col, - ConsoleChar::from_parts(c, self.fg_color), + ConsoleChar::from_parts(c, self.fg_color, ColorAttribute::Black), ); self.cursor_col += 1; @@ -248,3 +307,20 @@ impl DebugSink for dyn DisplayConsole { true } } + +static CONSOLES: IrqSafeSpinlock> = + IrqSafeSpinlock::new(Vec::new()); + +/// Adds a console device to a auto-flush list +pub fn add_console_autoflush(console: &'static dyn DisplayConsole) { + CONSOLES.lock().push(console); +} + +/// Periodically flushes data from console buffers onto their displays +pub fn task_update_consoles() -> TaskFlow { + for console in CONSOLES.lock().iter() { + let mut state = console.state().lock(); + console.flush(&mut state); + } + TaskFlow::Continue +} diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 1ba02fd3..b9c23bdf 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -26,6 +26,8 @@ struct Inner { height: u32, fg_color: u32, + #[allow(dead_code)] + bg_color: u32, } /// Framebuffer console device wrapper @@ -41,7 +43,7 @@ impl DebugSink for FramebufferConsole { } fn supports_control_sequences(&self) -> bool { - loop {} + true } } @@ -52,7 +54,7 @@ impl DisplayConsole for FramebufferConsole { fn reallocate_buffer(&self) -> Result<(), Error> { let mut state = self.state.lock(); - let mut inner = self.inner.lock(); + let inner = self.inner.lock(); let new_height = inner.height; state.buffer.reallocate(new_height) @@ -66,14 +68,14 @@ impl DisplayConsole for FramebufferConsole { let mut iter = state.buffer.flush_rows(); - while let Some((row_idx, row)) = iter.next() { + while let Some((row_idx, row)) = iter.next_dirty() { if row_idx >= inner.height { break; } inner.fill_row(row_idx * ch, ch, 0xFF000000); - for (col_idx, &chr) in row.into_iter().take(inner.width as _).enumerate() { + for (col_idx, &chr) in row.iter().take(inner.width as _).enumerate() { let glyph = chr.character(); if glyph == 0 { @@ -84,6 +86,18 @@ impl DisplayConsole for FramebufferConsole { inner.draw_glyph(font, (col_idx as u32) * cw, row_idx * ch, glyph); } } + + // Place cursor + let cursor_row = state.cursor_row; + let cursor_col = state.cursor_col; + + inner.fill_rect( + cursor_col * cw, + cursor_row * ch, + cw, + ch, + state.fg_color.as_rgba(), + ); } } @@ -107,6 +121,7 @@ impl FramebufferConsole { char_height, fg_color: 0, + bg_color: 0, }; Self { @@ -134,6 +149,17 @@ impl Inner { fb.fill_rows(y, h, val); } + + fn fill_rect(&mut self, x: u32, y: u32, w: u32, h: u32, val: u32) { + let mut fb = unsafe { self.framebuffer.lock() }; + + for i in 0..h { + let row = &mut fb[i + y]; + for j in 0..w { + row[(j + x) as usize] = val; + } + } + } } impl OriginDimensions for Inner { diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 62abdf20..1df8eb75 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -79,10 +79,6 @@ impl Device for LinearFramebuffer { fn name(&self) -> &'static str { "Linear Framebuffer" } - - unsafe fn init(&self) -> Result<(), Error> { - Ok(()) - } } impl DisplayDevice for LinearFramebuffer { diff --git a/src/device/interrupt.rs b/src/device/interrupt.rs index e673b226..20fe8ed8 100644 --- a/src/device/interrupt.rs +++ b/src/device/interrupt.rs @@ -69,6 +69,14 @@ pub trait ExternalInterruptController: Device { handler: &'static (dyn InterruptSource + Sync), ) -> Result<(), Error>; + /// Configures how an interrupt should be handled by the controller + fn configure_irq( + &self, + irq: Self::IrqNumber, + active_low: bool, + level_triggered: bool, + ) -> Result<(), Error>; + /// Enables given interrupt number/vector fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error>; } diff --git a/src/device/mod.rs b/src/device/mod.rs index c699c79d..fb4fc388 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,5 +1,4 @@ //! Device management and interfaces -use abi::error::Error; pub mod display; pub mod input; @@ -11,12 +10,12 @@ pub mod tty; /// General device interface pub trait Device { - /// Initializes the device to a state where it can be used. - /// - /// # Safety - /// - /// Unsafe to call if the device has already been initialized. - unsafe fn init(&self) -> Result<(), Error>; + // /// Initializes the device to a state where it can be used. + // /// + // /// # Safety + // /// + // /// Unsafe to call if the device has already been initialized. + // unsafe fn init(&self) -> Result<(), Error>; /// Returns a display name for the device fn name(&self) -> &'static str; diff --git a/src/device/platform.rs b/src/device/platform.rs index 84a62f8b..0faf3e5c 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -4,7 +4,7 @@ use abi::error::Error; use crate::debug::DebugSink; -use super::interrupt::ExternalInterruptController; +use super::{interrupt::ExternalInterruptController, timer::TimestampSource}; /// Platform interface for interacting with a general hardware set pub trait Platform { @@ -47,10 +47,10 @@ pub trait Platform { fn interrupt_controller(&self) -> &dyn ExternalInterruptController; - // /// Returns the platform's primary timestamp source. - // /// - // /// # Note - // /// - // /// May not be initialized at the moment of calling. - // fn timestamp_source(&self) -> &dyn TimestampSource; + /// Returns the platform's primary timestamp source. + /// + /// # Note + /// + /// May not be initialized at the moment of calling. + fn timestamp_source(&self) -> &dyn TimestampSource; } diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index 44e2a1e2..cc4765f4 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -1,8 +1,6 @@ //! Serial device interfaces use abi::error::Error; -use crate::debug::DebugSink; - use super::Device; #[cfg(target_arch = "aarch64")] diff --git a/src/device/tty.rs b/src/device/tty.rs index 4908168d..5be9bdbf 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -10,7 +10,6 @@ use abi::{ use vfs::CharDevice; use crate::{ - debug::DebugSink, device::display::fb_console::FramebufferConsole, proc::wait::Wait, sync::IrqSafeSpinlock, @@ -60,10 +59,6 @@ impl SerialDevice for CombinedTerminal { } impl Device for CombinedTerminal { - unsafe fn init(&self) -> Result<(), Error> { - todo!() - } - fn name(&self) -> &'static str { "Combined terminal device" } diff --git a/src/main.rs b/src/main.rs index 3eb7c487..27fce02b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,12 +16,16 @@ #![no_std] #![no_main] +use core::time::Duration; + use abi::{ error::Error, io::{FileMode, OpenOptions, RawFd}, }; +use device::display::console::task_update_consoles; use fs::{devfs, FileBlockAllocator, INITRD_DATA}; use memfs::MemoryFilesystem; +use task::tasklet; use vfs::{Filesystem, IoContext, VnodeRef}; extern crate yggdrasil_abi as abi; @@ -56,18 +60,11 @@ fn setup_root() -> Result { /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kernel_main() { - // use crate::{debug::LogLevel, mem::phys}; - // use crate::task::tasklet::{self, TaskFlow}; - // use core::time::Duration; - - // Schedule a tasklet to print memory usage stats every 10 seconds - // tasklet::add_periodic("mem-stats", Duration::from_secs(3), || { - // let phys_mem = phys::PHYSICAL_MEMORY.get().lock(); - // let stats = phys_mem.stats(); - // stats.dump(LogLevel::Debug); - - // TaskFlow::Continue - // }); + tasklet::add_periodic( + "update-console", + Duration::from_millis(15), + task_update_consoles, + ); let root = match setup_root() { Ok(root) => root, diff --git a/src/proc/wait.rs b/src/proc/wait.rs index ad1cbc5f..c45cc14b 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -5,6 +5,8 @@ use abi::error::Error; use alloc::{collections::LinkedList, rc::Rc}; use crate::{ + arch::PLATFORM, + device::platform::Platform, sync::IrqSafeSpinlock, task::process::{Process, ProcessState}, }; @@ -121,23 +123,23 @@ impl Wait { queue_lock = self.queue.lock(); - if let Some(_deadline) = deadline { - // let now = PLATFORM.timestamp_source().timestamp()?; + if let Some(deadline) = deadline { + let now = PLATFORM.timestamp_source().timestamp()?; - // if now > deadline { - // let mut cursor = queue_lock.cursor_front_mut(); + if now > deadline { + let mut cursor = queue_lock.cursor_front_mut(); - // while let Some(item) = cursor.current() { - // if item.id() == process.id() { - // cursor.remove_current(); - // return Err(Error::TimedOut); - // } else { - // cursor.move_next(); - // } - // } + while let Some(item) = cursor.current() { + if item.id() == process.id() { + cursor.remove_current(); + return Err(Error::TimedOut); + } else { + cursor.move_next(); + } + } - // // Most likely the process was killed by a signal - // } + // Most likely the process was killed by a signal + } } } } @@ -146,36 +148,35 @@ impl Wait { static TICK_LIST: IrqSafeSpinlock> = IrqSafeSpinlock::new(LinkedList::new()); /// Suspends current task until given deadline -pub fn sleep(_timeout: Duration, _remaining: &mut Duration) -> Result<(), Error> { - todo!(); - // static SLEEP_NOTIFY: Wait = Wait::new("sleep"); - // let now = PLATFORM.timestamp_source().timestamp()?; - // let deadline = now + timeout; +pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { + static SLEEP_NOTIFY: Wait = Wait::new("sleep"); + let now = PLATFORM.timestamp_source().timestamp()?; + let deadline = now + timeout; - // match SLEEP_NOTIFY.wait(Some(deadline)) { - // // Just what we expected - // Err(Error::TimedOut) => { - // *remaining = Duration::ZERO; - // Ok(()) - // } + match SLEEP_NOTIFY.wait(Some(deadline)) { + // Just what we expected + Err(Error::TimedOut) => { + *remaining = Duration::ZERO; + Ok(()) + } - // Ok(_) => panic!("This should not happen"), - // Err(e) => Err(e), - // } + Ok(_) => panic!("This should not happen"), + Err(e) => Err(e), + } } /// Updates all pending timeouts and wakes up the tasks that have reached theirs -pub fn tick(_now: Duration) { - // let mut list = TICK_LIST.lock(); - // let mut cursor = list.cursor_front_mut(); +pub fn tick(now: Duration) { + let mut list = TICK_LIST.lock(); + let mut cursor = list.cursor_front_mut(); - // while let Some(item) = cursor.current() { - // if now > item.deadline { - // let t = cursor.remove_current().unwrap(); + while let Some(item) = cursor.current() { + if now > item.deadline { + let t = cursor.remove_current().unwrap(); - // t.process.enqueue_somewhere(); - // } else { - // cursor.move_next(); - // } - // } + t.process.enqueue_somewhere(); + } else { + cursor.move_next(); + } + } } From ea6684d91633d6ccd4e534a2a4af66057e4e5987 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 4 Aug 2023 10:07:21 +0300 Subject: [PATCH 036/211] dev: better trait for device init --- src/arch/aarch64/gic/mod.rs | 17 ++++++++-- src/arch/aarch64/plat_qemu/mod.rs | 27 +++++---------- src/arch/aarch64/timer.rs | 11 +++++-- src/arch/x86_64/apic/ioapic.rs | 51 +++++++++++++++-------------- src/arch/x86_64/mod.rs | 9 +++-- src/arch/x86_64/peripherals/hpet.rs | 32 ++++++++++-------- src/device/mod.rs | 22 +++++++++---- src/device/serial/pl011.rs | 11 +++++-- 8 files changed, 105 insertions(+), 75 deletions(-) diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 23aa6d36..e9161f0e 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -6,7 +6,7 @@ use spinning_top::Spinlock; use crate::{ device::{ interrupt::{ExternalInterruptController, InterruptSource, IrqContext}, - Device, + Device, InitializableDevice, }, mem::device::{DeviceMemory, DeviceMemoryIo}, util::OneTimeInit, @@ -59,8 +59,12 @@ impl Device for Gic { fn name(&self) -> &'static str { "ARM Generic Interrupt Controller v2" } +} - unsafe fn init(&self) -> Result<(), Error> { +impl InitializableDevice for Gic { + type Options = (); + + unsafe fn init(&self, _opts: ()) -> Result<(), Error> { let gicd_mmio = DeviceMemory::map("GICv2 Distributor registers", self.gicd_base, 0x1000)?; let gicd_mmio_shared = DeviceMemoryIo::new(gicd_mmio.clone()); let gicd_mmio_banked = DeviceMemoryIo::new(gicd_mmio); @@ -104,6 +108,15 @@ impl ExternalInterruptController for Gic { Ok(()) } + fn configure_irq( + &self, + _irq: Self::IrqNumber, + _active_low: bool, + _level_triggered: bool, + ) -> Result<(), Error> { + todo!() + } + // unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { // // TODO message queue insertion should be moved // match target { diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs index 0750633c..e7e4c9b4 100644 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -9,7 +9,8 @@ use crate::{ interrupt::{ExternalInterruptController, InterruptSource}, platform::Platform, serial::pl011::Pl011, - Device, + timer::TimestampSource, + InitializableDevice, }, fs::devfs::{self, CharDeviceType}, }; @@ -34,12 +35,12 @@ impl Platform for QemuPlatform { unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { if is_bsp { - self.gic.init()?; + self.gic.init(())?; self.pl011.init_irq()?; devfs::add_char_device(&self.pl011, CharDeviceType::TtySerial)?; - self.local_timer.init()?; + self.local_timer.init(())?; self.local_timer.init_irq()?; } else { self.gic.init_smp_ap()?; @@ -54,7 +55,7 @@ impl Platform for QemuPlatform { } unsafe fn init_primary_debug_sink(&self) { - self.pl011.init().ok(); + self.pl011.init(()).ok(); } fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { @@ -67,25 +68,13 @@ impl Platform for QemuPlatform { &self.gic } - // unsafe fn init_primary_serial(&self) { - // self.pl011.init().ok(); - // } - fn name(&self) -> &'static str { "qemu" } - // fn primary_serial(&self) -> Option<&dyn SerialDevice> { - // Some(&self.pl011) - // } - - // fn interrupt_controller(&self) -> &dyn InterruptController { - // &self.gic - // } - - // fn timestamp_source(&self) -> &dyn TimestampSource { - // &self.local_timer - // } + fn timestamp_source(&self) -> &dyn TimestampSource { + &self.local_timer + } } /// AArch64 "virt" platform diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 2f010428..63416dd7 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -8,7 +8,10 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ arch::PLATFORM, - device::{interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device}, + device::{ + interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device, + InitializableDevice, + }, proc::wait, task::tasklet, }; @@ -27,8 +30,12 @@ impl Device for ArmTimer { fn name(&self) -> &'static str { "ARM Generic Timer" } +} - unsafe fn init(&self) -> Result<(), Error> { +impl InitializableDevice for ArmTimer { + type Options = (); + + unsafe fn init(&self, _opts: ()) -> Result<(), Error> { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); Ok(()) } diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 2517b19c..d1b160ee 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -6,7 +6,7 @@ use crate::{ arch::x86_64::apic::local::BSP_APIC_ID, device::{ interrupt::{ExternalInterruptController, InterruptSource}, - Device, + Device, InitializableDevice, }, mem::ConvertAddress, sync::IrqSafeSpinlock, @@ -202,30 +202,10 @@ impl ExternalInterruptController for IoApic { } } -impl IoApic { - /// Constructs an uninitialized I/O APIC interface - pub const fn new() -> Self { - Self { - inner: OneTimeInit::new(), - isa_redirections: OneTimeInit::new(), - table: IrqSafeSpinlock::new(Table::new()), - } - } +impl InitializableDevice for IoApic { + type Options = AcpiApic; - /// Handles an interrupt with given GSI number - pub fn handle_irq(&self, gsi: u32) { - let table = self.table.lock(); - - if let Some(handler) = table.vectors[gsi as usize] { - drop(table); - handler.handle_irq().expect("IRQ handler failed"); - } else { - panic!("No handler set for GSI #{}", gsi); - } - } - - /// Initializes the I/O APIC interface using information provided by ACPI - pub fn init_from_acpi(&self, info: &AcpiApic) -> Result<(), Error> { + unsafe fn init(&self, info: Self::Options) -> Result<(), Error> { let io_apic = info.io_apics.first().unwrap(); infoln!("I/O APIC at {:#x}", io_apic.address); @@ -271,6 +251,29 @@ impl IoApic { Ok(()) } +} + +impl IoApic { + /// Constructs an uninitialized I/O APIC interface + pub const fn new() -> Self { + Self { + inner: OneTimeInit::new(), + isa_redirections: OneTimeInit::new(), + table: IrqSafeSpinlock::new(Table::new()), + } + } + + /// Handles an interrupt with given GSI number + pub fn handle_irq(&self, gsi: u32) { + let table = self.table.lock(); + + if let Some(handler) = table.vectors[gsi as usize] { + drop(table); + handler.handle_irq().expect("IRQ handler failed"); + } else { + panic!("No handler set for GSI #{}", gsi); + } + } fn translate_irq(&self, irq: IrqNumber) -> u32 { let redir = self.isa_redirections.get(); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5da895a6..e90291fe 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -19,6 +19,7 @@ use crate::{ platform::Platform, timer::TimestampSource, tty::CombinedTerminal, + InitializableDevice, }, fs::{ devfs::{self, CharDeviceType}, @@ -190,8 +191,6 @@ impl Platform for X86_64 { const KERNEL_PHYS_BASE: usize = 0x400000; unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { - // self.apic.init()?; - if is_bsp { Self::disable_8259(); @@ -202,12 +201,12 @@ impl Platform for X86_64 { let hpet = HpetInfo::new(&acpi_tables).unwrap(); let platform_info = acpi_tables.platform_info().unwrap(); - let InterruptModel::Apic(apic_info) = &platform_info.interrupt_model else { + let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { panic!("Processor does not have APIC"); }; - self.ioapic.init_from_acpi(apic_info)?; - self.timer.init(&hpet).unwrap(); + self.ioapic.init(apic_info)?; + self.timer.init(hpet).unwrap(); // Enable IRQs for the devices self.ps2.init_irq()?; diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 0b9e9071..7bc5edb1 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -11,7 +11,10 @@ use tock_registers::{ use crate::{ arch::{x86_64::apic::IrqNumber, PLATFORM}, - device::{interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device}, + device::{ + interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device, + InitializableDevice, + }, mem::device::DeviceMemoryIo, proc::wait, sync::IrqSafeSpinlock, @@ -235,6 +238,21 @@ impl InterruptSource for Hpet { } } +impl InitializableDevice for Hpet { + type Options = AcpiHpet; + + unsafe fn init(&self, info: Self::Options) -> Result<(), Error> { + infoln!("Initializing HPET:"); + infoln!("Address: {:#x}", info.base_address); + + let inner = unsafe { Inner::new(info.base_address) }?; + + self.inner.init(IrqSafeSpinlock::new(inner)); + + Ok(()) + } +} + impl Device for Hpet { fn name(&self) -> &'static str { "HPET" @@ -248,16 +266,4 @@ impl Hpet { inner: OneTimeInit::new(), } } - - /// Initialize HPET from ACPI definition - pub fn init(&self, hpet: &AcpiHpet) -> Result<(), Error> { - infoln!("Initializing HPET:"); - infoln!("Address: {:#x}", hpet.base_address); - - let inner = unsafe { Inner::new(hpet.base_address) }?; - - self.inner.init(IrqSafeSpinlock::new(inner)); - - Ok(()) - } } diff --git a/src/device/mod.rs b/src/device/mod.rs index fb4fc388..0660d4a6 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,5 +1,7 @@ //! Device management and interfaces +use abi::error::Error; + pub mod display; pub mod input; pub mod interrupt; @@ -10,13 +12,19 @@ pub mod tty; /// General device interface pub trait Device { - // /// Initializes the device to a state where it can be used. - // /// - // /// # Safety - // /// - // /// Unsafe to call if the device has already been initialized. - // unsafe fn init(&self) -> Result<(), Error>; - /// Returns a display name for the device fn name(&self) -> &'static str; } + +/// Interface for device initialization +pub trait InitializableDevice { + /// Options provided when initializing the device + type Options; + + /// Initializes the device. + /// + /// # Safety + /// + /// Unsafe: only meant to be called once for each device. + unsafe fn init(&self, opts: Self::Options) -> Result<(), Error>; +} diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 2c011fe0..4144310c 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -10,11 +10,12 @@ use vfs::CharDevice; use super::SerialDevice; use crate::{ arch::{aarch64::gic::IrqNumber, PLATFORM}, + debug::DebugSink, device::{ interrupt::InterruptSource, platform::Platform, tty::{CharRing, TtyDevice}, - Device, + Device, InitializableDevice, }, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, @@ -170,8 +171,10 @@ impl InterruptSource for Pl011 { } } -impl Device for Pl011 { - unsafe fn init(&self) -> Result<(), Error> { +impl InitializableDevice for Pl011 { + type Options = (); + + unsafe fn init(&self, _opts: ()) -> Result<(), Error> { let mut inner = Pl011Inner { regs: DeviceMemoryIo::map("pl011 UART", self.base)?, }; @@ -180,7 +183,9 @@ impl Device for Pl011 { self.inner.init(IrqSafeSpinlock::new(inner)); Ok(()) } +} +impl Device for Pl011 { fn name(&self) -> &'static str { "pl011" } From 5dd3a252c66c77c8168389132b811e6d75384159 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 4 Aug 2023 11:00:31 +0300 Subject: [PATCH 037/211] dev: better key handling --- src/arch/x86_64/peripherals/ps2/codeset.rs | 9 +- src/arch/x86_64/peripherals/ps2/mod.rs | 196 ++++++++++++++++----- src/device/display/console.rs | 12 ++ src/device/tty.rs | 2 +- 4 files changed, 172 insertions(+), 47 deletions(-) diff --git a/src/arch/x86_64/peripherals/ps2/codeset.rs b/src/arch/x86_64/peripherals/ps2/codeset.rs index ceedeecd..d3a3ca0f 100644 --- a/src/arch/x86_64/peripherals/ps2/codeset.rs +++ b/src/arch/x86_64/peripherals/ps2/codeset.rs @@ -2,5 +2,12 @@ pub static CODE_SET_1_00: &[u8] = &[ 0x00, b'\x1b', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'-', b'=', b'\x7f', b'\t', b'q', b'w', b'e', b'r', b't', b'y', b'u', b'i', b'o', b'p', b'[', b']', b'\n', 0x00, b'a', b's', b'd', b'f', b'g', b'h', b'j', b'k', b'l', b';', b'\'', b'`', 0x00, b'\\', b'z', - b'x', b'c', b'v', b'b', b'n', b'm', b',', b'.', b'/', 0x00, + b'x', b'c', b'v', b'b', b'n', b'm', b',', b'.', b'/', 0x00, b'*', 0x00, b' ', +]; + +pub static CODE_SET_1_00_SHIFT: &[u8] = &[ + 0x00, b'\x1b', b'!', b'@', b'#', b'$', b'%', b'^', b'&', b'*', b'(', b')', b'_', b'+', b'\x7f', + b'\t', b'Q', b'W', b'E', b'R', b'T', b'Y', b'U', b'I', b'O', b'P', b'{', b'}', b'\n', 0x00, + b'A', b'S', b'D', b'F', b'G', b'H', b'J', b'K', b'L', b':', b'"', b'~', 0x00, b'|', b'Z', b'X', + b'C', b'V', b'B', b'N', b'M', b'<', b'>', b'?', 0x00, ]; diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index dbb355b1..7ed2ff24 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -13,19 +13,27 @@ use crate::{ sync::IrqSafeSpinlock, }; -use self::codeset::CODE_SET_1_00; +use self::codeset::{CODE_SET_1_00, CODE_SET_1_00_SHIFT}; mod codeset; -/// PS/2 controller driver -pub struct PS2Controller { +struct Inner { command: IoPort, data: IoPort, + + shift: bool, + ctrl: bool, + + tty: Option<&'static dyn TtyDevice<16>>, +} + +/// PS/2 controller driver +pub struct PS2Controller { primary_irq: IrqNumber, #[allow(unused)] auxiliary_irq: IrqNumber, - tty: IrqSafeSpinlock>>, + inner: IrqSafeSpinlock, } fn translate(codeset: &[u8], key: u8) -> Option { @@ -40,29 +48,62 @@ fn translate(codeset: &[u8], key: u8) -> Option { } } +impl Inner { + const STATUS_OUTPUT_FULL: u8 = 1 << 0; + const STATUS_INPUT_FULL: u8 = 1 << 1; + + fn send_command(&mut self, cmd: u8) { + while self.command.read() & Self::STATUS_INPUT_FULL != 0 { + core::hint::spin_loop(); + } + self.command.write(cmd); + } + + fn recv_timeout(&mut self, timeout: u64) -> Option { + let mut counter = 0; + while self.command.read() & Self::STATUS_OUTPUT_FULL == 0 { + counter += 1; + if counter > timeout { + return None; + } + core::hint::spin_loop(); + } + Some(self.data.read()) + } + + fn try_recv(&mut self) -> Option { + if self.command.read() & Inner::STATUS_OUTPUT_FULL != 0 { + Some(self.data.read()) + } else { + None + } + } +} + impl KeyboardDevice for PS2Controller { fn attach(&self, terminal: &'static dyn TtyDevice<16>) { - self.tty.lock().replace(terminal); + self.inner.lock().tty.replace(terminal); } } impl InterruptSource for PS2Controller { unsafe fn init_irq(&'static self) -> Result<(), Error> { + let mut inner = self.inner.lock(); let intc = PLATFORM.interrupt_controller(); intc.register_handler(self.primary_irq, self)?; // Disable PS/2 devices from sending any further data - self.send_command(0xAD); - self.send_command(0xA7); + inner.send_command(0xAD); + inner.send_command(0xA7); // Flush the buffer - while self.command.read() & Self::STATUS_OUTPUT_FULL != 0 { - self.data.read(); + while inner.command.read() & Inner::STATUS_OUTPUT_FULL != 0 { + inner.data.read(); } // Enable primary port - self.send_command(0xAE); + inner.send_command(0xAE); intc.enable_irq(self.primary_irq)?; @@ -70,34 +111,103 @@ impl InterruptSource for PS2Controller { } fn handle_irq(&self) -> Result { - let status = self.command.read(); - - if status & 1 == 0 { - return Ok(false); - } - - let key = self.data.read(); - - if key == 0xE0 { - self.data.read(); - infoln!("TODO: handle 0xE0"); - return Ok(true); - } - - self.data.read(); - - if key < 128 { - let Some(code) = translate(CODE_SET_1_00, key) else { - return Ok(true); + let mut count = 0; + let mut inner = self.inner.lock(); + loop { + let Some(mut scancode) = inner.try_recv() else { + break; }; - let terminal = self.tty.lock(); - if let Some(terminal) = &*terminal { - terminal.recv_byte(code); + let e0 = scancode == 0xE0; + + if e0 { + scancode = inner.recv_timeout(100000000).unwrap(); } + + if !e0 { + match scancode { + // LCtrl pressed + 0x1D => { + inner.ctrl = true; + continue; + } + // LCtrl released + 0x9D => { + inner.ctrl = false; + continue; + } + // LShift pressed + 0x2A => { + inner.shift = true; + continue; + } + // LShift released + 0xAA => { + inner.shift = false; + continue; + } + _ => {} + } + } + + if scancode > 128 { + continue; + } + + let key = match (inner.shift, inner.ctrl, e0) { + // No shift, no ctrl + (false, false, false) => translate(CODE_SET_1_00, scancode).unwrap_or(0), + // Shift, no ctrl + (true, false, false) => translate(CODE_SET_1_00_SHIFT, scancode).unwrap_or(0), + // No shift, ctrl + (false, true, false) => { + let key = translate(CODE_SET_1_00, scancode).unwrap_or(0); + + if key == b'c' { + 0x3 + } else { + 0 + } + } + // Other keys + _ => 0, + }; + + if key == 0 { + continue; + } + + if let Some(tty) = inner.tty { + tty.recv_byte(key); + } + + count += 1; } Ok(true) + + // let status = self.command.read(); + + // if status & 1 == 0 { + // return Ok(false); + // } + + // let key = self.data.read(); + + // if key == 0xE0 { + // self.data.read(); + // infoln!("TODO: handle 0xE0"); + // return Ok(true); + // } + + // self.data.read(); + + // if key < 128 { + // let terminal = self.tty.lock(); + + // } + + // Ok(true) } } @@ -108,9 +218,6 @@ impl Device for PS2Controller { } impl PS2Controller { - const STATUS_OUTPUT_FULL: u8 = 1 << 0; - const STATUS_INPUT_FULL: u8 = 1 << 1; - /// Constructs a new instance of the device pub const fn new( primary_irq: IrqNumber, @@ -118,19 +225,18 @@ impl PS2Controller { cmd_port: u16, data_port: u16, ) -> Self { + let inner = Inner { + command: IoPort::new(cmd_port), + data: IoPort::new(data_port), + shift: false, + ctrl: false, + tty: None, + }; + Self { primary_irq, auxiliary_irq, - command: IoPort::new(cmd_port), - data: IoPort::new(data_port), - tty: IrqSafeSpinlock::new(None), + inner: IrqSafeSpinlock::new(inner), } } - - fn send_command(&self, cmd: u8) { - while self.command.read() & Self::STATUS_INPUT_FULL != 0 { - core::hint::spin_loop(); - } - self.command.write(cmd); - } } diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 4aab22bc..c94826d3 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -266,11 +266,23 @@ impl ConsoleState { let mut flush = false; match c { + c if c > 127 => { + self.buffer.set_char( + self.cursor_row, + self.cursor_col, + ConsoleChar::from_parts(b'?', self.fg_color, ColorAttribute::Red), + ); + + self.cursor_col += 1; + } b'\n' => { self.cursor_row += 1; self.cursor_col = 0; flush = true; } + b'\x7f' => { + return false; + } _ => { self.buffer.set_char( self.cursor_row, diff --git a/src/device/tty.rs b/src/device/tty.rs index 5be9bdbf..13b2963e 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -165,7 +165,7 @@ pub trait TtyDevice: SerialDevice { } // byte == config.chars.interrupt - if byte == b'=' && config.line.contains(TerminalLineOptions::SIGNAL) { + if byte == config.chars.interrupt && config.line.contains(TerminalLineOptions::SIGNAL) { drop(config); let pgrp = ring.inner.lock().process_group; if let Some(pgrp) = pgrp { From 111514275cb492c4b79d573f0b0d1859b0b545d3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 5 Aug 2023 16:32:12 +0300 Subject: [PATCH 038/211] x86-64: SMP init --- Cargo.toml | 1 + build.rs | 46 +++++++ src/arch/mod.rs | 3 + src/arch/x86_64/apic/ioapic.rs | 7 +- src/arch/x86_64/apic/local.rs | 176 ++++++++++++++++++++++--- src/arch/x86_64/apic/mod.rs | 69 ++++++---- src/arch/x86_64/apic/vectors.S | 172 ++++++++++++++++++++++++ src/arch/x86_64/boot/ap_boot.S | 110 ++++++++++++++++ src/arch/x86_64/boot/mod.rs | 107 ++++----------- src/arch/x86_64/cpu.rs | 10 +- src/arch/x86_64/exception.rs | 34 +++-- src/arch/x86_64/gdt.rs | 111 ++++++++-------- src/arch/x86_64/mod.rs | 151 +++++++++++++++++++-- src/arch/x86_64/peripherals/ps2/mod.rs | 25 +--- src/arch/x86_64/registers/mod.rs | 2 + src/arch/x86_64/smp.rs | 124 +++++++++++++++++ src/arch/x86_64/table/fixed.rs | 4 +- src/arch/x86_64/vectors.S | 126 +----------------- src/debug.rs | 2 +- src/device/interrupt.rs | 11 -- src/device/platform.rs | 19 ++- src/panic.rs | 31 ++++- src/proc/wait.rs | 12 +- src/sync.rs | 10 ++ src/syscall/mod.rs | 2 - src/task/mod.rs | 5 +- src/task/process.rs | 8 ++ src/task/sched.rs | 24 +++- 28 files changed, 1020 insertions(+), 382 deletions(-) create mode 100644 build.rs create mode 100644 src/arch/x86_64/apic/vectors.S create mode 100644 src/arch/x86_64/boot/ap_boot.S create mode 100644 src/arch/x86_64/smp.rs diff --git a/Cargo.toml b/Cargo.toml index 805002a3..62264045 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "yggdrasil-kernel" version = "0.1.0" edition = "2021" +build = "build.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/build.rs b/build.rs new file mode 100644 index 00000000..c015254e --- /dev/null +++ b/build.rs @@ -0,0 +1,46 @@ +use std::{ + env, + io::{self, Write}, + path::PathBuf, + process::Command, +}; + +fn build_x86_64() { + const DEFAULT_8086_AS: &str = "nasm"; + const AP_BOOTSTRAP_S: &str = "src/arch/x86_64/boot/ap_boot.S"; + + println!("cargo:rerun-if-changed={}", AP_BOOTSTRAP_S); + + let out_dir = env::var("OUT_DIR").unwrap(); + let assembler = env::var("AS8086").unwrap_or(DEFAULT_8086_AS.to_owned()); + + let ap_bootstrap_out = PathBuf::from(out_dir).join("__x86_64_ap_boot.bin"); + + // Assemble the code + let output = Command::new(assembler.as_str()) + .args([ + "-fbin", + "-o", + ap_bootstrap_out.to_str().unwrap(), + AP_BOOTSTRAP_S, + ]) + .output() + .unwrap(); + + if !output.status.success() { + io::stderr().write_all(&output.stderr).ok(); + panic!("{}: could not assemble {}", assembler, AP_BOOTSTRAP_S); + } +} + +fn main() { + let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + + println!("cargo:rerun-if-changed=build.rs"); + + match arch.as_str() { + "x86_64" => build_x86_64(), + "aarch64" => (), + _ => panic!("Unknown target arch: {:?}", arch), + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index cdf04700..9e7cbd50 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -82,4 +82,7 @@ pub trait Architecture { /// Returns the local CPU's interrupt mask fn interrupt_mask() -> bool; + + /// Returns the count of present CPUs, including the BSP + fn cpu_count() -> usize; } diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index d1b160ee..91db01c4 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -244,7 +244,12 @@ impl InitializableDevice for IoApic { infoln!("Maximum GSI number: {}", max_gsi); - let inner = Inner { regs, max_gsi }; + let mut inner = Inner { regs, max_gsi }; + + // Mask all GSIs + for gsi in 0..max_gsi { + inner.set_gsi_enabled(gsi, false); + } self.inner.init(IrqSafeSpinlock::new(inner)); self.isa_redirections.init(isa_redirections); diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 4e7399fe..3f0b43fa 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -5,9 +5,14 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use crate::{arch::x86_64::registers::MSR_IA32_APIC_BASE, mem::ConvertAddress, util::OneTimeInit}; +use crate::{ + arch::x86_64::registers::MSR_IA32_APIC_BASE, device::interrupt::IpiDeliveryTarget, + mem::ConvertAddress, util::OneTimeInit, +}; -use super::APIC_TIMER_VECTOR; +use super::{ + APIC_IPI_VECTOR, APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR, +}; const TIMER_INTERVAL: u32 = 150000; @@ -20,7 +25,8 @@ register_bitfields! { ApicId OFFSET(24) NUMBITS(8) [] ], SpuriousVector [ - SoftwareEnable OFFSET(8) NUMBITS(1), + Vector OFFSET(0) NUMBITS(8) [], + SoftwareEnable OFFSET(8) NUMBITS(1) [], ], TimerLocalVectorEntry [ Vector OFFSET(0) NUMBITS(8) [], @@ -39,10 +45,40 @@ register_bitfields! { Masked = 1, Unmasked = 0, ], - Nmi OFFSET(8) NUMBITS(3) [ - IsNmi = 4, + DeliveryMode OFFSET(8) NUMBITS(3) [ + Nmi = 4, + ExtINT = 7 ], - ] + ], + ICR0 [ + Vector OFFSET(0) NUMBITS(8) [], + Destination OFFSET(8) NUMBITS(3) [ + Normal = 1, + Lowest = 2, + SMI = 3, + NMI = 4, + INIT = 5, + SIPI = 6 + ], + DeliveryStatus OFFSET(12) NUMBITS(1) [], + INIT0 OFFSET(14) NUMBITS(1) [ + Deassert = 0, + Assert = 1, + ], + INIT1 OFFSET(15) NUMBITS(1) [ + Deassert = 1, + Assert = 0, + ], + DestinationType OFFSET(18) NUMBITS(3) [ + Physical = 0, + This = 1, + All = 2, + AllExceptThis = 3, + ] + ], + ICR1 [ + PhysicalDestination OFFSET(24) NUMBITS(4) [] + ], } register_structs! { @@ -51,22 +87,32 @@ register_structs! { (0x00 => _0), (0x20 => Id: ReadOnly), (0x24 => _1), + (0x80 => TaskPriorityRegister: ReadWrite), + (0x84 => _13), (0xB0 => EndOfInterrupt: WriteOnly), (0xB4 => _2), (0xF0 => SpuriousVector: ReadWrite), (0xF4 => _3), + (0x100 => ISR0: ReadOnly), + (0x104 => _14), + (0x280 => ErrorStatus: ReadOnly), + (0x284 => _4), + (0x300 => ICR0: ReadWrite), + (0x304 => _5), + (0x310 => ICR1: ReadWrite), + (0x314 => _6), (0x320 => TimerLocalVectorEntry: ReadWrite), - (0x324 => _4), + (0x324 => _7), (0x350 => LInt0: ReadWrite), - (0x354 => _5), + (0x354 => _8), (0x360 => LInt1: ReadWrite), - (0x364 => _6), + (0x364 => _9), (0x380 => TimerInitCount: ReadWrite), - (0x384 => _7), + (0x384 => _10), (0x390 => TimerCurrentCount: ReadOnly), - (0x394 => _8), + (0x394 => _11), (0x3E0 => TimerDivideConfig: ReadWrite), - (0x3E4 => _9), + (0x3E4 => _12), (0x530 => @END), } } @@ -93,19 +139,34 @@ impl LocalApic { } Self::enable(); + // Configure spurious interrupt handler + regs.SpuriousVector.write( + SpuriousVector::SoftwareEnable::SET + + SpuriousVector::Vector.val(APIC_SPURIOUS_VECTOR + 32), + ); + + // Configure task priority register + regs.TaskPriorityRegister.set(0); // Enable timer - regs.TimerInitCount.set(TIMER_INTERVAL); regs.TimerDivideConfig.set(0x3); + regs.TimerInitCount.set(TIMER_INTERVAL); + // Configure local interrupt vectors regs.TimerLocalVectorEntry.write( TimerLocalVectorEntry::Vector.val(APIC_TIMER_VECTOR + 32) + TimerLocalVectorEntry::Mask::Unmasked + TimerLocalVectorEntry::TimerMode::Periodic, ); - - regs.SpuriousVector - .modify(SpuriousVector::SoftwareEnable::SET); + // LINT0 unmasked, leave LINT1 masked + regs.LInt0.write( + LocalVectorEntry::Mask::Unmasked + + LocalVectorEntry::Vector.val(APIC_LINT0_VECTOR + 32) + + LocalVectorEntry::DeliveryMode::ExtINT, + ); + regs.LInt1.write( + LocalVectorEntry::Mask::Masked + LocalVectorEntry::Vector.val(APIC_LINT1_VECTOR + 32), + ); Self { regs } } @@ -115,6 +176,85 @@ impl LocalApic { self.regs.EndOfInterrupt.set(0); } + /// Performs an application processor startup sequence. + /// + /// # Safety + /// + /// Unsafe: only meant to be called by the BSP during SMP init. + pub unsafe fn wakeup_cpu(&self, apic_id: u32, entry_vector: usize) { + infoln!("Waking up apic{}, entry = {:#x}", apic_id, entry_vector); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + let entry_vector = entry_vector >> 12; + + // INIT assert + self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); + + self.regs.ICR0.write( + ICR0::Destination::INIT + + ICR0::DestinationType::Physical + + ICR0::INIT0::Assert + + ICR0::INIT1::Assert, + ); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + // INIT deassert + self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); + + self.regs.ICR0.write( + ICR0::Destination::INIT + + ICR0::DestinationType::Physical + + ICR0::INIT0::Deassert + + ICR0::INIT1::Deassert, + ); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + // Send another SIPI type IPI because the spec says so + self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); + + self.regs.ICR0.write( + ICR0::Vector.val(entry_vector as u32) + + ICR0::Destination::SIPI + + ICR0::DestinationType::Physical, + ); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + } + + /// Issues an interprocessor interrupt for the target. + /// + /// # Safety + /// + /// Unsafe: this function may break the control flow on the target processors. + pub unsafe fn send_ipi(&self, target: IpiDeliveryTarget) { + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + match target { + IpiDeliveryTarget::AllExceptLocal => { + self.regs.ICR1.write(ICR1::PhysicalDestination.val(0)); + self.regs.ICR0.write( + ICR0::Vector.val(APIC_IPI_VECTOR + 32) + + ICR0::Destination::Normal + + ICR0::DestinationType::AllExceptThis, + ); + } + IpiDeliveryTarget::Specified(_) => todo!(), + } + } + #[inline] fn base() -> usize { MSR_IA32_APIC_BASE.read_base() as usize @@ -127,6 +267,8 @@ impl LocalApic { #[inline] fn enable() { - MSR_IA32_APIC_BASE.modify(MSR_IA32_APIC_BASE::ApicEnable::SET); + MSR_IA32_APIC_BASE.modify( + MSR_IA32_APIC_BASE::ApicEnable::SET + MSR_IA32_APIC_BASE::ExtendedEnable::CLEAR, + ); } } diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index a2f21c1b..5f5fdea5 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -1,5 +1,7 @@ //! x86-64 APIC interface (Local + I/O) +use core::arch::global_asm; + use abi::error::Error; use crate::{ @@ -16,7 +18,15 @@ pub mod local; // I/O APIC 0..MAX_EXTERNAL_VECTORS range is mapped to BSP Local APIC 2.. /// Fixed IRQ vector for Local APIC timer -pub const APIC_TIMER_VECTOR: u32 = 0; +pub const APIC_TIMER_VECTOR: u32 = 0x00; +/// Fixed IRQ vector for LINT0 line +pub const APIC_LINT0_VECTOR: u32 = 0x01; +/// Fixed IRQ vector for LINT1 line +pub const APIC_LINT1_VECTOR: u32 = 0x02; +/// Fixed vector for inter-processor interrupt +pub const APIC_IPI_VECTOR: u32 = 0x03; +/// Fixed vector for spurious interrupt +pub const APIC_SPURIOUS_VECTOR: u32 = 0xDF; /// Start of the I/O APIC IRQ range pub const APIC_EXTERNAL_OFFSET: u32 = 4; /// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC @@ -56,7 +66,8 @@ impl Table { /// Fills the IDT with interrupt vectors for this APIC pub fn setup_vectors(idt: &mut [exception::Entry]) { extern "C" { - static __x86_64_apic_vectors: [usize; 16]; + // IRQ vectors + static __x86_64_apic_vectors: [usize; 224]; } for (i, &entry) in unsafe { __x86_64_apic_vectors.iter() }.enumerate() { @@ -68,32 +79,44 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) { } } -fn apic_irq_inner(vector: u32) { +unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { let cpu = Cpu::local(); - - if vector == APIC_TIMER_VECTOR { - cpu.local_apic().clear_interrupt(); - unsafe { Cpu::local().queue().yield_cpu() } - } else if (APIC_EXTERNAL_OFFSET..APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS).contains(&vector) - { - PLATFORM.ioapic.handle_irq(vector - APIC_EXTERNAL_OFFSET); - cpu.local_apic().clear_interrupt(); - } else { - panic!("Got an interrupt on undefined vector: {}", vector); - } -} - -/// Main handler for APIC interrupts. -/// -/// # Safety -/// -/// Only meant to be called from the assembly irq vector. -pub unsafe extern "C" fn __x86_64_apic_irq_handler(vector: u32, frame: *mut IrqFrame) { let frame = &mut *frame; - apic_irq_inner(vector); + PLATFORM.ioapic.handle_irq(vector as u32); + cpu.local_apic().clear_interrupt(); if let Some(process) = Process::get_current() { process.handle_signal(frame); } } + +unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { + let frame = &mut *frame; + let cpu = Cpu::local(); + // Clear interrupt before switching, because otherwise we won't receive the next one + cpu.local_apic().clear_interrupt(); + cpu.queue().yield_cpu(); + + if let Some(process) = Process::get_current() { + process.handle_signal(frame); + } +} + +unsafe extern "C" fn dummy_irq_handler() { + todo!() +} + +unsafe extern "C" fn ipi_handler() { + let cpu = Cpu::local(); + todo!("Processor {} received an IPI", cpu.id()); +} + +global_asm!( + include_str!("vectors.S"), + local_timer_irq_handler = sym local_timer_irq_handler, + irq_handler = sym irq_handler, + ipi_handler = sym ipi_handler, + dummy_irq_handler = sym dummy_irq_handler, + options(att_syntax) +); diff --git a/src/arch/x86_64/apic/vectors.S b/src/arch/x86_64/apic/vectors.S new file mode 100644 index 00000000..082ac5af --- /dev/null +++ b/src/arch/x86_64/apic/vectors.S @@ -0,0 +1,172 @@ +.altmacro + +.global __x86_64_apic_vectors + +.set IRQ_REG_RAX, 0 * 8 +.set IRQ_REG_RCX, 1 * 8 +.set IRQ_REG_RDX, 2 * 8 +.set IRQ_REG_RBX, 3 * 8 +.set IRQ_REG_RSI, 4 * 8 +.set IRQ_REG_RDI, 5 * 8 +.set IRQ_REG_RBP, 6 * 8 +.set IRQ_REG_R8, 7 * 8 +.set IRQ_REG_R9, 8 * 8 +.set IRQ_REG_R10, 9 * 8 +.set IRQ_REG_R11, 10 * 8 +.set IRQ_REG_R12, 11 * 8 +.set IRQ_REG_R13, 12 * 8 +.set IRQ_REG_R14, 13 * 8 +.set IRQ_REG_R15, 14 * 8 + +// 15 registers + stack align word if needed +.set IRQ_STATE_SIZE, 15 * 8 + +.macro SWAPGS_IF_NEEDED, cs_off + cmpq $0x08, \cs_off(%rsp) + je 1f + swapgs +1: +.endm + +.macro IRQ_SAVE_STATE + // Save state + subq $IRQ_STATE_SIZE, %rsp + + movq %rax, IRQ_REG_RAX(%rsp) + movq %rcx, IRQ_REG_RCX(%rsp) + movq %rdx, IRQ_REG_RDX(%rsp) + movq %rbx, IRQ_REG_RBX(%rsp) + movq %rsi, IRQ_REG_RSI(%rsp) + movq %rdi, IRQ_REG_RDI(%rsp) + movq %rbp, IRQ_REG_RBP(%rsp) + movq %r8, IRQ_REG_R8(%rsp) + movq %r9, IRQ_REG_R9(%rsp) + movq %r10, IRQ_REG_R10(%rsp) + movq %r11, IRQ_REG_R11(%rsp) + movq %r12, IRQ_REG_R12(%rsp) + movq %r13, IRQ_REG_R13(%rsp) + movq %r14, IRQ_REG_R14(%rsp) + movq %r15, IRQ_REG_R15(%rsp) + + // Save current stack into %rbp + movq %rsp, %rbp + + // Force correct stack alignment + orq $0xF, %rsp + xorq $0xF, %rsp +.endm + +.macro IRQ_RESTORE_STATE + // Restore the stack pointer + movq %rbp, %rsp + + // Restore state + movq IRQ_REG_RAX(%rsp), %rax + movq IRQ_REG_RCX(%rsp), %rcx + movq IRQ_REG_RDX(%rsp), %rdx + movq IRQ_REG_RBX(%rsp), %rbx + movq IRQ_REG_RSI(%rsp), %rsi + movq IRQ_REG_RDI(%rsp), %rdi + movq IRQ_REG_RBP(%rsp), %rbp + movq IRQ_REG_R8(%rsp), %r8 + movq IRQ_REG_R9(%rsp), %r9 + movq IRQ_REG_R10(%rsp), %r10 + movq IRQ_REG_R11(%rsp), %r11 + movq IRQ_REG_R12(%rsp), %r12 + movq IRQ_REG_R13(%rsp), %r13 + movq IRQ_REG_R14(%rsp), %r14 + movq IRQ_REG_R15(%rsp), %r15 + + addq $IRQ_STATE_SIZE, %rsp +.endm + +.macro IRQ_VECTOR, n +irq_vector_\n: + // %rsp + 0: %rip + // %rsp + 8: %cs + SWAPGS_IF_NEEDED 8 + + IRQ_SAVE_STATE + + // Force correct segment registers + mov $0x10, %ax + mov %ax, %ss + mov %ax, %ds + mov %ax, %es + + mov $\n, %rdi + mov %rbp, %rsi + call {irq_handler} + + IRQ_RESTORE_STATE + + SWAPGS_IF_NEEDED 8 + + iretq +.endm + +.macro IRQ_VECTOR_ENTRY, n +.quad irq_vector_\n +.endm + +.macro IRQ_VECTORS, start, end +.set i, 0 +.rept \end - \start + IRQ_VECTOR %i + .set i, i+1 +.endr +.endm + +.macro IRQ_VECTOR_ENTRIES, start, end +.set i, 0 +.rept \end - \start + IRQ_VECTOR_ENTRY %i + .set i, i+1 +.endr +.endm + +.section .text +local_timer_vector: + SWAPGS_IF_NEEDED 8 + IRQ_SAVE_STATE + + mov %rbp, %rdi + call {local_timer_irq_handler} + + IRQ_RESTORE_STATE + SWAPGS_IF_NEEDED 8 + iretq + +ipi_vector: + SWAPGS_IF_NEEDED 8 + IRQ_SAVE_STATE + + call {ipi_handler} + jmp . + +dummy_vector: + SWAPGS_IF_NEEDED 8 + IRQ_SAVE_STATE + + call {dummy_irq_handler} + jmp . + +IRQ_VECTORS 4, 255 + +.section .rodata +// 224 vectors: 256 - 32 (exceptions) +.p2align 4 +.type __x86_64_apic_vectors, @object +__x86_64_apic_vectors: + // Local timer IRQ: 0 + .quad local_timer_vector + // Dummy entries (currently): 1..=2 + .quad dummy_vector + .quad dummy_vector + // IPI vector: 3 + .quad ipi_vector + // Regular IRQ vectors: 4..=222 + IRQ_VECTOR_ENTRIES 4, 223 + // Spurious interrupt vector: 223 + .quad dummy_vector +.size __x86_64_apic_vectors, . - __x86_64_apic_vectors diff --git a/src/arch/x86_64/boot/ap_boot.S b/src/arch/x86_64/boot/ap_boot.S new file mode 100644 index 00000000..a61dd054 --- /dev/null +++ b/src/arch/x86_64/boot/ap_boot.S @@ -0,0 +1,110 @@ +[org 0x7000] +[bits 16] + +; Data at 0x6000 + +; Layout: +; +0x00: cr3 (only u32) +; +0x08: stack_base +; +0x10: stack_size +; +0x18: entry + +__x86_64_ap_bootstrap: + cli + + ; Reset DS + mov ax, 0 + mov ds, ax + + ; Disable NMI + in al, 0x70 + or al, 0x80 + out 0x70, al + + ; Load GDT32 and enable protected mode + lgdt [gdt32_ptr] + mov eax, cr0 + or al, 1 + mov cr0, eax + + jmp 0x08:ap_start_32 + +[bits 32] +ap_start_32: + cli + + ; Proper DS + mov ax, 0x10 + mov ds, ax + + ; Enable PSE+PAE + mov eax, cr4 + or eax, (1 << 5) | (1 << 4) + mov cr4, eax + + ; Load CR3 + mov eax, dword [0x6000 + 0x00] + mov cr3, eax + + ; Enable EFER.LME + mov ecx, 0xC0000080 + rdmsr + or eax, 1 << 8 + wrmsr + + ; Enable paging + mov eax, cr0 + or eax, 1 << 31 + mov cr0, eax + + ; Load GDT64 + lgdt [gdt64_ptr] + + jmp 0x08:ap_start_64 + +[bits 64] +ap_start_64: + mov rax, 0x10 + mov ds, rax + mov es, rax + mov ss, rax + + ; Load stack + mov rsp, qword [0x6000 + 0x08] + add rsp, qword [0x6000 + 0x10] + mov rbp, rsp + + ; Jump to kernel entry + mov rax, qword [0x6000 + 0x18] + jmp rax + +align 4 +gdt32: + ; NULL + dq 0 + ; CS32 + dq 0xCF98000000FFFF + ; DS32 + dq 0xCF92000000FFFF +gdt32_end: + +align 4 +gdt32_ptr: + ; limit + dw gdt32_end - gdt32 - 1 + dd gdt32 + +align 4 +gdt64: + ; NULL + dq 0 + ; CS64 + dq 0x00209A0000000000 + ; DS64 + dq 0x0000920000000000 +gdt64_end: + +align 4 +gdt64_ptr: + dw gdt64_end - gdt64 - 1 + dd gdt64 diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 378ecdc1..552683a5 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,7 +1,6 @@ //! x86-64 boot and entry functions -use core::arch::global_asm; +use core::{arch::global_asm, sync::atomic::Ordering}; -use git_version::git_version; use yboot_proto::{ v1::{FramebufferOption, MemoryMap}, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, @@ -9,21 +8,18 @@ use yboot_proto::{ use crate::{ arch::{ - x86_64::{apic::local::LocalApic, cpu::Cpu, cpuid, exception, syscall, FB_CONSOLE}, - Architecture, ArchitectureImpl, PLATFORM, - }, - debug, - device::{display::console::add_console_autoflush, platform::Platform}, - fs::{devfs, Initrd, INITRD_DATA}, - mem::{ - heap, - phys::{self, PageUsage}, - ConvertAddress, KERNEL_VIRT_OFFSET, + x86_64::{ + apic::local::LocalApic, cpu::Cpu, cpuid, exception, kernel_main, syscall, + CPU_INIT_FENCE, + }, + Architecture, ArchitectureImpl, }, + device::platform::Platform, + mem::KERNEL_VIRT_OFFSET, task, }; -use super::ARCHITECTURE; +use super::{smp::CPU_COUNT, ARCHITECTURE}; const BOOT_STACK_SIZE: usize = 65536; @@ -64,85 +60,38 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { }, }; -fn setup_initrd() { - let initrd_start = unsafe { YBOOT_DATA.initrd_address } as usize; - let initrd_end = initrd_start + unsafe { YBOOT_DATA.initrd_size } as usize; - - if initrd_start == 0 || initrd_end <= initrd_start { - infoln!("No initrd loaded"); - return; - } - - let start_aligned = initrd_start & !0xFFF; - let end_aligned = initrd_end & !0xFFF; - - let data = unsafe { - core::slice::from_raw_parts( - initrd_start.virtualize() as *const _, - initrd_end - initrd_start, - ) - }; - - let initrd = Initrd { - phys_page_start: start_aligned, - phys_page_len: end_aligned - start_aligned, - data, - }; - - INITRD_DATA.init(initrd); -} - unsafe extern "C" fn __x86_64_upper_entry() -> ! { ArchitectureImpl::set_interrupt_mask(true); ARCHITECTURE.init_mmu(true); core::arch::asm!("wbinvd"); - ARCHITECTURE - .yboot_framebuffer - .init(YBOOT_DATA.opt_framebuffer); - ARCHITECTURE.init_primary_debug_sink(); + kernel_main(&YBOOT_DATA) +} - debug::init(); - infoln!("Yggdrasil kernel git {} starting", git_version!()); +/// Application processor entry point +pub extern "C" fn __x86_64_ap_entry() -> ! { + let cpu_id = CPU_COUNT.load(Ordering::Acquire); + // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base cpuid::feature_gate(); - if YBOOT_DATA.memory_map.address > 0xFFFFFFFF { - errorln!("Unhandled case: memory map is above 4GiB"); - loop { - ArchitectureImpl::wait_for_interrupt(); - } + infoln!("cpu{} initializing", cpu_id); + unsafe { + ARCHITECTURE.init_mmu(false); + core::arch::asm!("wbinvd"); + + Cpu::init_local(LocalApic::new(), cpu_id as u32); + syscall::init_syscall(); + exception::init_exceptions(cpu_id); + + ARCHITECTURE.init(false).unwrap(); } - if YBOOT_DATA.rsdp_address != 0 { - infoln!("ACPI RSDP at {:#x}", YBOOT_DATA.rsdp_address); - PLATFORM.init_rsdp(YBOOT_DATA.rsdp_address as _); - } + CPU_COUNT.fetch_add(1, Ordering::Release); - // Setup physical memory allocation - setup_initrd(); - ArchitectureImpl::init_physical_memory(&YBOOT_DATA.memory_map); + CPU_INIT_FENCE.wait_one(); - // Allocate memory for the kernel heap - let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) - .expect("Could not allocate a block for heap"); - heap::init_heap(heap_base.virtualize(), 16 * 0x1000); - - // Add console to flush list - add_console_autoflush(FB_CONSOLE.get()); - - // Also initializes local APIC - Cpu::init_local(LocalApic::new(), 0); - syscall::init_syscall(); - exception::init_exceptions(0); - - devfs::init(); - - PLATFORM.init(true).unwrap(); - - task::init().expect("Failed to initialize the scheduler"); - - task::enter() + unsafe { task::enter() } } global_asm!( diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 8566db6c..17f4f1d8 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -35,7 +35,9 @@ impl Cpu { /// /// Only meant to be called once per each CPU during their init. pub unsafe fn init_local(local_apic: LocalApic, id: u32) { - let tss_address = gdt::init(id); + let tss_address = gdt::init(); + + infoln!("Initialize CPU with id {}", id); let this = Box::new(Cpu { this: null_mut(), @@ -64,7 +66,11 @@ impl Cpu { /// Returns the system ID of the CPU pub fn local_id() -> u32 { - Self::local().id() + if let Some(cpu) = Self::get_local() { + cpu.id() + } else { + 0 + } } /// Returns this CPU's local data structure or None if it hasn't been set up yet diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index b81af7a5..be078e25 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -4,7 +4,7 @@ use core::{arch::global_asm, mem::size_of_val}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ - arch::x86_64::apic::{self, __x86_64_apic_irq_handler}, + arch::x86_64::apic, task::{context::TaskFrame, process::Process}, }; @@ -288,7 +288,18 @@ fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { } fn kernel_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) -> ! { - fatalln!("{:?} in KERNEL", kind); + let cr3: usize; + let cr2: usize; + unsafe { + core::arch::asm!("movq %cr3, {0}", out(reg) cr3, options(att_syntax)); + core::arch::asm!("movq %cr2, {0}", out(reg) cr2, options(att_syntax)); + } + + fatalln!("{:?} in KERNEL, frame {:p}, cr3 = {:#x}", kind, frame, cr3); + if kind == ExceptionKind::PageFault { + fatalln!("cr2 = {:#x}", cr2); + } + fatalln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); fatalln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); @@ -315,16 +326,18 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { /// # Safety /// /// Only meant to be called once per each CPU during their init. -pub unsafe fn init_exceptions(_cpu_index: usize) { - extern "C" { - static __x86_64_exception_vectors: [usize; 32]; - } +pub unsafe fn init_exceptions(cpu_index: usize) { + if cpu_index == 0 { + extern "C" { + static __x86_64_exception_vectors: [usize; 32]; + } - for (i, &entry) in __x86_64_exception_vectors.iter().enumerate() { - IDT[i] = Entry::new(entry, 0x08, Entry::PRESENT | Entry::INT32); - } + for (i, &entry) in __x86_64_exception_vectors.iter().enumerate() { + IDT[i] = Entry::new(entry, 0x08, Entry::PRESENT | Entry::INT32); + } - apic::setup_vectors(&mut IDT[32..]); + apic::setup_vectors(&mut IDT[32..]); + } let idtr = Pointer { limit: size_of_val(&IDT) as u16 - 1, @@ -337,6 +350,5 @@ pub unsafe fn init_exceptions(_cpu_index: usize) { global_asm!( include_str!("vectors.S"), exception_handler = sym __x86_64_exception_handler, - apic_irq_handler = sym __x86_64_apic_irq_handler, options(att_syntax) ); diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 8bcb32b7..300f50e6 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,6 +1,7 @@ //! x86-64 Global Descriptor Table interface -// TODO TSS -use core::mem::{size_of, size_of_val}; +use core::mem::size_of; + +use alloc::boxed::Box; #[allow(dead_code)] #[repr(packed)] @@ -80,6 +81,30 @@ impl Entry { flags: 0, limit_lo: 0, }; + const RING0_CS64: Self = Entry::new( + 0, + 0, + Entry::FLAG_LONG, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_EXECUTE, + ); + const RING0_DS64: Self = Entry::new( + 0, + 0, + 0, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_WRITE, + ); + const RING3_DS64: Self = Entry::new( + 0, + 0, + 0, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_RING3 | Entry::ACC_WRITE, + ); + const RING3_CS64: Self = Entry::new( + 0, + 0, + Entry::FLAG_LONG, + Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_RING3 | Entry::ACC_EXECUTE, + ); const fn new(base: u32, limit: u32, flags: u8, access: u8) -> Self { Self { @@ -91,70 +116,46 @@ impl Entry { limit_lo: (limit & 0xFFFF) as u16, } } + + const fn tss_low(base: u32, limit: u32) -> Self { + Self::new( + base, + limit, + Entry::FLAG_LONG, + Entry::ACC_ACCESS | Entry::ACC_PRESENT | Entry::ACC_EXECUTE, + ) + } } -const SIZE: usize = 7; - -// TODO per-CPU -#[no_mangle] -static mut TSS: Tss = Tss::NULL; - -static mut GDT: [Entry; SIZE] = [ - // 0x00, NULL - Entry::NULL, - // 0x08, Ring0 CS64 - Entry::new( - 0, - 0, - Entry::FLAG_LONG, - Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_EXECUTE, - ), - // 0x10, Ring0 DS64 - Entry::new( - 0, - 0, - 0, - Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_WRITE, - ), - // 0x18 (0x1B), Ring3 DS64 - Entry::new( - 0, - 0, - 0, - Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_RING3 | Entry::ACC_WRITE, - ), - // 0x20 (0x23), Ring3 CS64 - Entry::new( - 0, - 0, - Entry::FLAG_LONG, - Entry::ACC_PRESENT | Entry::ACC_SYSTEM | Entry::ACC_RING3 | Entry::ACC_EXECUTE, - ), - // 0x28, TSS - Entry::NULL, - Entry::NULL, -]; +// NULL, CS64, DS64, DS64, CS64, TSS, TSSext +const GDT_SIZE: usize = 7; /// Initializes the global descriptor table. /// /// # Safety /// /// Only meant to be called by the CPUs during their early init. -pub unsafe fn init(_id: u32) -> usize { - let tss_addr = &TSS as *const _ as usize; +pub unsafe fn init() -> usize { + // Won't be deallocated, so leaks are not a concern + let tss_addr = Box::into_raw(Box::new(Tss::NULL)) as usize; + let mut gdt = Box::new([ + Entry::NULL, + Entry::RING0_CS64, + Entry::RING0_DS64, + Entry::RING3_DS64, + Entry::RING3_CS64, + Entry::tss_low(tss_addr as u32, (size_of::() - 1) as u32), + Entry::NULL, + ]); - GDT[5] = Entry::new( - tss_addr as u32, - size_of_val(&TSS) as u32 - 1, - Entry::FLAG_LONG, - Entry::ACC_ACCESS | Entry::ACC_PRESENT | Entry::ACC_EXECUTE, - ); - let tss_upper = &mut GDT[6] as *mut _ as *mut u64; - tss_upper.write_unaligned((tss_addr >> 32) as u64); + let tss_high = &mut gdt[6] as *mut _ as *mut u64; + tss_high.write_unaligned((tss_addr >> 32) as u64); + + let gdt_addr = Box::into_raw(gdt) as usize; let gdtr = Pointer { - limit: size_of_val(&GDT) as u16 - 1, - offset: &GDT as *const _ as usize, + limit: (GDT_SIZE * size_of::()) as u16 - 1, + offset: gdt_addr, }; core::arch::asm!( diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index e90291fe..5fdab04c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,21 +1,28 @@ //! x86-64 architecture and platform implementation -use core::ptr::NonNull; +use core::{ptr::NonNull, sync::atomic::Ordering}; use abi::error::Error; -use acpi::{AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; -use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap}; +use acpi::{ + platform::ProcessorInfo, AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping, +}; +use cpu::Cpu; +use git_version::git_version; +use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap, LoadProtocolV1}; use crate::{ - arch::x86_64::table::{init_fixed_tables, KERNEL_TABLES}, - debug::DebugSink, + arch::x86_64::{ + apic::local::LocalApic, + table::{init_fixed_tables, KERNEL_TABLES}, + }, + debug::{self, DebugSink}, device::{ display::{ - console::{ConsoleBuffer, ConsoleRow, DisplayConsole}, + console::{add_console_autoflush, ConsoleBuffer, ConsoleRow, DisplayConsole}, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer, }, input::KeyboardDevice, - interrupt::{ExternalInterruptController, InterruptSource}, + interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, platform::Platform, timer::TimestampSource, tty::CombinedTerminal, @@ -23,12 +30,15 @@ use crate::{ }, fs::{ devfs::{self, CharDeviceType}, - INITRD_DATA, + Initrd, INITRD_DATA, }, mem::{ - phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, + heap, + phys::{self, reserved::reserve_region, PageUsage, PhysicalMemoryRegion}, ConvertAddress, }, + sync::SpinFence, + task, util::OneTimeInit, }; @@ -36,9 +46,10 @@ use self::{ apic::{ioapic::IoApic, IrqNumber}, intrinsics::IoPort, peripherals::{hpet::Hpet, ps2::PS2Controller}, + smp::CPU_COUNT, }; -use super::Architecture; +use super::{Architecture, CpuMessage}; #[macro_use] pub mod intrinsics; @@ -52,9 +63,12 @@ pub mod exception; pub mod gdt; pub mod peripherals; pub mod registers; +pub mod smp; pub mod syscall; pub mod table; +static CPU_INIT_FENCE: SpinFence = SpinFence::new(); + /// Helper trait to provide abstract access to available memory regions pub trait AbstractAvailableRegion { /// Returns page-aligned physical start address of the region @@ -134,6 +148,8 @@ pub struct X86_64 { yboot_framebuffer: OneTimeInit, rsdp_phys_base: OneTimeInit, + acpi_processor_info: OneTimeInit, + // TODO make this fully dynamic? combined_terminal: OneTimeInit, @@ -183,6 +199,10 @@ impl Architecture for X86_64 { // If IF is zero, interrupts are disabled (masked) flags & (1 << 9) == 0 } + + fn cpu_count() -> usize { + CPU_COUNT.load(Ordering::Acquire) + } } impl Platform for X86_64 { @@ -201,6 +221,10 @@ impl Platform for X86_64 { let hpet = HpetInfo::new(&acpi_tables).unwrap(); let platform_info = acpi_tables.platform_info().unwrap(); + if let Some(processor_info) = platform_info.processor_info { + self.acpi_processor_info.init(processor_info); + } + let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { panic!("Processor does not have APIC"); }; @@ -268,6 +292,16 @@ impl Platform for X86_64 { fn timestamp_source(&self) -> &dyn TimestampSource { &self.timer } + + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + let Some(local_apic) = Cpu::get_local().map(|cpu| cpu.local_apic()) else { + panic!("Local APIC has not been initialized yet"); + }; + + local_apic.send_ipi(target); + + Ok(()) + } } impl X86_64 { @@ -338,6 +372,8 @@ pub static ARCHITECTURE: X86_64 = X86_64 { rsdp_phys_base: OneTimeInit::new(), yboot_framebuffer: OneTimeInit::new(), + acpi_processor_info: OneTimeInit::new(), + combined_terminal: OneTimeInit::new(), ioapic: IoApic::new(), @@ -352,3 +388,98 @@ static FB_CONSOLE: OneTimeInit = OneTimeInit::new(); const EARLY_BUFFER_LINES: usize = 32; static mut EARLY_CONSOLE_BUFFER: [ConsoleRow; EARLY_BUFFER_LINES] = [ConsoleRow::zeroed(); EARLY_BUFFER_LINES]; + +fn setup_initrd(initrd_start: usize, initrd_end: usize) { + if initrd_start == 0 || initrd_end <= initrd_start { + infoln!("No initrd loaded"); + return; + } + + let start_aligned = initrd_start & !0xFFF; + let end_aligned = initrd_end & !0xFFF; + + let data = unsafe { + core::slice::from_raw_parts( + initrd_start.virtualize() as *const _, + initrd_end - initrd_start, + ) + }; + + let initrd = Initrd { + phys_page_start: start_aligned, + phys_page_len: end_aligned - start_aligned, + data, + }; + + INITRD_DATA.init(initrd); +} + +fn kernel_main(boot_data: &LoadProtocolV1) -> ! { + ARCHITECTURE + .yboot_framebuffer + .init(boot_data.opt_framebuffer); + + unsafe { + ARCHITECTURE.init_primary_debug_sink(); + } + + debug::init(); + infoln!("Yggdrasil kernel git {} starting", git_version!()); + + cpuid::feature_gate(); + + if boot_data.memory_map.address > 0xFFFFFFFF { + errorln!("Unhandled case: memory map is above 4GiB"); + loop { + X86_64::wait_for_interrupt(); + } + } + + if boot_data.rsdp_address != 0 { + infoln!("ACPI RSDP at {:#x}", boot_data.rsdp_address); + ARCHITECTURE.init_rsdp(boot_data.rsdp_address as _); + } + + // Reserve initrd + let initrd_start = boot_data.initrd_address as usize; + let initrd_end = initrd_start + boot_data.initrd_size as usize; + + setup_initrd(initrd_start, initrd_end); + + // Setup physical memory allocation + unsafe { + X86_64::init_physical_memory(&boot_data.memory_map); + } + + // Allocate memory for the kernel heap + let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) + .expect("Could not allocate a block for heap"); + unsafe { + heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + } + + // Add console to flush list + add_console_autoflush(FB_CONSOLE.get()); + + // Also initializes local APIC + unsafe { + Cpu::init_local(LocalApic::new(), 0); + syscall::init_syscall(); + + // TODO per-CPU IDTs? + exception::init_exceptions(0); + + devfs::init(); + + ARCHITECTURE.init(true).unwrap(); + + if let Some(info) = ARCHITECTURE.acpi_processor_info.try_get() { + smp::start_ap_cores(info); + } + + CPU_INIT_FENCE.signal(); + + task::init().expect("Failed to initialize the scheduler"); + task::enter() + } +} diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index 7ed2ff24..ec1b8820 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -184,30 +184,7 @@ impl InterruptSource for PS2Controller { count += 1; } - Ok(true) - - // let status = self.command.read(); - - // if status & 1 == 0 { - // return Ok(false); - // } - - // let key = self.data.read(); - - // if key == 0xE0 { - // self.data.read(); - // infoln!("TODO: handle 0xE0"); - // return Ok(true); - // } - - // self.data.read(); - - // if key < 128 { - // let terminal = self.tty.lock(); - - // } - - // Ok(true) + Ok(count != 0) } } diff --git a/src/arch/x86_64/registers/mod.rs b/src/arch/x86_64/registers/mod.rs index a1d063c2..b13cabe9 100644 --- a/src/arch/x86_64/registers/mod.rs +++ b/src/arch/x86_64/registers/mod.rs @@ -76,6 +76,8 @@ mod msr_ia32_apic_base { AddressPage OFFSET(12) NUMBITS(40) [], #[doc = "If set, the APIC is enabled"] ApicEnable OFFSET(11) NUMBITS(1) [], + #[doc = "If set, x2APIC mode is enabled"] + ExtendedEnable OFFSET(10) NUMBITS(1) [], #[doc = "If set, this CPU is a bootstrap processor"] BootstrapCpuCore OFFSET(8) NUMBITS(1) [], ] diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs new file mode 100644 index 00000000..a75fe525 --- /dev/null +++ b/src/arch/x86_64/smp.rs @@ -0,0 +1,124 @@ +//! x86-64 multiprocessing implementation +use core::{ + mem::size_of, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use acpi::platform::{ProcessorInfo, ProcessorState}; + +use crate::{ + arch::{x86_64::boot::__x86_64_ap_entry, Architecture, ArchitectureImpl}, + mem::{ + phys::{self, PageUsage}, + ConvertAddress, + }, + task::Cpu, +}; + +use super::table::KERNEL_TABLES; + +/// The number of CPUs present in the system +pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); + +static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86_64_ap_boot.bin")); + +const AP_STACK_PAGES: usize = 8; +const AP_BOOTSTRAP_DATA: usize = 0x6000; +const AP_BOOTSTRAP_CODE: usize = 0x7000; + +#[allow(dead_code)] +struct ApBootstrapData { + cr3: usize, + stack_base: usize, + stack_size: usize, + entry: usize, +} + +unsafe fn load_ap_bootstrap_code() { + let src_ptr = AP_BOOTSTRAP_BIN.as_ptr(); + let dst_ptr = AP_BOOTSTRAP_CODE as *mut u8; + + let size = AP_BOOTSTRAP_BIN.len(); + + assert!(size != 0, "Empty bootstrap code"); + assert!( + AP_BOOTSTRAP_CODE + size < 0x100000, + "Invalid bootstrap code placement: is not below 1MiB" + ); + + let src_slice = core::slice::from_raw_parts(src_ptr, size); + let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); + + dst_slice.copy_from_slice(src_slice); +} + +unsafe fn load_ap_bootstrap_data(src: &ApBootstrapData) { + let src_ptr = src as *const _ as *const u8; + let dst_ptr = AP_BOOTSTRAP_DATA as *mut u8; + let size = size_of::(); + + assert!( + AP_BOOTSTRAP_DATA + size < 0x100000, + "Invalid bootstrap data placement: is not below 1MiB" + ); + + let src_slice = core::slice::from_raw_parts(src_ptr, size); + let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); + + dst_slice.copy_from_slice(src_slice); + core::arch::asm!("wbinvd"); +} + +unsafe fn start_ap_core(apic_id: u32) { + assert!(ArchitectureImpl::interrupt_mask()); + + let bsp_cpu = Cpu::local(); + let bsp_apic = bsp_cpu.local_apic(); + + let cr3 = KERNEL_TABLES.physical_address(); + let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used) + .unwrap() + .virtualize(); + let stack_size = AP_STACK_PAGES * 0x1000; + + let data = ApBootstrapData { + cr3, + stack_base, + stack_size, + entry: __x86_64_ap_entry as usize, + }; + + load_ap_bootstrap_data(&data); + + let cpu_count = CPU_COUNT.load(Ordering::Acquire); + + // Send an IPI to wake up the AP + bsp_apic.wakeup_cpu(apic_id, AP_BOOTSTRAP_CODE); + + while cpu_count == CPU_COUNT.load(Ordering::Acquire) { + core::hint::spin_loop(); + } + + infoln!("cpu{} up", cpu_count); +} + +/// Starts up application processors specified by ACPI MADT. +/// +/// # Safety +/// +/// Only meant to be called once by the BSP. +pub unsafe fn start_ap_cores(info: &ProcessorInfo) { + let aps = &info.application_processors; + + if aps.is_empty() { + return; + } + + load_ap_bootstrap_code(); + + for ap in aps { + if ap.is_ap && ap.state == ProcessorState::WaitingForSipi { + start_ap_core(ap.local_apic_id); + } + } +} diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs index ef6a59dd..0bb7f2a8 100644 --- a/src/arch/x86_64/table/fixed.rs +++ b/src/arch/x86_64/table/fixed.rs @@ -132,6 +132,8 @@ pub unsafe fn init_fixed_tables() { // Global L0 let addr = KERNEL_TABLES.l1.physical_address(); - // No lower mapping anymore + + // Keep the lower mapping for AP bootstrapping + KERNEL_TABLES.l0[0] = PageEntry::table(addr, PageAttributes::empty()); KERNEL_TABLES.l0[511] = PageEntry::table(addr, PageAttributes::empty()); } diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index 4487b279..9b47b89b 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -65,96 +65,7 @@ __x86_64_exc_\n: 1: .endm -.set IRQ_REG_RAX, 0 * 8 -.set IRQ_REG_RCX, 1 * 8 -.set IRQ_REG_RDX, 2 * 8 -.set IRQ_REG_RBX, 3 * 8 -.set IRQ_REG_RSI, 4 * 8 -.set IRQ_REG_RDI, 5 * 8 -.set IRQ_REG_RBP, 6 * 8 -.set IRQ_REG_R8, 7 * 8 -.set IRQ_REG_R9, 8 * 8 -.set IRQ_REG_R10, 9 * 8 -.set IRQ_REG_R11, 10 * 8 -.set IRQ_REG_R12, 11 * 8 -.set IRQ_REG_R13, 12 * 8 -.set IRQ_REG_R14, 13 * 8 -.set IRQ_REG_R15, 14 * 8 - -// 15 registers + stack align word if needed -.set IRQ_STATE_SIZE, 15 * 8 - -.macro apic_vector, n -__x86_64_apic_irq_\n: - // %rsp + 0: %rip - // %rsp + 8: %cs - SWAPGS_IF_NEEDED 8 - - // Save state - subq $IRQ_STATE_SIZE, %rsp - - movq %rax, IRQ_REG_RAX(%rsp) - movq %rcx, IRQ_REG_RCX(%rsp) - movq %rdx, IRQ_REG_RDX(%rsp) - movq %rbx, IRQ_REG_RBX(%rsp) - movq %rsi, IRQ_REG_RSI(%rsp) - movq %rdi, IRQ_REG_RDI(%rsp) - movq %rbp, IRQ_REG_RBP(%rsp) - movq %r8, IRQ_REG_R8(%rsp) - movq %r9, IRQ_REG_R9(%rsp) - movq %r10, IRQ_REG_R10(%rsp) - movq %r11, IRQ_REG_R11(%rsp) - movq %r12, IRQ_REG_R12(%rsp) - movq %r13, IRQ_REG_R13(%rsp) - movq %r14, IRQ_REG_R14(%rsp) - movq %r15, IRQ_REG_R15(%rsp) - - // Save current stack into %rsi (arg 2) + %rbp - movq %rsp, %rsi - movq %rsp, %rbp - - // Force correct stack alignment - orq $0xF, %rsp - xorq $0xF, %rsp - - // Force correct segment registers - mov $0x10, %ax - mov %ax, %ss - mov %ax, %ds - mov %ax, %es - - mov $\n, %rdi - call {apic_irq_handler} - - // Restore the stack pointer - movq %rbp, %rsp - - // Restore state - movq IRQ_REG_RAX(%rsp), %rax - movq IRQ_REG_RCX(%rsp), %rcx - movq IRQ_REG_RDX(%rsp), %rdx - movq IRQ_REG_RBX(%rsp), %rbx - movq IRQ_REG_RSI(%rsp), %rsi - movq IRQ_REG_RDI(%rsp), %rdi - movq IRQ_REG_RBP(%rsp), %rbp - movq IRQ_REG_R8(%rsp), %r8 - movq IRQ_REG_R9(%rsp), %r9 - movq IRQ_REG_R10(%rsp), %r10 - movq IRQ_REG_R11(%rsp), %r11 - movq IRQ_REG_R12(%rsp), %r12 - movq IRQ_REG_R13(%rsp), %r13 - movq IRQ_REG_R14(%rsp), %r14 - movq IRQ_REG_R15(%rsp), %r15 - - addq $IRQ_STATE_SIZE, %rsp - - SWAPGS_IF_NEEDED 8 - - iretq -.endm - .global __x86_64_exception_vectors -.global __x86_64_apic_vectors .section .text __x86_64_exc_common: @@ -162,6 +73,7 @@ __x86_64_exc_common: // %rsp + 8: error code // %rsp + 16: %rip // %rsp + 24: %cs + SWAPGS_IF_NEEDED 24 EXC_SAVE_STATE @@ -221,23 +133,6 @@ ISR_NERR 29 ISR_YERR 30 ISR_NERR 31 -apic_vector 0 -apic_vector 1 -apic_vector 2 -apic_vector 3 -apic_vector 4 -apic_vector 5 -apic_vector 6 -apic_vector 7 -apic_vector 8 -apic_vector 9 -apic_vector 10 -apic_vector 11 -apic_vector 12 -apic_vector 13 -apic_vector 14 -apic_vector 15 - .section .rodata .global __x86_64_exception_vectors .p2align 4 @@ -275,23 +170,4 @@ __x86_64_exception_vectors: .quad __x86_64_exc_30 .quad __x86_64_exc_31 -.p2align 4 -__x86_64_apic_vectors: - .quad __x86_64_apic_irq_0 - .quad __x86_64_apic_irq_1 - .quad __x86_64_apic_irq_2 - .quad __x86_64_apic_irq_3 - .quad __x86_64_apic_irq_4 - .quad __x86_64_apic_irq_5 - .quad __x86_64_apic_irq_6 - .quad __x86_64_apic_irq_7 - .quad __x86_64_apic_irq_8 - .quad __x86_64_apic_irq_9 - .quad __x86_64_apic_irq_10 - .quad __x86_64_apic_irq_11 - .quad __x86_64_apic_irq_12 - .quad __x86_64_apic_irq_13 - .quad __x86_64_apic_irq_14 - .quad __x86_64_apic_irq_15 - .section .text diff --git a/src/debug.rs b/src/debug.rs index 9ee927b7..d841d995 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -58,7 +58,7 @@ macro_rules! log_print_raw { macro_rules! log_print { ($level:expr, $($args:tt)+) => { - log_print_raw!($level, "{}:{}: {}", file!(), line!(), format_args!($($args)+)) + log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::task::Cpu::local_id(), file!(), line!(), format_args!($($args)+)) }; } diff --git a/src/device/interrupt.rs b/src/device/interrupt.rs index 20fe8ed8..b361e267 100644 --- a/src/device/interrupt.rs +++ b/src/device/interrupt.rs @@ -44,17 +44,6 @@ pub trait InterruptSource: Device { // /// Returns the unique ID of this local interrupt controller // fn id(&self) -> Self::Id; // -// /// Sends a message to the requested set of CPUs through an interprocessor interrupt. -// /// -// /// # Note -// /// -// /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations -// /// may impose narrower restrictions. -// /// -// /// # Safety -// /// -// /// As the call may alter the flow of execution on CPUs, this function is unsafe. -// unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; // } /// Interface for a device responsible for routing and handling IRQs from external sources diff --git a/src/device/platform.rs b/src/device/platform.rs index 0faf3e5c..21bc60c5 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -2,9 +2,12 @@ use abi::error::Error; -use crate::debug::DebugSink; +use crate::{arch::CpuMessage, debug::DebugSink}; -use super::{interrupt::ExternalInterruptController, timer::TimestampSource}; +use super::{ + interrupt::{ExternalInterruptController, IpiDeliveryTarget}, + timer::TimestampSource, +}; /// Platform interface for interacting with a general hardware set pub trait Platform { @@ -53,4 +56,16 @@ pub trait Platform { /// /// May not be initialized at the moment of calling. fn timestamp_source(&self) -> &dyn TimestampSource; + + /// Sends a message to the requested set of CPUs through an interprocessor interrupt. + /// + /// # Note + /// + /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations + /// may impose narrower restrictions. + /// + /// # Safety + /// + /// As the call may alter the flow of execution on CPUs, this function is unsafe. + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; } diff --git a/src/panic.rs b/src/panic.rs index ff673bd9..f9b71a86 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,9 +2,11 @@ use core::sync::atomic::{AtomicBool, Ordering}; use crate::{ - arch::{Architecture, ArchitectureImpl}, + arch::{Architecture, ArchitectureImpl, CpuMessage, PLATFORM}, debug::{debug_internal, LogLevel}, + device::{interrupt::IpiDeliveryTarget, platform::Platform}, sync::SpinFence, + task::{sched::CpuQueue, Cpu}, }; // Just a fence to ensure secondary panics don't trash the screen @@ -37,14 +39,14 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .is_ok() { // Let other CPUs know we're screwed - // unsafe { - // PLATFORM - // .interrupt_controller() - // .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) - // .ok(); - // } + unsafe { + PLATFORM + .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) + .ok(); + } log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); + log_print_raw!(LogLevel::Fatal, "In CPU {}", Cpu::local_id()); log_print_raw!(LogLevel::Fatal, "Kernel panic "); if let Some(location) = pi.location() { @@ -64,6 +66,21 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { debug_internal(*msg, LogLevel::Fatal); log_print_raw!(LogLevel::Fatal, "\n"); } + + for (i, queue) in CpuQueue::all().enumerate() { + log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); + let lock = unsafe { queue.grab() }; + for item in lock.iter() { + log_print_raw!( + LogLevel::Fatal, + "* {} {:?} {:?}\n", + item.id(), + item.name(), + item.state() + ); + } + } + log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); log_print_raw!(LogLevel::Fatal, "X"); diff --git a/src/proc/wait.rs b/src/proc/wait.rs index c45cc14b..b8b5579b 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -70,10 +70,10 @@ impl Wait { drop(tick_lock); - if proc.state() != ProcessState::Terminated { - unsafe { - proc.set_wait_status(WaitStatus::Done); - } + unsafe { + proc.set_wait_status(WaitStatus::Done); + } + if proc.state() == ProcessState::Suspended { proc.enqueue_somewhere(); } } @@ -174,7 +174,9 @@ pub fn tick(now: Duration) { if now > item.deadline { let t = cursor.remove_current().unwrap(); - t.process.enqueue_somewhere(); + if t.process.state() == ProcessState::Suspended { + t.process.enqueue_somewhere(); + } } else { cursor.move_next(); } diff --git a/src/sync.rs b/src/sync.rs index d32b31cd..53c5d9f6 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -111,6 +111,16 @@ impl IrqSafeSpinlock { _irq: irq_guard, } } + + /// Returns an unsafe reference to the inner value. + /// + /// # Safety + /// + /// Unsafe: explicitly ignores proper access sharing. + #[allow(clippy::mut_from_ref)] + pub unsafe fn grab(&self) -> &mut T { + unsafe { &mut *self.inner.value.get() } + } } impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 11cf35d6..439f13c3 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -347,8 +347,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::SetSignalEntry => { let entry = args[0] as usize; let sp = args[1] as usize; - let proc = Process::current(); - warnln!("SetSignalEntry({}, {:#x}, {:#x})", proc.id(), entry, sp); Process::current().set_signal_entry(entry, sp); diff --git a/src/task/mod.rs b/src/task/mod.rs index f852a62b..2d71df64 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -6,6 +6,7 @@ use abi::error::Error; use alloc::{rc::Rc, string::String, vec::Vec}; use crate::{ + arch::{Architecture, ArchitectureImpl}, kernel_main, sync::{IrqSafeSpinlock, SpinFence}, task::sched::CpuQueue, @@ -75,9 +76,7 @@ pub fn spawn_kernel_closure, F: Fn() + Send + 'static>( /// Sets up CPU queues and gives them some processes to run pub fn init() -> Result<(), Error> { - // XXX - // let cpu_count = CPU_COUNT.load(Ordering::Acquire); - let cpu_count = 1; + let cpu_count = ArchitectureImpl::cpu_count(); // Create a queue for each CPU sched::init_queues(Vec::from_iter((0..cpu_count).map(|_| CpuQueue::new()))); diff --git a/src/task/process.rs b/src/task/process.rs index 785654e9..327a149f 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -14,6 +14,7 @@ use atomic_enum::atomic_enum; use vfs::VnodeRef; use crate::{ + arch::{Architecture, ArchitectureImpl}, mem::{table::AddressSpace, ForeignPointer}, proc::{ io::ProcessIo, @@ -204,6 +205,11 @@ impl Process { self.inner.lock().group_id } + /// Returns the CPU number this task in running on (or the last one) + pub fn cpu_id(&self) -> u32 { + self.cpu_id.load(Ordering::Acquire) + } + /// Selects a suitable CPU queue and submits the process for execution. /// /// # Panics @@ -225,6 +231,7 @@ impl Process { /// /// Currently, the code will panic if the process is queued/executing on any queue. pub fn enqueue_to(self: Rc, queue: &CpuQueue) { + let _irq = IrqGuard::acquire(); let current_state = self.state.swap(ProcessState::Ready, Ordering::SeqCst); if current_state == ProcessState::Terminated { @@ -390,6 +397,7 @@ impl CurrentProcess { pub unsafe fn new(inner: Rc) -> Self { // XXX // assert_eq!(DAIF.read(DAIF::I), 1); + assert!(ArchitectureImpl::interrupt_mask()); Self(inner) } diff --git a/src/task/sched.rs b/src/task/sched.rs index b6c68b9e..e42bd36c 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -88,8 +88,14 @@ impl CpuQueueInner { return Some(task); } // Drop suspended tasks from the queue - ProcessState::Suspended | ProcessState::Terminated => (), - e => panic!("Unexpected process state in CpuQueue: {:?}", e), + ProcessState::Suspended | ProcessState::Terminated | ProcessState::Running => (), + // e => panic!( + // "Unexpected process state in CpuQueue: {:?} ({} {:?}, cpu_id={})", + // e, + // task.id(), + // task.name(), + // task.cpu_id() + // ), } } @@ -106,7 +112,8 @@ impl CpuQueueInner { impl CpuQueue { /// Constructs an empty queue with its own idle task pub fn new() -> Self { - let idle = TaskContext::kernel(__idle, 0).expect("Could not construct an idle task"); + let idle = TaskContext::kernel(__idle, Cpu::local_id() as usize) + .expect("Could not construct an idle task"); Self { inner: { @@ -209,6 +216,7 @@ impl CpuQueue { /// Only meant to be called from Process impl. The function does not set any process accounting /// information, which may lead to invalid states. pub unsafe fn enqueue(&self, p: Rc) { + assert_eq!(p.state(), ProcessState::Ready); self.inner.lock().queue.push_back(p); } @@ -240,6 +248,16 @@ impl CpuQueue { self.inner.lock() } + /// Returns an unsafe reference to the queue. + /// + /// # Safety + /// + /// Only meant to be called to dump the queue contents when panicking. + #[allow(clippy::mut_from_ref)] + pub unsafe fn grab(&self) -> &mut CpuQueueInner { + self.inner.grab() + } + /// Returns the process currently being executed. /// /// # Note From 8254a42ad73ec1ed2e2c74d9217d971ac08748bc Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 5 Aug 2023 19:10:44 +0300 Subject: [PATCH 039/211] dev: better debugging interfaces --- src/arch/x86_64/mod.rs | 69 +++++++++++--------- src/arch/x86_64/peripherals/mod.rs | 1 + src/arch/x86_64/peripherals/serial.rs | 90 +++++++++++++++++++++++++++ src/debug.rs | 76 +++++++++------------- src/device/platform.rs | 12 +--- src/proc/io.rs | 5 +- src/syscall/mod.rs | 1 - src/task/process.rs | 2 +- src/util.rs | 8 ++- 9 files changed, 169 insertions(+), 95 deletions(-) create mode 100644 src/arch/x86_64/peripherals/serial.rs diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5fdab04c..5aa4a19e 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -14,7 +14,7 @@ use crate::{ apic::local::LocalApic, table::{init_fixed_tables, KERNEL_TABLES}, }, - debug::{self, DebugSink}, + debug::{self, DebugSink, LogLevel}, device::{ display::{ console::{add_console_autoflush, ConsoleBuffer, ConsoleRow, DisplayConsole}, @@ -45,7 +45,7 @@ use crate::{ use self::{ apic::{ioapic::IoApic, IrqNumber}, intrinsics::IoPort, - peripherals::{hpet::Hpet, ps2::PS2Controller}, + peripherals::{hpet::Hpet, ps2::PS2Controller, serial::ComPort}, smp::CPU_COUNT, }; @@ -159,6 +159,7 @@ pub struct X86_64 { // Static devices ps2: PS2Controller, + com1_3: ComPort, } impl Architecture for X86_64 { @@ -248,40 +249,46 @@ impl Platform for X86_64 { Ok(()) } - unsafe fn init_primary_debug_sink(&self) { - let Some(fb) = self.yboot_framebuffer.try_get() else { - // TODO fallback to serial as primary - return; - }; + unsafe fn init_debug(&'static self) { + // Serial output + self.com1_3.init(()).ok(); + debug::add_sink(self.com1_3.port_a(), LogLevel::Debug); - LINEAR_FB.init( - LinearFramebuffer::from_physical_bits( - fb.res_address as _, - fb.res_size as _, - fb.res_stride as _, - fb.res_width, - fb.res_height, - ) - .unwrap(), - ); - FB_CONSOLE.init(FramebufferConsole::from_framebuffer( - LINEAR_FB.get(), - &bitmap_font::tamzen::FONT_6x12, - ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 255, 32), - )); + // Graphical output + if let Some(fb) = self.yboot_framebuffer.try_get() { + LINEAR_FB.init( + LinearFramebuffer::from_physical_bits( + fb.res_address as _, + fb.res_size as _, + fb.res_stride as _, + fb.res_width, + fb.res_height, + ) + .unwrap(), + ); + FB_CONSOLE.init(FramebufferConsole::from_framebuffer( + LINEAR_FB.get(), + &bitmap_font::tamzen::FONT_6x12, + ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 255, 32), + )); + + debug::add_sink(FB_CONSOLE.get(), LogLevel::Info); + } + + debug::reset(); } fn name(&self) -> &'static str { "x86-64" } - fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { - if let Some(console) = FB_CONSOLE.try_get() { - Some(console) - } else { - None - } - } + // fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { + // if let Some(console) = FB_CONSOLE.try_get() { + // Some(console) + // } else { + // None + // } + // } fn interrupt_controller( &self, @@ -380,6 +387,7 @@ pub static ARCHITECTURE: X86_64 = X86_64 { timer: Hpet::uninit(), ps2: PS2Controller::new(IrqNumber::Isa(1), IrqNumber::Isa(12), 0x64, 0x60), + com1_3: ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)), }; static LINEAR_FB: OneTimeInit = OneTimeInit::new(); @@ -420,10 +428,9 @@ fn kernel_main(boot_data: &LoadProtocolV1) -> ! { .init(boot_data.opt_framebuffer); unsafe { - ARCHITECTURE.init_primary_debug_sink(); + ARCHITECTURE.init_debug(); } - debug::init(); infoln!("Yggdrasil kernel git {} starting", git_version!()); cpuid::feature_gate(); diff --git a/src/arch/x86_64/peripherals/mod.rs b/src/arch/x86_64/peripherals/mod.rs index a6229c1f..d2183adc 100644 --- a/src/arch/x86_64/peripherals/mod.rs +++ b/src/arch/x86_64/peripherals/mod.rs @@ -2,3 +2,4 @@ pub mod hpet; pub mod ps2; +pub mod serial; diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs new file mode 100644 index 00000000..8aa50788 --- /dev/null +++ b/src/arch/x86_64/peripherals/serial.rs @@ -0,0 +1,90 @@ +use abi::error::Error; + +use crate::{ + arch::x86_64::{apic::IrqNumber, intrinsics::IoPort}, + debug::DebugSink, + device::{serial::SerialDevice, Device, InitializableDevice}, + sync::IrqSafeSpinlock, +}; + +// Single port +struct Inner { + dr: IoPort, + lsr: IoPort, +} + +pub struct Port { + inner: IrqSafeSpinlock, +} + +// Port pair +pub struct ComPort { + port_a: Port, + port_b: Port, + irq: IrqNumber, +} + +impl DebugSink for Port { + fn putc(&self, c: u8) -> Result<(), Error> { + self.send(c) + } +} + +impl SerialDevice for Port { + fn send(&self, byte: u8) -> Result<(), Error> { + let inner = self.inner.lock(); + + while inner.lsr.read() & Self::LSR_THRE == 0 { + core::hint::spin_loop(); + } + + inner.dr.write(byte); + Ok(()) + } + + fn receive(&self, blocking: bool) -> Result { + todo!() + } +} + +impl Device for Port { + fn name(&self) -> &'static str { + "com-port" + } +} + +impl Port { + const LSR_THRE: u8 = 1 << 5; + + pub const fn new(base: u16) -> Self { + Self { + inner: IrqSafeSpinlock::new(Inner { + dr: IoPort::new(base), + lsr: IoPort::new(base + 5), + }), + } + } +} + +impl InitializableDevice for ComPort { + type Options = (); + + unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { + // TODO proper init + Ok(()) + } +} + +impl ComPort { + pub const fn new(port_a: u16, port_b: u16, irq: IrqNumber) -> Self { + Self { + port_a: Port::new(port_a), + port_b: Port::new(port_b), + irq, + } + } + + pub fn port_a(&self) -> &Port { + &self.port_a + } +} diff --git a/src/debug.rs b/src/debug.rs index d841d995..1d36c72b 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -3,14 +3,9 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use crate::{arch::PLATFORM, device::platform::Platform, sync::IrqSafeSpinlock, util::OneTimeInit}; +use crate::{sync::IrqSafeSpinlock, util::StaticVector}; -// use crate::{ -// arch::PLATFORM, -// device::{platform::Platform, serial::SerialDevice}, -// sync::IrqSafeSpinlock, -// util::OneTimeInit, -// }; +const MAX_DEBUG_SINKS: usize = 4; /// Defines the severity of the message #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -46,10 +41,6 @@ pub trait DebugSink { } } -struct DebugPrinter { - sink: &'static dyn DebugSink, -} - macro_rules! log_print_raw { ($level:expr, $($args:tt)+) => { $crate::debug::debug_internal(format_args!($($args)+), $level) @@ -87,8 +78,14 @@ debug_tpl!($ warn, warnln, Warning); debug_tpl!($ error, errorln, Error); debug_tpl!($ fatal, fatalln, Fatal); -#[no_mangle] -static DEBUG_PRINTER: OneTimeInit> = OneTimeInit::new(); +#[derive(Clone, Copy)] +struct DebugSinkWrapper { + inner: &'static dyn DebugSink, + level: LogLevel, +} + +static DEBUG_SINKS: IrqSafeSpinlock> = + IrqSafeSpinlock::new(StaticVector::new()); // impl LogLevel { // fn log_prefix(self) -> &'static str { @@ -112,13 +109,6 @@ static DEBUG_PRINTER: OneTimeInit> = OneTimeInit:: // } // } -impl fmt::Write for DebugPrinter { - fn write_str(&mut self, s: &str) -> fmt::Result { - self.sink.puts(s).ok(); - Ok(()) - } -} - /// Prints a hex-dump of a slice, appending a virtual address offset to the output pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { for (i, b) in data.iter().enumerate() { @@ -140,39 +130,31 @@ pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { } } -/// Initializes the debug logging faclities. -/// -/// # Panics -/// -/// Will panic if called more than once. -pub fn init() { - DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { - sink: PLATFORM.primary_debug_sink().unwrap(), - })); - // unsafe { - // vfs::init_debug_hook(&move |args| { - // debug_internal(args, LogLevel::Debug); - // }); - // } +impl fmt::Write for DebugSinkWrapper { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.inner.puts(s).ok(); + Ok(()) + } } -// pub fn init_with_sink(sink: &'static dyn DebugSink) { -// DEBUG_PRINTER.init(IrqSafeSpinlock::new(DebugPrinter { sink })); -// } +pub fn add_sink(sink: &'static dyn DebugSink, level: LogLevel) { + DEBUG_SINKS + .lock() + .push(DebugSinkWrapper { inner: sink, level }); +} + +pub fn reset() { + log_print_raw!(LogLevel::Info, "\x1b[0m"); +} #[doc(hidden)] pub fn debug_internal(args: Arguments, level: LogLevel) { use fmt::Write; - if level <= LogLevel::Debug { - return; - } - - if DEBUG_PRINTER.is_initialized() { - let mut printer = DEBUG_PRINTER.get().lock(); - - // printer.write_str(level.log_prefix()).ok(); - printer.write_fmt(args).ok(); - // printer.write_str(level.log_suffix()).ok(); + for sink in DEBUG_SINKS.lock().iter_mut() { + if level < sink.level { + continue; + } + sink.write_fmt(args).ok(); } } diff --git a/src/device/platform.rs b/src/device/platform.rs index 21bc60c5..1f99bf7a 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -24,24 +24,16 @@ pub trait Platform { /// Unsafe to call if the platform has already been initialized. unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error>; - // TODO use init_debug instead and let the platform code handle it - /// Initializes the primary output device to provide the debugging output as early as possible. + /// Initializes the debug devices to provide early debug output. /// /// # Safety /// /// Unsafe to call if the device has already been initialized. - unsafe fn init_primary_debug_sink(&self); + unsafe fn init_debug(&'static self); /// Returns a display name for the platform fn name(&self) -> &'static str; - /// Returns a reference to the primary debug output device. - /// - /// # Note - /// - /// May not be initialized at the moment of calling. - fn primary_debug_sink(&self) -> Option<&dyn DebugSink>; - /// Returns a reference to the platform's interrupt controller. /// /// # Note diff --git a/src/proc/io.rs b/src/proc/io.rs index ecf03f6a..e91a5915 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -21,10 +21,7 @@ impl ProcessIo { /// Returns a file given descriptor refers to pub fn file(&self, fd: RawFd) -> Result { - debugln!("fd = {:?}", fd); - let res = self.files.get(&fd).cloned().unwrap(); - // .ok_or(Error::InvalidFile) - Ok(res) + self.files.get(&fd).cloned().ok_or(Error::InvalidFile) } /// Iterates over the file descriptors in [ProcessIo] and removes them if predicate is `false` diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 439f13c3..c4c53d4e 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -138,7 +138,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let proc = Process::current(); run_with_io_at(at, |at, mut io| { - debugln!("run_with_io_at {:?}", at); let file = io.ioctx().open(at, path, opts, mode)?; if proc.session_terminal().is_none() && diff --git a/src/task/process.rs b/src/task/process.rs index 327a149f..cfc85081 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -424,7 +424,7 @@ impl CurrentProcess { self.inner.lock().exit_status = status.into(); let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); assert_eq!(current_state, ProcessState::Running); - infoln!("Process {} exited with code {:?}", self.id(), status); + debugln!("Process {} exited with code {:?}", self.id(), status); match current_state { ProcessState::Suspended => { diff --git a/src/util.rs b/src/util.rs index daec101b..8ead65b7 100644 --- a/src/util.rs +++ b/src/util.rs @@ -2,7 +2,7 @@ use core::{ cell::UnsafeCell, mem::MaybeUninit, - ops::Deref, + ops::{Deref, DerefMut}, panic, sync::atomic::{AtomicBool, Ordering}, }; @@ -125,3 +125,9 @@ impl Deref for StaticVector { unsafe { MaybeUninit::slice_assume_init_ref(&self.data[..self.len]) } } } + +impl DerefMut for StaticVector { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { MaybeUninit::slice_assume_init_mut(&mut self.data[..self.len]) } + } +} From 724ed4f00a03f7f576c8df5cde83653c0322f6c9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 6 Aug 2023 17:06:34 +0300 Subject: [PATCH 040/211] dev: better console emulation --- src/debug.rs | 51 ++++--- src/device/display/console.rs | 248 ++++++++++++++++++++++++++----- src/device/display/fb_console.rs | 17 ++- src/device/tty.rs | 33 ++-- src/util.rs | 5 + 5 files changed, 280 insertions(+), 74 deletions(-) diff --git a/src/debug.rs b/src/debug.rs index 1d36c72b..3fe6f320 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -87,27 +87,27 @@ struct DebugSinkWrapper { static DEBUG_SINKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(StaticVector::new()); -// impl LogLevel { -// fn log_prefix(self) -> &'static str { -// match self { -// LogLevel::Debug => "", -// LogLevel::Info => "\x1b[36m\x1b[1m", -// LogLevel::Warning => "\x1b[33m\x1b[1m", -// LogLevel::Error => "\x1b[31m\x1b[1m", -// LogLevel::Fatal => "\x1b[38;2;255;0;0m\x1b[1m", -// } -// } -// -// fn log_suffix(self) -> &'static str { -// match self { -// LogLevel::Debug => "", -// LogLevel::Info => "\x1b[0m", -// LogLevel::Warning => "\x1b[0m", -// LogLevel::Error => "\x1b[0m", -// LogLevel::Fatal => "\x1b[0m", -// } -// } -// } +impl LogLevel { + fn log_prefix(self) -> &'static str { + match self { + LogLevel::Debug => "", + LogLevel::Info => "\x1b[36m\x1b[1m", + LogLevel::Warning => "\x1b[33m\x1b[1m", + LogLevel::Error => "\x1b[31m\x1b[1m", + LogLevel::Fatal => "\x1b[38;2;255;0;0m\x1b[1m", + } + } + + fn log_suffix(self) -> &'static str { + match self { + LogLevel::Debug => "", + LogLevel::Info => "\x1b[0m", + LogLevel::Warning => "\x1b[0m", + LogLevel::Error => "\x1b[0m", + LogLevel::Fatal => "\x1b[0m", + } + } +} /// Prints a hex-dump of a slice, appending a virtual address offset to the output pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { @@ -155,6 +155,15 @@ pub fn debug_internal(args: Arguments, level: LogLevel) { if level < sink.level { continue; } + + if sink.inner.supports_control_sequences() { + sink.write_str(level.log_prefix()).ok(); + } + sink.write_fmt(args).ok(); + + if sink.inner.supports_control_sequences() { + sink.write_str(level.log_suffix()).ok(); + } } } diff --git a/src/device/display/console.rs b/src/device/display/console.rs index c94826d3..806f374f 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -3,6 +3,7 @@ use core::mem::size_of; use abi::{error::Error, primitive_enum}; use alloc::vec::Vec; +use bitflags::bitflags; use crate::{ debug::DebugSink, @@ -12,10 +13,14 @@ use crate::{ }, sync::IrqSafeSpinlock, task::tasklet::TaskFlow, + util::StaticVector, }; const CONSOLE_ROW_LEN: usize = 128; +const DEFAULT_FG_COLOR: ColorAttribute = ColorAttribute::White; +const DEFAULT_BG_COLOR: ColorAttribute = ColorAttribute::Blue; + primitive_enum! { #[doc = "Color attribute of a console character"] pub enum ColorAttribute: u8 { @@ -26,29 +31,61 @@ primitive_enum! { #[doc = "..."] Green = 2, #[doc = "..."] - Blue = 3, + Yellow = 3, + #[doc = "..."] + Blue = 4, + #[doc = "..."] + Magenta = 5, + #[doc = "..."] + Cyan = 6, #[doc = "..."] White = 7, } } +bitflags! { + #[derive(Clone, Copy)] + pub struct Attributes: u8 { + const BOLD = 1 << 0; + } +} + impl ColorAttribute { + pub fn from_vt100(val: u8) -> Self { + match val { + 0..=7 => Self::try_from(val).unwrap(), + _ => todo!(), + } + } + /// Converts the attribute to RGBA representation - pub fn as_rgba(&self) -> u32 { - match self { - Self::Black => 0xFF000000, - Self::Red => 0xFFFF0000, - Self::Green => 0xFF00FF00, - Self::Blue => 0xFF0000FF, - Self::White => 0xFFFFFFFF, + pub fn as_rgba(&self, bold: bool) -> u32 { + let color = match self { + Self::Black => 0x000000, + Self::Red => 0x7F0000, + Self::Green => 0x007F00, + Self::Yellow => 0x7F7F00, + Self::Blue => 0x00007F, + Self::Magenta => 0x7F007F, + Self::Cyan => 0x007F7F, + Self::White => 0x7F7F7F, + }; + if bold { + color * 2 + } else { + color } } } /// Represents a single character with its attributes #[derive(Clone, Copy)] -#[repr(transparent)] -pub struct ConsoleChar(u16); +#[repr(C)] +pub struct ConsoleChar { + attributes: u16, + char: u8, + _pad: u8, +} /// Represents a single line in the console buffer #[derive(Clone, Copy)] @@ -65,6 +102,12 @@ pub struct ConsoleBuffer { height: u32, } +pub enum EscapeState { + Normal, + Escape, + Csi, +} + /// Common state for console output devices pub struct ConsoleState { /// Current cursor row @@ -73,6 +116,11 @@ pub struct ConsoleState { pub cursor_col: u32, /// Current foreground color pub fg_color: ColorAttribute, + pub bg_color: ColorAttribute, + pub attributes: Attributes, + + esc_args: StaticVector, + esc_state: EscapeState, /// Row buffer pub buffer: ConsoleBuffer, @@ -105,32 +153,43 @@ pub trait DisplayConsole { impl ConsoleChar { /// Empty character - pub const BLANK: Self = Self(0); + pub const BLANK: Self = Self { + attributes: 0, + char: 0, + _pad: 0, + }; /// Constructs a console character from a char and its attributes #[inline] - pub fn from_parts(c: u8, fg: ColorAttribute, bg: ColorAttribute) -> Self { - let fg = (u8::from(fg) as u16) << 8; - let bg = (u8::from(bg) as u16) << 12; - Self(fg | bg | (c as u16)) + pub fn from_parts( + char: u8, + fg: ColorAttribute, + bg: ColorAttribute, + attributes: Attributes, + ) -> Self { + let attributes = ((attributes.bits() as u16) << 8) + | ((u8::from(bg) as u16) << 4) + | (u8::from(fg) as u16); + Self { + attributes, + char, + _pad: 0, + } } - /// Returns the foreground color attribute #[inline] - pub fn fg_color(self) -> ColorAttribute { - ColorAttribute::try_from((self.0 >> 8) as u8).unwrap_or(ColorAttribute::White) - } + pub fn attributes(self) -> (ColorAttribute, ColorAttribute, Attributes) { + let fg = ColorAttribute::try_from((self.attributes & 0xF) as u8).unwrap(); + let bg = ColorAttribute::try_from(((self.attributes >> 4) & 0xF) as u8).unwrap(); + let attributes = Attributes::from_bits((self.attributes >> 8) as u8).unwrap(); - /// Returns the background color attribute - #[inline] - pub fn bg_color(self) -> ColorAttribute { - ColorAttribute::try_from((self.0 >> 12) as u8).unwrap_or(ColorAttribute::Black) + (fg, bg, attributes) } /// Returns the character data of this [ConsoleChar] #[inline] pub const fn character(self) -> u8 { - self.0 as u8 + self.char } } @@ -161,8 +220,12 @@ impl ConsoleRow { /// Constructs a row filled with blank characters pub const fn zeroed() -> Self { Self { - dirty: 0, - chars: [ConsoleChar::BLANK; CONSOLE_ROW_LEN], + dirty: 1, + chars: [ConsoleChar { + attributes: ((DEFAULT_BG_COLOR as u8) as u16) << 4, + char: b' ', + _pad: 0, + }; CONSOLE_ROW_LEN], } } @@ -181,9 +244,10 @@ impl ConsoleRow { } /// Clears the console row with blank characters - pub fn clear(&mut self) { + pub fn clear(&mut self, bg: ColorAttribute) { self.dirty = 1; - self.chars.fill(ConsoleChar::BLANK); + self.chars + .fill(ConsoleChar::from_parts(b' ', bg, bg, Attributes::empty())); } } @@ -239,9 +303,15 @@ impl ConsoleBuffer { } } - fn scroll_once(&mut self) { + fn clear(&mut self, bg: ColorAttribute) { + for row in self.rows.iter_mut() { + row.clear(bg); + } + } + + fn scroll_once(&mut self, bg: ColorAttribute) { self.rows.copy_within(1.., 0); - self.rows[(self.height - 1) as usize].clear(); + self.rows[(self.height - 1) as usize].clear(bg); // Mark everything dirty self.rows.iter_mut().for_each(|row| { @@ -256,38 +326,49 @@ impl ConsoleState { Self { cursor_row: 0, cursor_col: 0, - fg_color: ColorAttribute::White, + fg_color: DEFAULT_FG_COLOR, + bg_color: DEFAULT_BG_COLOR, + attributes: Attributes::empty(), + + esc_args: StaticVector::new(), + esc_state: EscapeState::Normal, buffer, } } - fn putc(&mut self, c: u8) -> bool { + fn putc_normal(&mut self, c: u8) -> bool { let mut flush = false; match c { - c if c > 127 => { + c if c >= 127 => { self.buffer.set_char( self.cursor_row, self.cursor_col, - ConsoleChar::from_parts(b'?', self.fg_color, ColorAttribute::Red), + ConsoleChar::from_parts( + b'?', + self.fg_color, + ColorAttribute::Red, + self.attributes, + ), ); self.cursor_col += 1; } + b'\x1b' => { + self.esc_state = EscapeState::Escape; + return false; + } b'\n' => { self.cursor_row += 1; self.cursor_col = 0; flush = true; } - b'\x7f' => { - return false; - } _ => { self.buffer.set_char( self.cursor_row, self.cursor_col, - ConsoleChar::from_parts(c, self.fg_color, ColorAttribute::Black), + ConsoleChar::from_parts(c, self.fg_color, self.bg_color, self.attributes), ); self.cursor_col += 1; @@ -300,13 +381,102 @@ impl ConsoleState { } if self.cursor_row == self.buffer.height { - self.buffer.scroll_once(); + self.buffer.scroll_once(self.bg_color); self.cursor_row = self.buffer.height - 1; flush = true; } flush } + + fn handle_csi(&mut self, c: u8) -> bool { + match c { + // Move back one character + b'D' => { + if self.cursor_col > 0 { + self.cursor_col = self.cursor_col - 1; + } + } + // Manipulate display attributes + b'm' => match self.esc_args[0] { + // Reset + 0 => { + self.fg_color = DEFAULT_FG_COLOR; + self.bg_color = DEFAULT_BG_COLOR; + self.attributes = Attributes::empty(); + } + // Bold + 1 => { + self.attributes |= Attributes::BOLD; + } + // Foreground colors + 31..=37 => { + let vt_color = self.esc_args[0] % 10; + self.fg_color = ColorAttribute::from_vt100(vt_color as u8); + } + _ => loop {}, + }, + // Move cursor to position + b'f' => { + let row = self.esc_args[1].clamp(1, self.buffer.height) - 1; + let col = self.esc_args[1].clamp(1, CONSOLE_ROW_LEN as u32) - 1; + + self.cursor_row = row; + self.cursor_col = col; + } + // Clear rows/columns/screen + b'J' => match self.esc_args[0] { + // Erase lines down + 0 => loop {}, + // Erase lines up + 1 => loop {}, + // Erase all + 2 => { + self.buffer.clear(self.bg_color); + } + _ => (), + }, + _ => loop {}, + } + + self.esc_state = EscapeState::Normal; + false + } + + fn handle_csi_byte(&mut self, c: u8) -> bool { + match c { + b'0'..=b'9' => { + let arg = self.esc_args.last_mut().unwrap(); + *arg *= 10; + *arg += (c - b'0') as u32; + false + } + b';' => { + self.esc_args.push(0); + false + } + _ => self.handle_csi(c), + } + } + + fn putc(&mut self, c: u8) -> bool { + match self.esc_state { + EscapeState::Normal => self.putc_normal(c), + EscapeState::Escape => match c { + b'[' => { + self.esc_state = EscapeState::Csi; + self.esc_args.clear(); + self.esc_args.push(0); + false + } + _ => { + self.esc_state = EscapeState::Normal; + false + } + }, + EscapeState::Csi => self.handle_csi_byte(c), + } + } } impl DebugSink for dyn DisplayConsole { diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index b9c23bdf..a93e8f9d 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -12,7 +12,7 @@ use embedded_graphics::{ use crate::{debug::DebugSink, sync::IrqSafeSpinlock}; use super::{ - console::{ConsoleBuffer, ConsoleState, DisplayConsole}, + console::{Attributes, ConsoleBuffer, ConsoleState, DisplayConsole}, linear_fb::LinearFramebuffer, DisplayDevice, DisplayDimensions, }; @@ -73,16 +73,23 @@ impl DisplayConsole for FramebufferConsole { break; } - inner.fill_row(row_idx * ch, ch, 0xFF000000); - for (col_idx, &chr) in row.iter().take(inner.width as _).enumerate() { let glyph = chr.character(); + let (fg, bg, attr) = chr.attributes(); if glyph == 0 { continue; } - inner.fg_color = chr.fg_color().as_rgba(); + inner.fg_color = fg.as_rgba(attr.contains(Attributes::BOLD)); + + inner.fill_rect( + (col_idx as u32) * cw, + row_idx * ch, + cw, + ch, + bg.as_rgba(false), + ); inner.draw_glyph(font, (col_idx as u32) * cw, row_idx * ch, glyph); } } @@ -96,7 +103,7 @@ impl DisplayConsole for FramebufferConsole { cursor_row * ch, cw, ch, - state.fg_color.as_rgba(), + state.fg_color.as_rgba(false), ); } } diff --git a/src/device/tty.rs b/src/device/tty.rs index 13b2963e..c667c8d2 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -155,10 +155,11 @@ pub trait TtyDevice: SerialDevice { } self.send(byte).ok(); } else if config.line.contains(TerminalLineOptions::ECHO) { - // TODO proper control if byte.is_ascii_control() { - self.send(b'^').ok(); - self.send(byte + 0x40).ok(); + if byte != config.chars.erase && byte != config.chars.werase { + self.send(b'^').ok(); + self.send(byte + 0x40).ok(); + } } else { self.send(byte).ok(); } @@ -202,11 +203,22 @@ pub trait TtyDevice: SerialDevice { let byte = ring.getc()?; config = ring.config.lock(); - if byte == config.chars.eof && config.is_canonical() { - break; - } + if config.is_canonical() { + if byte == config.chars.eof { + break; + } else if byte == config.chars.erase { + // Erase + if off != 0 { + self.raw_write(b"\x1b[D \x1b[D"); + off -= 1; + rem += 1; + } - // TODO handle special characters + continue; + } else if byte == config.chars.werase { + todo!() + } + } data[off] = byte; off += 1; @@ -230,8 +242,11 @@ pub trait TtyDevice: SerialDevice { } /// Writes raw data to the terminal bypassing the processing functions - fn raw_write(&self, _data: &[u8]) -> Result { - todo!(); + fn raw_write(&self, data: &[u8]) -> Result { + for &byte in data { + self.send(byte)?; + } + Ok(data.len()) } } diff --git a/src/util.rs b/src/util.rs index 8ead65b7..0dc483fe 100644 --- a/src/util.rs +++ b/src/util.rs @@ -116,6 +116,11 @@ impl StaticVector { pub fn is_empty(&self) -> bool { self.len == 0 } + + pub fn clear(&mut self) { + // TODO actually drop contents + self.len = 0; + } } impl Deref for StaticVector { From 5502ef10f942956d7bf18d79c383c5572aa337c5 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 6 Aug 2023 20:16:23 +0300 Subject: [PATCH 041/211] x86-64: fix incorrect Cpu::get_local --- src/arch/x86_64/boot/mod.rs | 26 ++++++++++++++++++++++++-- src/arch/x86_64/cpu.rs | 9 +++------ src/arch/x86_64/mod.rs | 1 + 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 552683a5..cbaf5c22 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,6 +1,7 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; +use tock_registers::interfaces::Writeable; use yboot_proto::{ v1::{FramebufferOption, MemoryMap}, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, @@ -9,8 +10,8 @@ use yboot_proto::{ use crate::{ arch::{ x86_64::{ - apic::local::LocalApic, cpu::Cpu, cpuid, exception, kernel_main, syscall, - CPU_INIT_FENCE, + apic::local::LocalApic, cpu::Cpu, cpuid, exception, kernel_main, + registers::MSR_IA32_KERNEL_GS_BASE, syscall, CPU_INIT_FENCE, }, Architecture, ArchitectureImpl, }, @@ -60,9 +61,18 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { }, }; +static UNINIT_CPU: usize = 0; + unsafe extern "C" fn __x86_64_upper_entry() -> ! { ArchitectureImpl::set_interrupt_mask(true); + // Point %gs to a dummy structure so that Cpu::get_local() works properly even before the CPU + // data structure is initialized + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + core::arch::asm!("swapgs"); + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + core::arch::asm!("swapgs"); + ARCHITECTURE.init_mmu(true); core::arch::asm!("wbinvd"); @@ -72,6 +82,16 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { /// Application processor entry point pub extern "C" fn __x86_64_ap_entry() -> ! { let cpu_id = CPU_COUNT.load(Ordering::Acquire); + + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + unsafe { + core::arch::asm!("swapgs"); + } + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + unsafe { + core::arch::asm!("swapgs"); + } + // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base cpuid::feature_gate(); @@ -91,6 +111,8 @@ pub extern "C" fn __x86_64_ap_entry() -> ! { CPU_INIT_FENCE.wait_one(); + infoln!("cpu{} initialized", cpu_id); + unsafe { task::enter() } } diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 17f4f1d8..9737049a 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -35,10 +35,10 @@ impl Cpu { /// /// Only meant to be called once per each CPU during their init. pub unsafe fn init_local(local_apic: LocalApic, id: u32) { - let tss_address = gdt::init(); - infoln!("Initialize CPU with id {}", id); + let tss_address = gdt::init(); + let this = Box::new(Cpu { this: null_mut(), tss_address, @@ -52,10 +52,7 @@ impl Cpu { (*this).this = this; MSR_IA32_KERNEL_GS_BASE.set(this as u64); - core::arch::asm!("swapgs"); - - // Force some incorrect value into the register for debugging - MSR_IA32_KERNEL_GS_BASE.set(0xDEADB0D1u64); + core::arch::asm!("wbinvd; swapgs"); } /// Returns this CPU's local data structure diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5aa4a19e..55c76d18 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -471,6 +471,7 @@ fn kernel_main(boot_data: &LoadProtocolV1) -> ! { // Also initializes local APIC unsafe { Cpu::init_local(LocalApic::new(), 0); + syscall::init_syscall(); // TODO per-CPU IDTs? From 7b525c408576bc244c7f38b17cd21375eee6157f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 7 Aug 2023 09:53:47 +0300 Subject: [PATCH 042/211] dev: fix console panicking in panic handler --- lib/vfs/src/node.rs | 7 ++- src/arch/x86_64/apic/ioapic.rs | 3 +- src/arch/x86_64/apic/local.rs | 2 +- src/arch/x86_64/apic/vectors.S | 16 +++---- src/arch/x86_64/boot/ap_boot.S | 3 ++ src/arch/x86_64/exception.rs | 5 ++ src/arch/x86_64/mod.rs | 17 +++---- src/arch/x86_64/peripherals/hpet.rs | 15 ++++-- src/arch/x86_64/vectors.S | 6 +-- src/device/display/console.rs | 71 +++++++++++++++-------------- src/panic.rs | 15 +++++- src/sync.rs | 23 ++++++++-- src/task/sched.rs | 5 +- 13 files changed, 118 insertions(+), 70 deletions(-) diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index 7ef5a652..53921e5f 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -120,7 +120,12 @@ impl Vnode { #[inline] pub fn data(&self) -> RefMut>> { - self.data.borrow_mut() + match self.data.try_borrow_mut() { + Ok(r) => r, + Err(e) => { + panic!("{:?} on {:?}", e, self.name()) + } + } } #[inline] diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 91db01c4..4cc5c33a 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -273,10 +273,9 @@ impl IoApic { let table = self.table.lock(); if let Some(handler) = table.vectors[gsi as usize] { - drop(table); handler.handle_irq().expect("IRQ handler failed"); } else { - panic!("No handler set for GSI #{}", gsi); + warnln!("No handler set for GSI #{}", gsi); } } diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 3f0b43fa..df5d9b6f 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -247,7 +247,7 @@ impl LocalApic { self.regs.ICR1.write(ICR1::PhysicalDestination.val(0)); self.regs.ICR0.write( ICR0::Vector.val(APIC_IPI_VECTOR + 32) - + ICR0::Destination::Normal + + ICR0::Destination::NMI + ICR0::DestinationType::AllExceptThis, ); } diff --git a/src/arch/x86_64/apic/vectors.S b/src/arch/x86_64/apic/vectors.S index 082ac5af..6b90c713 100644 --- a/src/arch/x86_64/apic/vectors.S +++ b/src/arch/x86_64/apic/vectors.S @@ -21,7 +21,7 @@ // 15 registers + stack align word if needed .set IRQ_STATE_SIZE, 15 * 8 -.macro SWAPGS_IF_NEEDED, cs_off +.macro IRQ_SWAPGS_IF_NEEDED, cs_off cmpq $0x08, \cs_off(%rsp) je 1f swapgs @@ -84,8 +84,7 @@ irq_vector_\n: // %rsp + 0: %rip // %rsp + 8: %cs - SWAPGS_IF_NEEDED 8 - + IRQ_SWAPGS_IF_NEEDED 8 IRQ_SAVE_STATE // Force correct segment registers @@ -99,8 +98,7 @@ irq_vector_\n: call {irq_handler} IRQ_RESTORE_STATE - - SWAPGS_IF_NEEDED 8 + IRQ_SWAPGS_IF_NEEDED 8 iretq .endm @@ -127,25 +125,25 @@ irq_vector_\n: .section .text local_timer_vector: - SWAPGS_IF_NEEDED 8 + IRQ_SWAPGS_IF_NEEDED 8 IRQ_SAVE_STATE mov %rbp, %rdi call {local_timer_irq_handler} IRQ_RESTORE_STATE - SWAPGS_IF_NEEDED 8 + IRQ_SWAPGS_IF_NEEDED 8 iretq ipi_vector: - SWAPGS_IF_NEEDED 8 + IRQ_SWAPGS_IF_NEEDED 8 IRQ_SAVE_STATE call {ipi_handler} jmp . dummy_vector: - SWAPGS_IF_NEEDED 8 + IRQ_SWAPGS_IF_NEEDED 8 IRQ_SAVE_STATE call {dummy_irq_handler} diff --git a/src/arch/x86_64/boot/ap_boot.S b/src/arch/x86_64/boot/ap_boot.S index a61dd054..aaa40571 100644 --- a/src/arch/x86_64/boot/ap_boot.S +++ b/src/arch/x86_64/boot/ap_boot.S @@ -74,6 +74,9 @@ ap_start_64: add rsp, qword [0x6000 + 0x10] mov rbp, rsp + ; Push a dummy value onto the stack for align + push qword 0 + ; Jump to kernel entry mov rax, qword [0x6000 + 0x18] jmp rax diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index be078e25..55b1cbb1 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -5,6 +5,7 @@ use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ arch::x86_64::apic, + panic, task::{context::TaskFrame, process::Process}, }; @@ -317,6 +318,10 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { Process::current().handle_signal(frame); } } else { + if kind == ExceptionKind::NonMaskableInterrupt { + panic::panic_secondary(); + } + kernel_exception_inner(kind, frame) } } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 55c76d18..ac4136f3 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -14,7 +14,7 @@ use crate::{ apic::local::LocalApic, table::{init_fixed_tables, KERNEL_TABLES}, }, - debug::{self, DebugSink, LogLevel}, + debug::{self, LogLevel}, device::{ display::{ console::{add_console_autoflush, ConsoleBuffer, ConsoleRow, DisplayConsole}, @@ -269,7 +269,7 @@ impl Platform for X86_64 { FB_CONSOLE.init(FramebufferConsole::from_framebuffer( LINEAR_FB.get(), &bitmap_font::tamzen::FONT_6x12, - ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 255, 32), + ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 32), )); debug::add_sink(FB_CONSOLE.get(), LogLevel::Info); @@ -282,14 +282,6 @@ impl Platform for X86_64 { "x86-64" } - // fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { - // if let Some(console) = FB_CONSOLE.try_get() { - // Some(console) - // } else { - // None - // } - // } - fn interrupt_controller( &self, ) -> &dyn ExternalInterruptController { @@ -301,6 +293,11 @@ impl Platform for X86_64 { } unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + if !CPU_INIT_FENCE.try_wait_all(1) { + // Don't send an IPI: SMP not initialized yet + return Ok(()); + } + let Some(local_apic) = Cpu::get_local().map(|cpu| cpu.local_apic()) else { panic!("Local APIC has not been initialized yet"); }; diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 7bc5edb1..30766e6f 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -179,10 +179,17 @@ impl InterruptSource for Hpet { let tim0 = &inner.regs.Timers[0]; - assert!(tim0.ConfigurationCapability.matches_all( - TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit - + TimerConfigurationCapablities::TM_PER_INT_CAP::SupportsPeriodic - )); + assert!(tim0 + .ConfigurationCapability + .matches_all(TimerConfigurationCapablities::TM_PER_INT_CAP::SupportsPeriodic)); + + if period > 0x100000000 + && !tim0 + .ConfigurationCapability + .matches_all(TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit) + { + panic!("Period is too large and timer doesn't support 64-bit"); + } let mut tim0_irq = None; let tim0_irqs = tim0 diff --git a/src/arch/x86_64/vectors.S b/src/arch/x86_64/vectors.S index 9b47b89b..8afd3942 100644 --- a/src/arch/x86_64/vectors.S +++ b/src/arch/x86_64/vectors.S @@ -58,7 +58,7 @@ __x86_64_exc_\n: jmp __x86_64_exc_common .endm -.macro SWAPGS_IF_NEEDED, cs_off +.macro EXC_SWAPGS_IF_NEEDED, cs_off cmpq $0x08, \cs_off(%rsp) je 1f swapgs @@ -74,7 +74,7 @@ __x86_64_exc_common: // %rsp + 16: %rip // %rsp + 24: %cs - SWAPGS_IF_NEEDED 24 + EXC_SWAPGS_IF_NEEDED 24 EXC_SAVE_STATE @@ -96,7 +96,7 @@ __x86_64_exc_common: // Remove error code and number from the stack addq $16, %rsp - SWAPGS_IF_NEEDED 8 + EXC_SWAPGS_IF_NEEDED 8 iretq diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 806f374f..1554c4ba 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -17,6 +17,7 @@ use crate::{ }; const CONSOLE_ROW_LEN: usize = 128; +const MAX_CSI_ARGS: usize = 8; const DEFAULT_FG_COLOR: ColorAttribute = ColorAttribute::White; const DEFAULT_BG_COLOR: ColorAttribute = ColorAttribute::Blue; @@ -54,7 +55,7 @@ impl ColorAttribute { pub fn from_vt100(val: u8) -> Self { match val { 0..=7 => Self::try_from(val).unwrap(), - _ => todo!(), + _ => ColorAttribute::Red, } } @@ -98,7 +99,6 @@ pub struct ConsoleRow { /// need to be flushed to the display pub struct ConsoleBuffer { rows: &'static mut [ConsoleRow], - width: u32, height: u32, } @@ -119,7 +119,7 @@ pub struct ConsoleState { pub bg_color: ColorAttribute, pub attributes: Attributes, - esc_args: StaticVector, + esc_args: StaticVector, esc_state: EscapeState, /// Row buffer @@ -179,9 +179,12 @@ impl ConsoleChar { #[inline] pub fn attributes(self) -> (ColorAttribute, ColorAttribute, Attributes) { - let fg = ColorAttribute::try_from((self.attributes & 0xF) as u8).unwrap(); - let bg = ColorAttribute::try_from(((self.attributes >> 4) & 0xF) as u8).unwrap(); - let attributes = Attributes::from_bits((self.attributes >> 8) as u8).unwrap(); + let fg = + ColorAttribute::try_from((self.attributes & 0xF) as u8).unwrap_or(DEFAULT_FG_COLOR); + let bg = ColorAttribute::try_from(((self.attributes >> 4) & 0xF) as u8) + .unwrap_or(DEFAULT_BG_COLOR); + let attributes = + Attributes::from_bits((self.attributes >> 8) as u8).unwrap_or(Attributes::empty()); (fg, bg, attributes) } @@ -253,12 +256,8 @@ impl ConsoleRow { impl ConsoleBuffer { /// Constructs a fixed-size console buffer - pub const fn fixed(rows: &'static mut [ConsoleRow], width: u32, height: u32) -> Self { - Self { - rows, - width, - height, - } + pub const fn fixed(rows: &'static mut [ConsoleRow], height: u32) -> Self { + Self { rows, height } } /// Reallocates the internal buffer with a new size @@ -290,6 +289,7 @@ impl ConsoleBuffer { Ok(()) } + #[inline(never)] fn set_char(&mut self, row: u32, col: u32, c: ConsoleChar) { self.rows[row as usize].dirty = 1; self.rows[row as usize].chars[col as usize] = c; @@ -375,7 +375,7 @@ impl ConsoleState { } } - if self.cursor_col == self.buffer.width { + if self.cursor_col == CONSOLE_ROW_LEN as u32 { self.cursor_col = 0; self.cursor_row += 1; } @@ -398,24 +398,28 @@ impl ConsoleState { } } // Manipulate display attributes - b'm' => match self.esc_args[0] { - // Reset - 0 => { - self.fg_color = DEFAULT_FG_COLOR; - self.bg_color = DEFAULT_BG_COLOR; - self.attributes = Attributes::empty(); + b'm' => { + if let Some(arg) = self.esc_args.get(0) { + match arg { + // Reset + 0 => { + self.fg_color = DEFAULT_FG_COLOR; + self.bg_color = DEFAULT_BG_COLOR; + self.attributes = Attributes::empty(); + } + // Bold + 1 => { + self.attributes |= Attributes::BOLD; + } + // Foreground colors + 31..=37 => { + let vt_color = self.esc_args[0] % 10; + self.fg_color = ColorAttribute::from_vt100(vt_color as u8); + } + _ => (), + } } - // Bold - 1 => { - self.attributes |= Attributes::BOLD; - } - // Foreground colors - 31..=37 => { - let vt_color = self.esc_args[0] % 10; - self.fg_color = ColorAttribute::from_vt100(vt_color as u8); - } - _ => loop {}, - }, + } // Move cursor to position b'f' => { let row = self.esc_args[1].clamp(1, self.buffer.height) - 1; @@ -427,16 +431,16 @@ impl ConsoleState { // Clear rows/columns/screen b'J' => match self.esc_args[0] { // Erase lines down - 0 => loop {}, + 0 => (), // Erase lines up - 1 => loop {}, + 1 => (), // Erase all 2 => { self.buffer.clear(self.bg_color); } _ => (), }, - _ => loop {}, + _ => (), } self.esc_state = EscapeState::Normal; @@ -471,6 +475,7 @@ impl ConsoleState { } _ => { self.esc_state = EscapeState::Normal; + self.esc_args.clear(); false } }, diff --git a/src/panic.rs b/src/panic.rs index f9b71a86..807a62fd 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -5,19 +5,22 @@ use crate::{ arch::{Architecture, ArchitectureImpl, CpuMessage, PLATFORM}, debug::{debug_internal, LogLevel}, device::{interrupt::IpiDeliveryTarget, platform::Platform}, - sync::SpinFence, + sync::{hack_locks, SpinFence}, task::{sched::CpuQueue, Cpu}, }; // Just a fence to ensure secondary panics don't trash the screen static PANIC_FINISHED_FENCE: SpinFence = SpinFence::new(); +static PANIC_HANDLED_FENCE: SpinFence = SpinFence::new(); + /// Panic handler for CPUs other than the one that initiated it pub fn panic_secondary() -> ! { unsafe { ArchitectureImpl::set_interrupt_mask(true); } + PANIC_HANDLED_FENCE.signal(); PANIC_FINISHED_FENCE.wait_one(); log_print_raw!(LogLevel::Fatal, "X"); @@ -32,6 +35,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { unsafe { ArchitectureImpl::set_interrupt_mask(true); } + static PANIC_HAPPENED: AtomicBool = AtomicBool::new(false); if PANIC_HAPPENED @@ -45,8 +49,15 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .ok(); } + let ap_count = ArchitectureImpl::cpu_count() - 1; + PANIC_HANDLED_FENCE.wait_all(ap_count); + + unsafe { + hack_locks(); + } + log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); - log_print_raw!(LogLevel::Fatal, "In CPU {}", Cpu::local_id()); + log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); log_print_raw!(LogLevel::Fatal, "Kernel panic "); if let Some(location) = pi.location() { diff --git a/src/sync.rs b/src/sync.rs index 53c5d9f6..c29d37dd 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -7,6 +7,12 @@ use core::{ use crate::arch::{Architecture, ArchitectureImpl}; +static LOCK_HACK: AtomicBool = AtomicBool::new(false); + +pub unsafe fn hack_locks() { + LOCK_HACK.store(true, Ordering::Release); +} + /// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is /// met. pub struct SpinFence { @@ -51,6 +57,9 @@ impl SpinlockInner { fn lock(&self) -> SpinlockInnerGuard { // Loop until the lock can be acquired + if LOCK_HACK.load(Ordering::Acquire) { + return SpinlockInnerGuard { lock: self }; + } while self .state .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) @@ -79,10 +88,12 @@ impl<'a, T> DerefMut for SpinlockInnerGuard<'a, T> { impl<'a, T> Drop for SpinlockInnerGuard<'a, T> { fn drop(&mut self) { - self.lock - .state - .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) - .unwrap(); + if !LOCK_HACK.load(Ordering::Acquire) { + self.lock + .state + .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) + .unwrap(); + } } } @@ -187,4 +198,8 @@ impl SpinFence { pub fn wait_one(&self) { self.wait_all(1); } + + pub fn try_wait_all(&self, count: usize) -> bool { + self.value.load(Ordering::Acquire) >= count + } } diff --git a/src/task/sched.rs b/src/task/sched.rs index e42bd36c..7bde8732 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -88,7 +88,10 @@ impl CpuQueueInner { return Some(task); } // Drop suspended tasks from the queue - ProcessState::Suspended | ProcessState::Terminated | ProcessState::Running => (), + ProcessState::Suspended | ProcessState::Terminated => (), + ProcessState::Running => { + // TODO fix this finally + } // e => panic!( // "Unexpected process state in CpuQueue: {:?} ({} {:?}, cpu_id={})", // e, From 44a888c39aa806441581c83012c30f885e9962c9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 7 Aug 2023 10:28:04 +0300 Subject: [PATCH 043/211] aarch64: make kernel build --- src/arch/aarch64/mod.rs | 9 ++++++--- src/arch/aarch64/plat_qemu/mod.rs | 17 ++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 0ddc263e..180dbdcf 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -108,6 +108,10 @@ impl Architecture for AArch64 { fn interrupt_mask() -> bool { DAIF.read(DAIF::I) != 0 } + + fn cpu_count() -> usize { + CPU_COUNT.load(Ordering::Acquire) + } } impl AArch64 { @@ -203,10 +207,9 @@ pub fn kernel_main(dtb_phys: usize) -> ! { AArch64::set_interrupt_mask(true); ARCHITECTURE.init_device_tree(dtb_phys); - PLATFORM.init_primary_debug_sink(); + PLATFORM.init_debug(); } - // Setup debugging functions - debug::init(); + debug::reset(); exception::init_exceptions(); diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs index e7e4c9b4..ea019ec1 100644 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -4,9 +4,10 @@ use abi::error::Error; use tock_registers::interfaces::Writeable; use crate::{ - debug::DebugSink, + arch::CpuMessage, + debug::{self, DebugSink, LogLevel}, device::{ - interrupt::{ExternalInterruptController, InterruptSource}, + interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, platform::Platform, serial::pl011::Pl011, timer::TimestampSource, @@ -54,12 +55,9 @@ impl Platform for QemuPlatform { Ok(()) } - unsafe fn init_primary_debug_sink(&self) { + unsafe fn init_debug(&'static self) { self.pl011.init(()).ok(); - } - - fn primary_debug_sink(&self) -> Option<&dyn DebugSink> { - Some(&self.pl011) + debug::add_sink(&self.pl011, LogLevel::Debug); } fn interrupt_controller( @@ -75,6 +73,11 @@ impl Platform for QemuPlatform { fn timestamp_source(&self) -> &dyn TimestampSource { &self.local_timer } + + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + infoln!("TODO send ipi"); + loop {} + } } /// AArch64 "virt" platform From afc3af54fa86d7f63806def8e7ee5aa23f97be5e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 8 Aug 2023 20:00:24 +0300 Subject: [PATCH 044/211] aarch64: initial support for Orange Pi 3 --- Cargo.toml | 14 +- src/arch/aarch64/boot/entry.S | 77 +++++++++++ src/arch/aarch64/boot/mod.rs | 95 +++++-------- src/arch/aarch64/devtree.rs | 22 ++- src/arch/aarch64/mod.rs | 33 ++++- src/arch/aarch64/plat_orange_pi3/mod.rs | 99 +++++++++++++ src/arch/aarch64/plat_orange_pi3/r_wdog.rs | 78 +++++++++++ src/arch/aarch64/plat_orange_pi3/serial.rs | 153 +++++++++++++++++++++ src/arch/aarch64/plat_qemu/mod.rs | 12 +- src/arch/aarch64/timer.rs | 4 +- src/arch/mod.rs | 7 +- src/device/mod.rs | 1 + src/device/platform.rs | 2 + src/device/serial/mod.rs | 2 +- src/device/serial/pl011.rs | 2 +- src/device/tty.rs | 144 ++++++++++--------- src/main.rs | 15 +- src/mem/mod.rs | 71 +++++++++- src/panic.rs | 22 +-- src/util.rs | 2 +- 20 files changed, 687 insertions(+), 168 deletions(-) create mode 100644 src/arch/aarch64/boot/entry.S create mode 100644 src/arch/aarch64/plat_orange_pi3/mod.rs create mode 100644 src/arch/aarch64/plat_orange_pi3/r_wdog.rs create mode 100644 src/arch/aarch64/plat_orange_pi3/serial.rs diff --git a/Cargo.toml b/Cargo.toml index 62264045..bcb5f89a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,10 +18,13 @@ spinning_top = "0.2.5" # static_assertions = "1.1.0" tock-registers = "0.8.1" cfg-if = "1.0.0" -bitmap-font = { version = "0.3.0" } -embedded-graphics = "0.8.0" git-version = "0.3.5" +aarch64-cpu = "9.3.1" + +bitmap-font = { version = "0.3.0", optional = true } +embedded-graphics = { version = "0.8.0", optional = true } + [dependencies.elf] version = "0.7.2" git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" @@ -30,9 +33,14 @@ features = ["no_std_stream"] [target.'cfg(target_arch = "aarch64")'.dependencies] fdt-rs = { version = "0.4.3", default-features = false } -aarch64-cpu = "9.3.1" [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } aml = "0.16.4" acpi = "4.1.1" + +[features] +default = [] +fb_console = ["bitmap-font", "embedded-graphics"] +aarch64_qemu = [] +aarch64_orange_pi3 = [] diff --git a/src/arch/aarch64/boot/entry.S b/src/arch/aarch64/boot/entry.S new file mode 100644 index 00000000..8bf12667 --- /dev/null +++ b/src/arch/aarch64/boot/entry.S @@ -0,0 +1,77 @@ +.set CNTHCTL_EL2_EL1PCTEN, 1 << 0 +.set CNTHCTL_EL2_EL1PCEN, 1 << 1 + +.set HCR_EL2_RW_EL1IsAArch64, 1 << 31 + +.set SCTLR_EL2_RES1, 0x30C50830 + +.set SPSR_EL2_EL1h, 0x5 +.set SPSR_EL2_MASK_DAIF, 0xF << 6 + +.macro MOV_L reg, value + mov \reg, #((\value) & 0xFFFF) + movk \reg, #((\value) >> 16), lsl #16 +.endm + +.global __aarch64_entry +.section .text.entry +__aarch64_entry: + // x0 -- dtb_phys + + // Multiple processor cores may or may not be running at this point + mrs x1, mpidr_el1 + ands x1, x1, #0xF + bne 1f + + mrs x8, CurrentEL + lsr x8, x8, #2 + cmp x8, #2 + bne .el1 + + // Leave EL2 +.el2: + mrs x8, CNTHCTL_EL2 + orr x8, x8, #(CNTHCTL_EL2_EL1PCTEN | CNTHCTL_EL2_EL1PCEN) + msr CNTHCTL_EL2, x8 + msr CNTVOFF_EL2, xzr + + MOV_L x8, SCTLR_EL2_RES1 + msr SCTLR_EL2, x8 + + mov x8, #HCR_EL2_RW_EL1IsAArch64 + msr HCR_EL2, x8 + + mov x8, #SPSR_EL2_EL1h + orr x8, x8, #SPSR_EL2_MASK_DAIF + msr SPSR_EL2, x8 + + adr x8, .el1 + msr ELR_EL2, x8 + + isb + eret +.el1: + dsb sy + isb + + // Zero .bss + adr x8, __bss_start_phys + adr x9, __bss_end_phys +1: + cmp x8, x9 + beq 2f + strb wzr, [x8] + add x8, x8, #1 + b 1b +2: + + // BSP in SMP or uniprocessor + ldr x1, ={stack_bottom} + {stack_size} - {kernel_virt_offset} + mov sp, x1 + + bl {kernel_lower_entry} - {kernel_virt_offset} + + // AP in a SMP system + // TODO spin loop for this method of init +1: + b . diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 87130ca7..df2b33ea 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -1,8 +1,17 @@ //! Main entry point for the AArch64 platforms -use core::{arch::asm, sync::atomic::Ordering}; +use core::{ + arch::{asm, global_asm}, + sync::atomic::Ordering, +}; -use aarch64_cpu::registers::{CurrentEL, CPACR_EL1}; -use tock_registers::interfaces::{ReadWriteable, Readable}; +use aarch64_cpu::{ + asm::barrier, + registers::{ + CurrentEL, CNTHCTL_EL2, CNTVOFF_EL2, CPACR_EL1, ELR_EL2, HCR_EL2, SCTLR_EL2, SPSR_EL2, + SP_EL1, + }, +}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use super::{ cpu::Cpu, exception, kernel_main, smp::CPU_COUNT, AArch64, KernelStack, ARCHITECTURE, @@ -19,13 +28,20 @@ use crate::{ pub(super) static CPU_INIT_FENCE: SpinFence = SpinFence::new(); -fn __aarch64_common_lower_entry() { +extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { // Unmask FP operations CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); - if CurrentEL.read(CurrentEL::EL) != 1 { - panic!("Only EL1 is supported for now"); + unsafe { + ARCHITECTURE.init_mmu(true); } + + let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE).virtualize() }; + let elr = absolute_address!(__aarch64_bsp_upper_entry); + + barrier::dsb(barrier::SY); + barrier::isb(barrier::SY); + enter_higher_half(sp as usize, elr, dtb_phys); } fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { @@ -39,27 +55,16 @@ fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { } pub(super) extern "C" fn __aarch64_ap_lower_entry(sp: usize) -> ! { - __aarch64_common_lower_entry(); + loop {} + // __aarch64_common_lower_entry(); - unsafe { - ARCHITECTURE.init_mmu(false); - } + // unsafe { + // ARCHITECTURE.init_mmu(false); + // } - let sp = unsafe { sp.virtualize() }; - let elr = absolute_address!(__aarch64_ap_upper_entry); - enter_higher_half(sp, elr, 0); -} - -extern "C" fn __aarch64_bsp_lower_entry(dtb_phys: usize) -> ! { - __aarch64_common_lower_entry(); - - unsafe { - ARCHITECTURE.init_mmu(true); - } - - let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE).virtualize() }; - let elr = absolute_address!(__aarch64_bsp_upper_entry); - enter_higher_half(sp as usize, elr, dtb_phys); + // let sp = unsafe { sp.virtualize() }; + // let elr = absolute_address!(__aarch64_ap_upper_entry); + // enter_higher_half(sp, elr, 0); } extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { @@ -91,38 +96,14 @@ extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { } } -#[link_section = ".text.entry"] -#[no_mangle] -#[naked] -unsafe extern "C" fn __aarch64_entry() -> ! { - // Setup the stack and pass on to a proper function - asm!( - r#" - // Multiple processor cores may or may not be running at this point - mrs x1, mpidr_el1 - ands x1, x1, #0xF - bne 1f - - // BSP in SMP or uniprocessor - ldr x1, ={stack_bottom} + {stack_size} - {kernel_virt_offset} - mov sp, x1 - - bl {kernel_lower_entry} - {kernel_virt_offset} - - // AP in a SMP system - // TODO spin loop for this method of init -1: - b . -"#, - kernel_lower_entry = sym __aarch64_bsp_lower_entry, - stack_bottom = sym BSP_STACK, - stack_size = const BOOT_STACK_SIZE, - kernel_virt_offset = const KERNEL_VIRT_OFFSET, - options(noreturn) - ); -} - -#[link_section = ".bss"] static BSP_STACK: KernelStack = KernelStack { data: [0; BOOT_STACK_SIZE], }; + +global_asm!( + include_str!("entry.S"), + kernel_lower_entry = sym el1_bsp_lower_entry, + stack_bottom = sym BSP_STACK, + stack_size = const BOOT_STACK_SIZE, + kernel_virt_offset = const KERNEL_VIRT_OFFSET +); diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index 3f39a8d0..1058ebde 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -7,14 +7,16 @@ use fdt_rs::{ use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; +const INDEX_BUFFER_SIZE: usize = 65536; + #[repr(C, align(0x10))] -struct FdtIndexBuffer([u8; 32768]); +struct FdtIndexBuffer([u8; INDEX_BUFFER_SIZE]); static mut FDT_INDEX_BUFFER: FdtIndexBuffer = FdtIndexBuffer::zeroed(); impl FdtIndexBuffer { const fn zeroed() -> Self { - Self([0; 32768]) + Self([0; INDEX_BUFFER_SIZE]) } } @@ -80,16 +82,24 @@ impl Iterator for FdtMemoryRegionIter<'_> { break None; }; - if item.name().unwrap_or("").starts_with("memory@") { + let name = item.name().unwrap_or(""); + + if name.starts_with("memory@") || name == "memory" { let reg = item .props() .find(|p| p.name().unwrap_or("") == "reg") .unwrap(); + #[cfg(not(feature = "aarch64_orange_pi3"))] break Some(PhysicalMemoryRegion { base: reg.u64(0).unwrap() as usize, size: reg.u64(1).unwrap() as usize, }); + #[cfg(feature = "aarch64_orange_pi3")] + break Some(PhysicalMemoryRegion { + base: reg.u32(0).unwrap() as usize, + size: reg.u32(1).unwrap() as usize, + }); } } } @@ -141,11 +151,13 @@ fn dump_node(node: &TNode, depth: usize, level: LogLevel) { log_print_raw!(level, "{:?} {{\n", node_name); for prop in node.props() { indent(level, depth + 1); - let name = prop.name().unwrap(); + let name = prop.name().unwrap_or(""); log_print_raw!(level, "{name:?} = "); match name { - "compatible" | "stdout-path" => log_print_raw!(level, "{:?}", prop.str().unwrap()), + "compatible" | "stdout-path" => { + log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) + } _ => log_print_raw!(level, "{:x?}", prop.raw()), } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 180dbdcf..3a3cef2c 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -4,8 +4,8 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1}; use abi::error::Error; +use cfg_if::cfg_if; use fdt_rs::prelude::PropReader; -use plat_qemu::PLATFORM; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ @@ -30,7 +30,19 @@ use self::{ table::{init_fixed_tables, KERNEL_TABLES}, }; -pub mod plat_qemu; +pub mod plat_orange_pi3; +cfg_if! { + if #[cfg(feature = "aarch64_qemu")] { + pub mod plat_qemu; + + pub use plat_qemu::{PlatformImpl, PLATFORM}; + } else if #[cfg(feature = "aarch64_orange_pi3")] { + + pub use plat_orange_pi3::{PlatformImpl, PLATFORM}; + } else { + compile_error!("No platform selected for AArch64"); + } +} pub mod boot; pub mod context; @@ -156,6 +168,7 @@ impl AArch64 { ); let regions = FdtMemoryRegionIter::new(dt); + phys::init_from_iter(regions) } } @@ -173,7 +186,14 @@ fn setup_initrd() { return; }; + #[cfg(feature = "aarch64_orange_pi3")] + let initrd_start = initrd_start.u32(0).unwrap() as usize; + #[cfg(feature = "aarch64_orange_pi3")] + let initrd_end = initrd_end.u32(0).unwrap() as usize; + + #[cfg(not(feature = "aarch64_orange_pi3"))] let initrd_start = initrd_start.u64(0).unwrap() as usize; + #[cfg(not(feature = "aarch64_orange_pi3"))] let initrd_end = initrd_end.u64(0).unwrap() as usize; let start_aligned = initrd_start & !0xFFF; @@ -206,8 +226,8 @@ pub fn kernel_main(dtb_phys: usize) -> ! { unsafe { AArch64::set_interrupt_mask(true); - ARCHITECTURE.init_device_tree(dtb_phys); PLATFORM.init_debug(); + ARCHITECTURE.init_device_tree(dtb_phys); } debug::reset(); @@ -217,6 +237,7 @@ pub fn kernel_main(dtb_phys: usize) -> ! { setup_initrd(); debugln!("Initializing {} platform", PLATFORM.name()); + unsafe { ARCHITECTURE .init_physical_memory(dtb_phys) @@ -240,10 +261,10 @@ pub fn kernel_main(dtb_phys: usize) -> ! { // ); // } - Cpu::init_ipi_queues(); + // Cpu::init_ipi_queues(); - CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); + // CPU_INIT_FENCE.signal(); + // CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); task::init().expect("Failed to initialize the scheduler"); diff --git a/src/arch/aarch64/plat_orange_pi3/mod.rs b/src/arch/aarch64/plat_orange_pi3/mod.rs new file mode 100644 index 00000000..7d2eb9df --- /dev/null +++ b/src/arch/aarch64/plat_orange_pi3/mod.rs @@ -0,0 +1,99 @@ +/// +/// # Booting using u-boot +/// +/// > fatload mmc 0:1 0x40000000 uRamdisk +/// > fatload mmc 0:1 0x4d000000 sun50i-h6-orangepi-3.dtb +/// > loady 0x44000000 +/// ... +/// > bootm 0x44000000 0x40000000 0x4d000000 +/// +use abi::error::Error; + +use crate::{ + arch::CpuMessage, + debug::{self, LogLevel}, + device::{ + interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, + platform::Platform, + timer::TimestampSource, + InitializableDevice, + }, + fs::devfs::{self, CharDeviceType}, +}; + +use self::{r_wdog::RWatchdog, serial::Serial}; + +use super::{ + gic::{Gic, IrqNumber}, + timer::ArmTimer, +}; + +pub mod r_wdog; +pub mod serial; + +pub struct PlatformImpl { + uart0: Serial, + r_wdog: RWatchdog, + pub gic: Gic, + timer: ArmTimer, +} + +impl Platform for PlatformImpl { + type IrqNumber = IrqNumber; + + const KERNEL_PHYS_BASE: usize = 0x50000000; + + unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { + if is_bsp { + self.gic.init(())?; + + self.timer.init(())?; + self.r_wdog.init(())?; + + self.timer.init_irq()?; + self.uart0.init_irq()?; + + devfs::add_char_device(&self.uart0, CharDeviceType::TtySerial)?; + } else { + todo!(); + } + + Ok(()) + } + + unsafe fn init_debug(&'static self) { + self.uart0.init(()).unwrap(); + debug::add_sink(&self.uart0, LogLevel::Debug); + } + + fn interrupt_controller( + &self, + ) -> &dyn ExternalInterruptController { + &self.gic + } + + fn timestamp_source(&self) -> &dyn TimestampSource { + &self.timer + } + + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + todo!(); + } + + fn name(&self) -> &'static str { + "Orange Pi 3" + } + + unsafe fn reset(&self) -> ! { + self.r_wdog.reset_board(); + } +} + +pub static PLATFORM: PlatformImpl = unsafe { + PlatformImpl { + uart0: Serial::new(0x05000000, IrqNumber::new(32)), + r_wdog: RWatchdog::new(0x07020400), + gic: Gic::new(0x03021000, 0x03022000), + timer: ArmTimer::new(IrqNumber::new(30)), + } +}; diff --git a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs b/src/arch/aarch64/plat_orange_pi3/r_wdog.rs new file mode 100644 index 00000000..4f86a935 --- /dev/null +++ b/src/arch/aarch64/plat_orange_pi3/r_wdog.rs @@ -0,0 +1,78 @@ +use abi::error::Error; +use tock_registers::{ + interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, +}; + +use crate::{ + device::InitializableDevice, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +register_bitfields! { + u32, + CTRL [ + KEY OFFSET(1) NUMBITS(12) [ + Value = 0xA57 + ], + RESTART OFFSET(0) NUMBITS(1) [] + ], + CFG [ + CONFIG OFFSET(0) NUMBITS(2) [ + System = 1, + ] + ], + MODE [ + EN OFFSET(0) NUMBITS(1) [], + ] +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x00 => IRQ_EN: ReadWrite), + (0x04 => IRQ_STA: ReadWrite), + (0x08 => _0), + (0x10 => CTRL: ReadWrite), + (0x14 => CFG: ReadWrite), + (0x18 => MODE: ReadWrite), + (0x1C => @END), + } +} + +pub struct RWatchdog { + inner: OneTimeInit>>, + base: usize, +} + +impl InitializableDevice for RWatchdog { + type Options = (); + + unsafe fn init(&self, opts: Self::Options) -> Result<(), Error> { + let regs = DeviceMemoryIo::map("r_wdog", self.base)?; + + self.inner.init(IrqSafeSpinlock::new(regs)); + + Ok(()) + } +} + +impl RWatchdog { + pub unsafe fn reset_board(&self) -> ! { + let regs = self.inner.get().lock(); + + regs.CFG.write(CFG::CONFIG::System); + regs.MODE.write(MODE::EN::SET); + regs.CTRL.write(CTRL::KEY::Value + CTRL::RESTART::SET); + + loop { + core::arch::asm!("wfe"); + } + } + + pub const unsafe fn new(base: usize) -> Self { + Self { + inner: OneTimeInit::new(), + base, + } + } +} diff --git a/src/arch/aarch64/plat_orange_pi3/serial.rs b/src/arch/aarch64/plat_orange_pi3/serial.rs new file mode 100644 index 00000000..00bea9c9 --- /dev/null +++ b/src/arch/aarch64/plat_orange_pi3/serial.rs @@ -0,0 +1,153 @@ +use abi::{error::Error, io::DeviceRequest}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite}, +}; +use vfs::CharDevice; + +use crate::{ + arch::aarch64::gic::IrqNumber, + arch::PLATFORM, + debug::DebugSink, + device::{ + interrupt::InterruptSource, + platform::Platform, + serial::SerialDevice, + tty::{CharRing, TtyDevice}, + Device, InitializableDevice, + }, + mem::device::DeviceMemoryIo, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +register_bitfields! { + u32, + USR [ + TFE OFFSET(2) NUMBITS(1) [], + TFNF OFFSET(1) NUMBITS(1) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x00 => DLL: ReadWrite), + (0x04 => _0), + (0x7C => USR: ReadOnly), + (0x80 => @END), + } +} + +struct Inner { + regs: DeviceMemoryIo, +} + +pub struct Serial { + inner: OneTimeInit>, + ring: CharRing<16>, + base: usize, + irq: IrqNumber, +} + +impl CharDevice for Serial { + fn read(&'static self, _blocking: bool, data: &mut [u8]) -> Result { + self.line_read(data) + } + + fn write(&self, _blocking: bool, data: &[u8]) -> Result { + self.line_write(data) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(id as _); + Ok(()) + } + _ => Err(Error::InvalidArgument), + } + } +} + +impl TtyDevice<16> for Serial { + fn ring(&self) -> &CharRing<16> { + &self.ring + } +} + +impl InterruptSource for Serial { + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = PLATFORM.interrupt_controller(); + + intc.register_handler(self.irq, self)?; + intc.enable_irq(self.irq)?; + + Ok(()) + } + + fn handle_irq(&self) -> Result { + let byte = self.inner.get().lock().regs.DLL.get(); + + self.recv_byte(byte as u8); + + Ok(true) + } +} + +impl DebugSink for Serial { + fn putc(&self, c: u8) -> Result<(), Error> { + self.send(c).ok(); + Ok(()) + } +} + +impl SerialDevice for Serial { + fn send(&self, byte: u8) -> Result<(), Error> { + let inner = self.inner.get().lock(); + if byte == b'\n' { + while inner.regs.USR.matches_all(USR::TFE::CLEAR) { + core::hint::spin_loop(); + } + inner.regs.DLL.set(b'\r' as u32); + } + while inner.regs.USR.matches_all(USR::TFE::CLEAR) { + core::hint::spin_loop(); + } + inner.regs.DLL.set(byte as u32); + Ok(()) + } + + fn receive(&self, blocking: bool) -> Result { + loop {} + } +} + +impl InitializableDevice for Serial { + type Options = (); + + unsafe fn init(&self, opts: Self::Options) -> Result<(), Error> { + let regs = DeviceMemoryIo::::map("h6-uart", self.base)?; + self.inner.init(IrqSafeSpinlock::new(Inner { regs })); + Ok(()) + } +} + +impl Device for Serial { + fn name(&self) -> &'static str { + "Allwinner H6 UART" + } +} + +impl Serial { + pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self { + Self { + inner: OneTimeInit::new(), + ring: CharRing::new(), + + base, + irq, + } + } +} diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs index ea019ec1..fdd0ca09 100644 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -22,14 +22,14 @@ use super::{ }; /// AArch64 "virt" platform implementation -pub struct QemuPlatform { +pub struct PlatformImpl { /// ... pub gic: Gic, pl011: Pl011, local_timer: ArmTimer, } -impl Platform for QemuPlatform { +impl Platform for PlatformImpl { type IrqNumber = IrqNumber; const KERNEL_PHYS_BASE: usize = 0x40080000; @@ -78,11 +78,15 @@ impl Platform for QemuPlatform { infoln!("TODO send ipi"); loop {} } + + unsafe fn reset(&self) -> ! { + loop {} + } } /// AArch64 "virt" platform -pub static PLATFORM: QemuPlatform = unsafe { - QemuPlatform { +pub static PLATFORM: PlatformImpl = unsafe { + PlatformImpl { pl011: Pl011::new(0x09000000, IrqNumber::new(33)), gic: Gic::new(0x08000000, 0x08010000), local_timer: ArmTimer::new(IrqNumber::new(30)), diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 63416dd7..62b69964 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -7,7 +7,7 @@ use abi::error::Error; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::PLATFORM, + arch::{aarch64::gic::IrqNumber, PLATFORM}, device::{ interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device, InitializableDevice, @@ -16,7 +16,7 @@ use crate::{ task::tasklet, }; -use super::{cpu::Cpu, gic::IrqNumber}; +use super::cpu::Cpu; /// ARM Generic Timer driver pub struct ArmTimer { diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 9e7cbd50..82a08831 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -20,14 +20,11 @@ macro_rules! absolute_address { }}; } +pub mod aarch64; cfg_if! { if #[cfg(target_arch = "aarch64")] { - pub mod aarch64; - - pub use aarch64::plat_qemu::{QemuPlatform as PlatformImpl, PLATFORM}; - pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; - + pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE, PlatformImpl, PLATFORM}; } else if #[cfg(target_arch = "x86_64")] { pub mod x86_64; diff --git a/src/device/mod.rs b/src/device/mod.rs index 0660d4a6..b0eee8b7 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -2,6 +2,7 @@ use abi::error::Error; +#[cfg(feature = "fb_console")] pub mod display; pub mod input; pub mod interrupt; diff --git a/src/device/platform.rs b/src/device/platform.rs index 1f99bf7a..5d2d4562 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -60,4 +60,6 @@ pub trait Platform { /// /// As the call may alter the flow of execution on CPUs, this function is unsafe. unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; + + unsafe fn reset(&self) -> !; } diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index cc4765f4..b195d1e1 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -3,7 +3,7 @@ use abi::error::Error; use super::Device; -#[cfg(target_arch = "aarch64")] +#[cfg(all(target_arch = "aarch64", not(feature = "aarch64_orange_pi3")))] pub mod pl011; /// Generic serial device interface diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 4144310c..e6e3fcd3 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -9,7 +9,7 @@ use vfs::CharDevice; use super::SerialDevice; use crate::{ - arch::{aarch64::gic::IrqNumber, PLATFORM}, + arch::{aarch64::gic::IrqNumber, PlatformImpl, PLATFORM}, debug::DebugSink, device::{ interrupt::InterruptSource, diff --git a/src/device/tty.rs b/src/device/tty.rs index c667c8d2..22f0ffef 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -1,91 +1,105 @@ //! Terminal driver implementation use abi::{ error::Error, - io::{ - DeviceRequest, TerminalInputOptions, TerminalLineOptions, TerminalOptions, - TerminalOutputOptions, - }, + io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, process::Signal, }; -use vfs::CharDevice; use crate::{ - device::display::fb_console::FramebufferConsole, proc::wait::Wait, sync::IrqSafeSpinlock, task::{process::Process, ProcessId}, }; -use super::{ - display::console::DisplayConsole, input::KeyboardDevice, serial::SerialDevice, Device, -}; +use super::serial::SerialDevice; -// TODO rewrite this -/// Helper device to combine a display and a keyboard input into a single terminal -pub struct CombinedTerminal { - #[allow(dead_code)] - input: &'static dyn KeyboardDevice, - output: &'static dyn DisplayConsole, +#[cfg(feature = "fb_console")] +pub mod combined { + use abi::{error::Error, io::DeviceRequest}; + use vfs::CharDevice; - input_ring: CharRing<16>, -} + use crate::device::{ + display::{console::DisplayConsole, fb_console::FramebufferConsole}, + input::KeyboardDevice, + serial::SerialDevice, + Device, + }; -impl CombinedTerminal { - /// Create a combined terminal device from a keyboard and console output devices - pub fn new(input: &'static dyn KeyboardDevice, output: &'static FramebufferConsole) -> Self { - Self { - input, - output, - input_ring: CharRing::new(), - } - } -} + use super::{CharRing, TtyDevice}; -impl TtyDevice<16> for CombinedTerminal { - fn ring(&self) -> &CharRing<16> { - &self.input_ring - } -} + // TODO rewrite this + /// Helper device to combine a display and a keyboard input into a single terminal + pub struct CombinedTerminal { + #[allow(dead_code)] + input: &'static dyn KeyboardDevice, + output: &'static dyn DisplayConsole, -impl SerialDevice for CombinedTerminal { - fn receive(&self, _blocking: bool) -> Result { - todo!() + input_ring: CharRing<16>, } - fn send(&self, byte: u8) -> Result<(), Error> { - self.output.write_char(byte); - Ok(()) - } -} - -impl Device for CombinedTerminal { - fn name(&self) -> &'static str { - "Combined terminal device" - } -} - -impl CharDevice for CombinedTerminal { - fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { - assert!(blocking); - self.line_read(data) - } - - fn write(&self, blocking: bool, data: &[u8]) -> Result { - assert!(blocking); - self.line_write(data) - } - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - match req { - &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(id as _); - Ok(()) + impl CombinedTerminal { + /// Create a combined terminal device from a keyboard and console output devices + pub fn new( + input: &'static dyn KeyboardDevice, + output: &'static FramebufferConsole, + ) -> Self { + Self { + input, + output, + input_ring: CharRing::new(), + } + } + } + + impl TtyDevice<16> for CombinedTerminal { + fn ring(&self) -> &CharRing<16> { + &self.input_ring + } + } + + impl SerialDevice for CombinedTerminal { + fn receive(&self, _blocking: bool) -> Result { + todo!() + } + + fn send(&self, byte: u8) -> Result<(), Error> { + self.output.write_char(byte); + Ok(()) + } + } + + impl Device for CombinedTerminal { + fn name(&self) -> &'static str { + "Combined terminal device" + } + } + + impl CharDevice for CombinedTerminal { + fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { + assert!(blocking); + self.line_read(data) + } + + fn write(&self, blocking: bool, data: &[u8]) -> Result { + assert!(blocking); + self.line_write(data) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(id as _); + Ok(()) + } + _ => Err(Error::InvalidArgument), } - _ => Err(Error::InvalidArgument), } } } +#[cfg(feature = "fb_console")] +pub use combined::CombinedTerminal; + struct CharRingInner { rd: usize, wr: usize, @@ -209,7 +223,7 @@ pub trait TtyDevice: SerialDevice { } else if byte == config.chars.erase { // Erase if off != 0 { - self.raw_write(b"\x1b[D \x1b[D"); + self.raw_write(b"\x1b[D \x1b[D")?; off -= 1; rem += 1; } diff --git a/src/main.rs b/src/main.rs index 27fce02b..e277d847 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,6 @@ use abi::{ error::Error, io::{FileMode, OpenOptions, RawFd}, }; -use device::display::console::task_update_consoles; use fs::{devfs, FileBlockAllocator, INITRD_DATA}; use memfs::MemoryFilesystem; use task::tasklet; @@ -60,11 +59,15 @@ fn setup_root() -> Result { /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kernel_main() { - tasklet::add_periodic( - "update-console", - Duration::from_millis(15), - task_update_consoles, - ); + #[cfg(feature = "fb_console")] + { + use device::display::console::task_update_consoles; + tasklet::add_periodic( + "update-console", + Duration::from_millis(15), + task_update_consoles, + ); + } let root = match setup_root() { Ok(root) => root, diff --git a/src/mem/mod.rs b/src/mem/mod.rs index fcd6fab8..02cfd4f5 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,7 +1,7 @@ //! Memory management utilities and types // use core::{alloc::Layout, mem::size_of}; -use core::{alloc::Layout, mem::size_of}; +use core::{alloc::Layout, ffi::c_void, mem::size_of}; use abi::error::Error; @@ -286,3 +286,72 @@ pub fn validate_user_region( Ok(()) } + +#[no_mangle] +unsafe extern "C" fn memcpy(p0: *mut c_void, p1: *const c_void, len: usize) -> *mut c_void { + let mut offset = 0; + while offset < len { + let c = (p1 as *const u8).add(offset).read_volatile(); + (p0 as *mut u8).add(offset).write_volatile(c); + + offset += 1; + } + p0 +} + +#[no_mangle] +unsafe extern "C" fn memcmp(p0: *const c_void, p1: *const c_void, len: usize) -> i32 { + let mut offset = 0; + + if len == 0 { + return 0; + } + + while offset < len { + let c0 = (p0 as *const u8).add(offset).read_volatile(); + let c1 = (p1 as *const u8).add(offset).read_volatile(); + + if c0 > c1 { + return (c0 - c1) as i32; + } else if c0 < c1 { + return -((c1 - c0) as i32); + } + + offset += 1; + } + + 0 +} + +#[no_mangle] +unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { + if dst as usize == src as usize { + return dst; + } + + if (src.add(len) as usize) <= (dst as usize) || (dst.add(len) as usize) <= (src as usize) { + return memcpy(dst, src, len); + } + + if (dst as usize) < (src as usize) { + let a = src as usize - dst as usize; + memcpy(dst, src, a); + memcpy(src as *mut _, src.add(a), len - a); + } else { + let a = dst as usize - src as usize; + memcpy(dst.add(a), dst, len - a); + memcpy(dst, src, a); + } + + dst +} + +#[no_mangle] +unsafe extern "C" fn memset(dst: *mut c_void, val: i32, len: usize) -> *mut c_void { + let mut offset = 0; + while offset < len { + (dst as *mut u8).add(offset).write_volatile(val as u8); + offset += 1; + } + dst +} diff --git a/src/panic.rs b/src/panic.rs index 807a62fd..0f515619 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -42,19 +42,19 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() { - // Let other CPUs know we're screwed - unsafe { - PLATFORM - .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) - .ok(); - } + // // Let other CPUs know we're screwed + // unsafe { + // PLATFORM + // .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) + // .ok(); + // } - let ap_count = ArchitectureImpl::cpu_count() - 1; - PANIC_HANDLED_FENCE.wait_all(ap_count); + // let ap_count = ArchitectureImpl::cpu_count() - 1; + // PANIC_HANDLED_FENCE.wait_all(ap_count); - unsafe { - hack_locks(); - } + // unsafe { + // hack_locks(); + // } log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); diff --git a/src/util.rs b/src/util.rs index 0dc483fe..acf40ad4 100644 --- a/src/util.rs +++ b/src/util.rs @@ -99,7 +99,7 @@ impl StaticVector { /// /// Will panic if the vector is full. pub fn push(&mut self, value: T) { - if self.len == N { + if self.len >= N { panic!("Static vector overflow: reached limit of {}", N); } From a824e5209558256a98cdb446052a465bba4d324f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 9 Aug 2023 00:14:25 +0300 Subject: [PATCH 045/211] aarch64: fix warnings --- src/arch/aarch64/boot/mod.rs | 17 +++++++-------- src/arch/aarch64/mod.rs | 4 ++-- src/arch/aarch64/plat_orange_pi3/mod.rs | 24 +++++++++++++--------- src/arch/aarch64/plat_orange_pi3/r_wdog.rs | 14 ++++++++++++- src/arch/aarch64/plat_orange_pi3/serial.rs | 13 +++++++++--- src/arch/aarch64/plat_qemu/mod.rs | 10 +++++---- src/debug.rs | 2 ++ src/device/platform.rs | 8 +++++++- src/device/serial/pl011.rs | 2 +- src/main.rs | 7 ++++--- src/mem/mod.rs | 1 + src/panic.rs | 5 ++--- src/sync.rs | 6 ++++++ src/util.rs | 9 +++++++- 14 files changed, 83 insertions(+), 39 deletions(-) diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index df2b33ea..b00b4c39 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -4,14 +4,8 @@ use core::{ sync::atomic::Ordering, }; -use aarch64_cpu::{ - asm::barrier, - registers::{ - CurrentEL, CNTHCTL_EL2, CNTVOFF_EL2, CPACR_EL1, ELR_EL2, HCR_EL2, SCTLR_EL2, SPSR_EL2, - SP_EL1, - }, -}; -use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use aarch64_cpu::{asm::barrier, registers::CPACR_EL1}; +use tock_registers::interfaces::ReadWriteable; use super::{ cpu::Cpu, exception, kernel_main, smp::CPU_COUNT, AArch64, KernelStack, ARCHITECTURE, @@ -26,6 +20,7 @@ use crate::{ task, }; +#[allow(dead_code)] pub(super) static CPU_INIT_FENCE: SpinFence = SpinFence::new(); extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { @@ -54,8 +49,10 @@ fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { } } -pub(super) extern "C" fn __aarch64_ap_lower_entry(sp: usize) -> ! { - loop {} +pub(super) extern "C" fn __aarch64_ap_lower_entry(_sp: usize) -> ! { + loop { + core::hint::spin_loop(); + } // __aarch64_common_lower_entry(); // unsafe { diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 3a3cef2c..9e442f34 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -10,7 +10,7 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ arch::{ - aarch64::{boot::CPU_INIT_FENCE, cpu::Cpu, devtree::FdtMemoryRegionIter, smp::CPU_COUNT}, + aarch64::{cpu::Cpu, devtree::FdtMemoryRegionIter, smp::CPU_COUNT}, Architecture, }, debug, @@ -30,13 +30,13 @@ use self::{ table::{init_fixed_tables, KERNEL_TABLES}, }; -pub mod plat_orange_pi3; cfg_if! { if #[cfg(feature = "aarch64_qemu")] { pub mod plat_qemu; pub use plat_qemu::{PlatformImpl, PLATFORM}; } else if #[cfg(feature = "aarch64_orange_pi3")] { + pub mod plat_orange_pi3; pub use plat_orange_pi3::{PlatformImpl, PLATFORM}; } else { diff --git a/src/arch/aarch64/plat_orange_pi3/mod.rs b/src/arch/aarch64/plat_orange_pi3/mod.rs index 7d2eb9df..8b650c3d 100644 --- a/src/arch/aarch64/plat_orange_pi3/mod.rs +++ b/src/arch/aarch64/plat_orange_pi3/mod.rs @@ -1,12 +1,13 @@ -/// -/// # Booting using u-boot -/// -/// > fatload mmc 0:1 0x40000000 uRamdisk -/// > fatload mmc 0:1 0x4d000000 sun50i-h6-orangepi-3.dtb -/// > loady 0x44000000 -/// ... -/// > bootm 0x44000000 0x40000000 0x4d000000 -/// +//! Orange Pi 3 (Allwinner H6/sun50i-h6 SoC) +//! +//! # Booting using u-boot +//! +//! > fatload mmc 0:1 0x40000000 uRamdisk +//! > fatload mmc 0:1 0x4d000000 sun50i-h6-orangepi-3.dtb +//! > loady 0x44000000 +//! ... +//! > bootm 0x44000000 0x40000000 0x4d000000 +//! use abi::error::Error; use crate::{ @@ -31,9 +32,11 @@ use super::{ pub mod r_wdog; pub mod serial; +/// Orange Pi 3 implementation pub struct PlatformImpl { uart0: Serial, r_wdog: RWatchdog, + /// ... pub gic: Gic, timer: ArmTimer, } @@ -76,7 +79,7 @@ impl Platform for PlatformImpl { &self.timer } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { todo!(); } @@ -89,6 +92,7 @@ impl Platform for PlatformImpl { } } +/// Orange Pi 3 platform implementation pub static PLATFORM: PlatformImpl = unsafe { PlatformImpl { uart0: Serial::new(0x05000000, IrqNumber::new(32)), diff --git a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs b/src/arch/aarch64/plat_orange_pi3/r_wdog.rs index 4f86a935..5deac6a9 100644 --- a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs +++ b/src/arch/aarch64/plat_orange_pi3/r_wdog.rs @@ -1,3 +1,4 @@ +//! Allwinner H6 R Watchdog driver use abi::error::Error; use tock_registers::{ interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, @@ -39,6 +40,7 @@ register_structs! { } } +/// Allwinner H6 R Watchdog pub struct RWatchdog { inner: OneTimeInit>>, base: usize, @@ -47,7 +49,7 @@ pub struct RWatchdog { impl InitializableDevice for RWatchdog { type Options = (); - unsafe fn init(&self, opts: Self::Options) -> Result<(), Error> { + unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { let regs = DeviceMemoryIo::map("r_wdog", self.base)?; self.inner.init(IrqSafeSpinlock::new(regs)); @@ -57,6 +59,11 @@ impl InitializableDevice for RWatchdog { } impl RWatchdog { + /// Performs a reset through watchdog. + /// + /// # Safety + /// + /// Only meant to be called by platform reset code. pub unsafe fn reset_board(&self) -> ! { let regs = self.inner.get().lock(); @@ -69,6 +76,11 @@ impl RWatchdog { } } + /// Constructs an instance of the device at `base`. + /// + /// # Safety + /// + /// The caller must ensure the address is valid. pub const unsafe fn new(base: usize) -> Self { Self { inner: OneTimeInit::new(), diff --git a/src/arch/aarch64/plat_orange_pi3/serial.rs b/src/arch/aarch64/plat_orange_pi3/serial.rs index 00bea9c9..864189a0 100644 --- a/src/arch/aarch64/plat_orange_pi3/serial.rs +++ b/src/arch/aarch64/plat_orange_pi3/serial.rs @@ -1,3 +1,4 @@ +//! Allwinner H6 UART driver use abi::{error::Error, io::DeviceRequest}; use tock_registers::{ interfaces::{Readable, Writeable}, @@ -44,6 +45,7 @@ struct Inner { regs: DeviceMemoryIo, } +/// Allwinner H6 UART pub struct Serial { inner: OneTimeInit>, ring: CharRing<16>, @@ -119,15 +121,15 @@ impl SerialDevice for Serial { Ok(()) } - fn receive(&self, blocking: bool) -> Result { - loop {} + fn receive(&self, _blocking: bool) -> Result { + todo!() } } impl InitializableDevice for Serial { type Options = (); - unsafe fn init(&self, opts: Self::Options) -> Result<(), Error> { + unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { let regs = DeviceMemoryIo::::map("h6-uart", self.base)?; self.inner.init(IrqSafeSpinlock::new(Inner { regs })); Ok(()) @@ -141,6 +143,11 @@ impl Device for Serial { } impl Serial { + /// Constructs an instance of the device at `base`. + /// + /// # Safety + /// + /// The caller must ensure the address is valid. pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self { Self { inner: OneTimeInit::new(), diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs index fdd0ca09..7135a4ea 100644 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -5,7 +5,7 @@ use tock_registers::interfaces::Writeable; use crate::{ arch::CpuMessage, - debug::{self, DebugSink, LogLevel}, + debug::{self, LogLevel}, device::{ interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, platform::Platform, @@ -74,13 +74,15 @@ impl Platform for PlatformImpl { &self.local_timer } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { infoln!("TODO send ipi"); - loop {} + loop { + core::hint::spin_loop(); + } } unsafe fn reset(&self) -> ! { - loop {} + todo!() } } diff --git a/src/debug.rs b/src/debug.rs index 3fe6f320..4940ccc9 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -137,12 +137,14 @@ impl fmt::Write for DebugSinkWrapper { } } +/// Adds a debugging output sink pub fn add_sink(sink: &'static dyn DebugSink, level: LogLevel) { DEBUG_SINKS .lock() .push(DebugSinkWrapper { inner: sink, level }); } +/// Resets the debugging terminal by clearing it pub fn reset() { log_print_raw!(LogLevel::Info, "\x1b[0m"); } diff --git a/src/device/platform.rs b/src/device/platform.rs index 5d2d4562..2d59fd76 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -2,7 +2,7 @@ use abi::error::Error; -use crate::{arch::CpuMessage, debug::DebugSink}; +use crate::arch::CpuMessage; use super::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget}, @@ -61,5 +61,11 @@ pub trait Platform { /// As the call may alter the flow of execution on CPUs, this function is unsafe. unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; + /// Performs a CPU reset. + /// + /// # Safety + /// + /// The caller must ensure it is actually safe to reset, i.e. no critical processes will be + /// aborted and no data will be lost. unsafe fn reset(&self) -> !; } diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index e6e3fcd3..4144310c 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -9,7 +9,7 @@ use vfs::CharDevice; use super::SerialDevice; use crate::{ - arch::{aarch64::gic::IrqNumber, PlatformImpl, PLATFORM}, + arch::{aarch64::gic::IrqNumber, PLATFORM}, debug::DebugSink, device::{ interrupt::InterruptSource, diff --git a/src/main.rs b/src/main.rs index e277d847..784597b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,15 +16,12 @@ #![no_std] #![no_main] -use core::time::Duration; - use abi::{ error::Error, io::{FileMode, OpenOptions, RawFd}, }; use fs::{devfs, FileBlockAllocator, INITRD_DATA}; use memfs::MemoryFilesystem; -use task::tasklet; use vfs::{Filesystem, IoContext, VnodeRef}; extern crate yggdrasil_abi as abi; @@ -61,7 +58,11 @@ fn setup_root() -> Result { pub fn kernel_main() { #[cfg(feature = "fb_console")] { + use core::time::Duration; + use device::display::console::task_update_consoles; + use task::tasklet; + tasklet::add_periodic( "update-console", Duration::from_millis(15), diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 02cfd4f5..a3f3f062 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -311,6 +311,7 @@ unsafe extern "C" fn memcmp(p0: *const c_void, p1: *const c_void, len: usize) -> let c0 = (p0 as *const u8).add(offset).read_volatile(); let c1 = (p1 as *const u8).add(offset).read_volatile(); + #[allow(clippy::comparison_chain)] if c0 > c1 { return (c0 - c1) as i32; } else if c0 < c1 { diff --git a/src/panic.rs b/src/panic.rs index 0f515619..e003320d 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,10 +2,9 @@ use core::sync::atomic::{AtomicBool, Ordering}; use crate::{ - arch::{Architecture, ArchitectureImpl, CpuMessage, PLATFORM}, + arch::{Architecture, ArchitectureImpl}, debug::{debug_internal, LogLevel}, - device::{interrupt::IpiDeliveryTarget, platform::Platform}, - sync::{hack_locks, SpinFence}, + sync::SpinFence, task::{sched::CpuQueue, Cpu}, }; diff --git a/src/sync.rs b/src/sync.rs index c29d37dd..dc8debf9 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -9,6 +9,11 @@ use crate::arch::{Architecture, ArchitectureImpl}; static LOCK_HACK: AtomicBool = AtomicBool::new(false); +/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. +/// +/// # Safety +/// +/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. pub unsafe fn hack_locks() { LOCK_HACK.store(true, Ordering::Release); } @@ -199,6 +204,7 @@ impl SpinFence { self.wait_all(1); } + /// Returns `true` if the fence has been signalled at least the amount of times specified pub fn try_wait_all(&self, count: usize) -> bool { self.value.load(Ordering::Acquire) >= count } diff --git a/src/util.rs b/src/util.rs index acf40ad4..9db3ad13 100644 --- a/src/util.rs +++ b/src/util.rs @@ -117,9 +117,16 @@ impl StaticVector { self.len == 0 } + /// Clears the vector, dropping all elements pub fn clear(&mut self) { - // TODO actually drop contents + let old_len = self.len; self.len = 0; + + for i in 0..old_len { + unsafe { + self.data[i].assume_init_drop(); + } + } } } From c49c9fdacdbd3fcbe4693d7a28f01223700eee0e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 9 Aug 2023 00:17:39 +0300 Subject: [PATCH 046/211] ***: fix cfg_if --- src/arch/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 82a08831..f246e135 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -20,9 +20,9 @@ macro_rules! absolute_address { }}; } -pub mod aarch64; cfg_if! { if #[cfg(target_arch = "aarch64")] { + pub mod aarch64; pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE, PlatformImpl, PLATFORM}; } else if #[cfg(target_arch = "x86_64")] { From 37e6fc098ba4aad11589b7f00a1b5d3336cbf1ce Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 9 Aug 2023 10:36:32 +0300 Subject: [PATCH 047/211] dev: implement #*-cells according to the spec --- Cargo.toml | 3 +- src/arch/aarch64/devtree.rs | 99 +++++++++++++++++++++++++++++++++---- src/arch/aarch64/mod.rs | 30 ++++++----- src/arch/mod.rs | 2 +- 4 files changed, 110 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bcb5f89a..c6ec5ca1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,8 @@ aarch64-cpu = "9.3.1" bitmap-font = { version = "0.3.0", optional = true } embedded-graphics = { version = "0.8.0", optional = true } +fdt-rs = { version = "0.4.3", default-features = false } + [dependencies.elf] version = "0.7.2" git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" @@ -32,7 +34,6 @@ default-features = false features = ["no_std_stream"] [target.'cfg(target_arch = "aarch64")'.dependencies] -fdt-rs = { version = "0.4.3", default-features = false } [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index 1058ebde..c16670b7 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -25,10 +25,33 @@ pub type TNode<'a> = DevTreeIndexNode<'a, 'a, 'a>; /// Device tree property pub type TProp<'a> = DevTreeIndexProp<'a, 'a, 'a>; +/// Helper trait to provide extra functionality for [DevTreeIndexProp] +pub trait DevTreeIndexPropExt { + /// Reads a cell value from single-type cell array at given cell index + fn cell1_array_item(&self, index: usize, cells: usize) -> Option; + /// Reads a cell pair from cell pair array at given pair index + fn cell2_array_item(&self, index: usize, cells0: usize, cells1: usize) -> Option<(u64, u64)>; + + /// Reads a cell value from the property at given offset + fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option; +} + +/// Helper trait to provide extra functionality for [DevTreeIndexNode] +pub trait DevTreeIndexNodeExt { + /// Returns the root node's `#address-cells` property, or the default value defined by the + /// specification if it's absent + fn address_cells(&self) -> usize; + /// Returns the root node's `#size-cells` property, or the default value defined by the + /// specification if it's absent + fn size_cells(&self) -> usize; +} + /// Iterator for physical memory regions present in the device tree #[derive(Clone)] pub struct FdtMemoryRegionIter<'a> { inner: DevTreeIndexNodeSiblingIter<'a, 'a, 'a>, + address_cells: usize, + size_cells: usize, } /// Device tree wrapper struct @@ -63,13 +86,73 @@ impl<'a> DeviceTree<'a> { pub fn size(&self) -> usize { self.tree.totalsize() } + + /// Returns the root node's `#address-cells` property, or the default value defined by the + /// specification if it's absent + pub fn address_cells(&self) -> usize { + self.index.root().address_cells() + } + + /// Returns the root node's `#size-cells` property, or the default value defined by the + /// specification if it's absent + pub fn size_cells(&self) -> usize { + self.index.root().size_cells() + } +} + +impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { + fn address_cells(&self) -> usize { + self.props() + .find(|p| p.name().unwrap_or("") == "#address-cells") + .map(|p| p.u32(0).unwrap() as usize) + .unwrap_or(2) + } + + fn size_cells(&self) -> usize { + self.props() + .find(|p| p.name().unwrap_or("") == "#size-cells") + .map(|p| p.u32(0).unwrap() as usize) + .unwrap_or(1) + } +} + +impl<'a, 'i, 'dt> DevTreeIndexPropExt for DevTreeIndexProp<'a, 'i, 'dt> { + fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option { + match cell_size { + 1 => self.u32(u32_offset).map(|x| x as u64).ok(), + 2 => { + let high = self.u32(u32_offset).ok()? as u64; + let low = self.u32(u32_offset + 1).ok()? as u64; + + Some((high << 32) | low) + } + _ => unimplemented!(), + } + } + + fn cell1_array_item(&self, index: usize, cells: usize) -> Option { + self.read_cell(index * cells, cells) + } + + fn cell2_array_item(&self, index: usize, cells0: usize, cells1: usize) -> Option<(u64, u64)> { + let u32_index = index * (cells0 + cells1); + let cell0 = self.read_cell(u32_index, cells0)?; + let cell1 = self.read_cell(u32_index + cells0, cells1)?; + Some((cell0, cell1)) + } } impl<'a> FdtMemoryRegionIter<'a> { /// Constructs a memory region iterator for given device tree pub fn new(dt: &'a DeviceTree) -> Self { let inner = dt.index.root().children(); - Self { inner } + let address_cells = dt.address_cells(); + let size_cells = dt.size_cells(); + Self { + inner, + address_cells, + size_cells, + } } } @@ -90,15 +173,13 @@ impl Iterator for FdtMemoryRegionIter<'_> { .find(|p| p.name().unwrap_or("") == "reg") .unwrap(); - #[cfg(not(feature = "aarch64_orange_pi3"))] + let (base, size) = reg + .cell2_array_item(0, self.address_cells, self.size_cells) + .unwrap(); + break Some(PhysicalMemoryRegion { - base: reg.u64(0).unwrap() as usize, - size: reg.u64(1).unwrap() as usize, - }); - #[cfg(feature = "aarch64_orange_pi3")] - break Some(PhysicalMemoryRegion { - base: reg.u32(0).unwrap() as usize, - size: reg.u32(1).unwrap() as usize, + base: base as usize, + size: size as usize, }); } } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 9e442f34..e2c23c05 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -5,7 +5,6 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1}; use abi::error::Error; use cfg_if::cfg_if; -use fdt_rs::prelude::PropReader; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ @@ -26,14 +25,14 @@ use crate::{ }; use self::{ - devtree::DeviceTree, + devtree::{DevTreeIndexPropExt, DeviceTree}, table::{init_fixed_tables, KERNEL_TABLES}, }; +pub mod plat_qemu; + cfg_if! { if #[cfg(feature = "aarch64_qemu")] { - pub mod plat_qemu; - pub use plat_qemu::{PlatformImpl, PLATFORM}; } else if #[cfg(feature = "aarch64_orange_pi3")] { pub mod plat_orange_pi3; @@ -163,7 +162,7 @@ impl AArch64 { "dtb", PhysicalMemoryRegion { base: dtb_phys, - size: dt.size(), + size: (dt.size() + 0xFFF) & !0xFFF, }, ); @@ -175,6 +174,7 @@ impl AArch64 { fn setup_initrd() { let dt = ARCHITECTURE.device_tree(); + let Some(chosen) = dt.node_by_path("/chosen") else { return; }; @@ -186,15 +186,19 @@ fn setup_initrd() { return; }; - #[cfg(feature = "aarch64_orange_pi3")] - let initrd_start = initrd_start.u32(0).unwrap() as usize; - #[cfg(feature = "aarch64_orange_pi3")] - let initrd_end = initrd_end.u32(0).unwrap() as usize; + let address_cells = dt.address_cells(); - #[cfg(not(feature = "aarch64_orange_pi3"))] - let initrd_start = initrd_start.u64(0).unwrap() as usize; - #[cfg(not(feature = "aarch64_orange_pi3"))] - let initrd_end = initrd_end.u64(0).unwrap() as usize; + let Some(initrd_start) = initrd_start.cell1_array_item(0, address_cells) else { + infoln!("No initrd specified"); + return; + }; + let Some(initrd_end) = initrd_end.cell1_array_item(0, address_cells) else { + infoln!("No initrd specified"); + return; + }; + + let initrd_start = initrd_start as usize; + let initrd_end = initrd_end as usize; let start_aligned = initrd_start & !0xFFF; let end_aligned = initrd_end & !0xFFF; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f246e135..82a08831 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -20,9 +20,9 @@ macro_rules! absolute_address { }}; } +pub mod aarch64; cfg_if! { if #[cfg(target_arch = "aarch64")] { - pub mod aarch64; pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE, PlatformImpl, PLATFORM}; } else if #[cfg(target_arch = "x86_64")] { From e694c1cef0cab3306c2da5783f7e8006c4de2772 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 13 Aug 2023 21:23:58 +0300 Subject: [PATCH 048/211] dev: use device tree instead of hardcoded board impls --- Cargo.toml | 2 + lib/device-api/Cargo.toml | 14 ++ lib/device-api/macros/Cargo.toml | 14 ++ lib/device-api/macros/src/lib.rs | 0 lib/device-api/src/bus.rs | 7 + lib/device-api/src/device.rs | 37 +++++ lib/device-api/src/interrupt.rs | 116 ++++++++++++++ lib/device-api/src/lib.rs | 24 +++ lib/device-api/src/manager.rs | 25 +++ lib/device-api/src/serial.rs | 7 + lib/device-api/src/timer.rs | 31 ++++ src/arch/aarch64/boot/entry.S | 67 +++++--- src/arch/aarch64/boot/mod.rs | 47 +++--- src/arch/aarch64/context.rs | 2 +- src/arch/aarch64/cpu.rs | 19 ++- src/arch/aarch64/devtree.rs | 54 ++++++- src/arch/aarch64/exception.rs | 39 +++-- src/arch/aarch64/gic/gicc.rs | 6 +- src/arch/aarch64/gic/gicd.rs | 18 +-- src/arch/aarch64/gic/mod.rs | 235 ++++++++++++++-------------- src/arch/aarch64/mod.rs | 250 +++++++++++++++++++++++++----- src/arch/aarch64/plat_qemu/mod.rs | 34 ++-- src/arch/aarch64/smp.rs | 159 +++++++++---------- src/arch/aarch64/table.rs | 5 +- src/arch/aarch64/timer.rs | 106 +++++++++---- src/arch/mod.rs | 89 +++++++++-- src/arch/x86_64/boot/mod.rs | 51 +++--- src/device/bus/mod.rs | 3 + src/device/bus/simple_bus.rs | 27 ++++ src/device/interrupt.rs | 88 ----------- src/device/mod.rs | 196 ++++++++++++++++++++--- src/device/platform.rs | 51 ------ src/device/power/arm_psci.rs | 81 ++++++++++ src/device/power/mod.rs | 3 + src/device/serial/mod.rs | 14 +- src/device/serial/pl011.rs | 122 ++++++++------- src/device/serial/sunxi_uart.rs | 154 ++++++++++++++++++ src/device/tty.rs | 3 +- src/main.rs | 2 + src/mem/device.rs | 2 + src/mem/heap.rs | 17 +- src/mem/mod.rs | 7 +- src/mem/phys/mod.rs | 10 +- src/panic.rs | 28 ++-- src/proc/wait.rs | 7 +- src/task/mod.rs | 2 +- src/task/process.rs | 75 ++++++--- src/task/sched.rs | 38 +++-- 48 files changed, 1656 insertions(+), 732 deletions(-) create mode 100644 lib/device-api/Cargo.toml create mode 100644 lib/device-api/macros/Cargo.toml create mode 100644 lib/device-api/macros/src/lib.rs create mode 100644 lib/device-api/src/bus.rs create mode 100644 lib/device-api/src/device.rs create mode 100644 lib/device-api/src/interrupt.rs create mode 100644 lib/device-api/src/lib.rs create mode 100644 lib/device-api/src/manager.rs create mode 100644 lib/device-api/src/serial.rs create mode 100644 lib/device-api/src/timer.rs create mode 100644 src/device/bus/mod.rs create mode 100644 src/device/bus/simple_bus.rs delete mode 100644 src/device/interrupt.rs create mode 100644 src/device/power/arm_psci.rs create mode 100644 src/device/power/mod.rs create mode 100644 src/device/serial/sunxi_uart.rs diff --git a/Cargo.toml b/Cargo.toml index c6ec5ca1..7d68103e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ build = "build.rs" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } memfs = { path = "lib/memfs" } +device-api = { path = "lib/device-api", features = ["derive"] } atomic_enum = "0.2.0" bitflags = "2.3.3" @@ -45,3 +46,4 @@ default = [] fb_console = ["bitmap-font", "embedded-graphics"] aarch64_qemu = [] aarch64_orange_pi3 = [] +aarch64_rpi3b = [] diff --git a/lib/device-api/Cargo.toml b/lib/device-api/Cargo.toml new file mode 100644 index 00000000..814a018b --- /dev/null +++ b/lib/device-api/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "device-api" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +macros = { path = "macros", optional = true } + +[features] +default = [] +derive = ["macros"] diff --git a/lib/device-api/macros/Cargo.toml b/lib/device-api/macros/Cargo.toml new file mode 100644 index 00000000..aee949a6 --- /dev/null +++ b/lib/device-api/macros/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "macros" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0.66" +quote = "1.0.32" +syn = { version = "2.0.28", features = ["full"] } diff --git a/lib/device-api/macros/src/lib.rs b/lib/device-api/macros/src/lib.rs new file mode 100644 index 00000000..e69de29b diff --git a/lib/device-api/src/bus.rs b/lib/device-api/src/bus.rs new file mode 100644 index 00000000..b61b7e35 --- /dev/null +++ b/lib/device-api/src/bus.rs @@ -0,0 +1,7 @@ +use yggdrasil_abi::error::Error; + +use crate::manager::DeviceManager; + +pub trait Bus { + fn enumerate(&self, manager: &mut DeviceManager) -> Result<(), Error>; +} diff --git a/lib/device-api/src/device.rs b/lib/device-api/src/device.rs new file mode 100644 index 00000000..5ce73b4a --- /dev/null +++ b/lib/device-api/src/device.rs @@ -0,0 +1,37 @@ +use yggdrasil_abi::error::Error; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[repr(transparent)] +pub struct DeviceId(u64); + +pub trait Device: Send + 'static { + fn display_name(&self) -> &'static str; + + /// Initializes the device, making it ready for operation. + /// The method is also responsible for registering the device with appropriate OS subsystems + /// (e.g. registering a terminal ttySn for a serial port) + /// + /// # Safety + /// + /// The caller must make sure the function is only called once. + unsafe fn init(&'static self) -> Result<(), Error> { + Ok(()) + } + + /// Initializes the IRQ handling options on this device: binds its IRQ(s) to their handlers and + /// enables their reception. + /// + /// # Safety + /// + /// The caller must make sure the function is only called once. The caller must also make sure + /// the function is not called before the device's [Device::init] is called. + unsafe fn init_irq(&'static self) -> Result<(), Error> { + Ok(()) + } +} + +impl From for DeviceId { + fn from(value: usize) -> Self { + Self(value as u64) + } +} diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs new file mode 100644 index 00000000..decc14c8 --- /dev/null +++ b/lib/device-api/src/interrupt.rs @@ -0,0 +1,116 @@ +use yggdrasil_abi::error::Error; + +use crate::Device; + +#[derive(Default, Clone, Copy, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum IrqLevel { + #[default] + Default, + ActiveHigh, + ActiveLow, +} + +#[derive(Default, Clone, Copy, PartialEq, Eq, Debug)] +#[repr(u32)] +pub enum IrqTrigger { + #[default] + Default, + Edge, + Level, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IpiDeliveryTarget { + Specific(usize), + ThisCpu, + OtherCpus, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IrqNumber { + Private(u32), + Shared(u32), +} + +#[derive(Default, Clone, Copy, Debug)] +pub struct IrqOptions { + pub level: IrqLevel, + pub trigger: IrqTrigger, +} + +pub trait InterruptTable { + fn handler(&self, index: usize) -> Option<&'static dyn InterruptHandler>; +} + +pub trait ExternalInterruptController { + /// Performs IRQ delivery method configuration and registers a handler to execute when it is + /// fired + fn register_irq( + &self, + irq: IrqNumber, + options: IrqOptions, + handler: &'static dyn InterruptHandler, + ) -> Result<(), Error>; + + /// Enables the specified IRQ (unmasks it) + fn enable_irq(&self, irq: IrqNumber) -> Result<(), Error>; + + /// Handles a single pending interrupt on this controller. + /// The function is intended for interrupt controllers which have internal registers to track + /// the IRQ index and the order of interrupt handling (if multiple are handled in sequence) is + /// platform/controller specific. + fn handle_pending_irqs(&self) {} + /// Handles a single pending interrupt with a known index on this controller. + /// The function is intended for interrupt controllers where vectors "know" their interrupt + /// index. + fn handle_specific_irq(&self, #[allow(unused)] index: usize) {} +} + +pub trait LocalInterruptController { + type IpiMessage; + + fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error>; + + /// Initializes the local interrupt controller for an Application Processor instance. + /// + /// # Safety + /// + /// The caller must ensure this function is only called once per each AP (and only for APs). + unsafe fn init_ap(&self) -> Result<(), Error>; +} + +pub trait InterruptHandler: Device { + fn handle_irq(&self) -> bool; +} + +pub struct FixedInterruptTable { + entries: [Option<&'static dyn InterruptHandler>; SIZE], +} + +impl FixedInterruptTable { + pub const fn new() -> Self { + Self { + entries: [None; SIZE], + } + } + + pub fn insert( + &mut self, + index: usize, + handler: &'static dyn InterruptHandler, + ) -> Result<(), Error> { + if self.entries[index].is_some() { + todo!(); + } + + self.entries[index] = Some(handler); + Ok(()) + } +} + +impl InterruptTable for FixedInterruptTable { + fn handler(&self, index: usize) -> Option<&'static dyn InterruptHandler> { + self.entries[index] + } +} diff --git a/lib/device-api/src/lib.rs b/lib/device-api/src/lib.rs new file mode 100644 index 00000000..d73fceb6 --- /dev/null +++ b/lib/device-api/src/lib.rs @@ -0,0 +1,24 @@ +#![no_std] + +extern crate alloc; + +pub mod bus; +pub mod device; +pub mod interrupt; +pub mod manager; +pub mod serial; +pub mod timer; + +pub use device::{Device, DeviceId}; +use yggdrasil_abi::error::Error; + +pub trait CpuBringupDevice: Device { + /// Starts a CPU with given index, providing it with some argument value and instruction + /// pointer from which its execution should begin. + /// + /// # Safety + /// + /// This function is unsafe because it can have unexpected effects on the system state if + /// misused. + unsafe fn start_cpu(&self, id: usize, ip: usize, arg0: usize) -> Result<(), Error>; +} diff --git a/lib/device-api/src/manager.rs b/lib/device-api/src/manager.rs new file mode 100644 index 00000000..13573456 --- /dev/null +++ b/lib/device-api/src/manager.rs @@ -0,0 +1,25 @@ +use alloc::vec::Vec; + +use crate::{Device, DeviceId}; + +pub struct DeviceManager { + devices: Vec<&'static dyn Device>, +} + +impl DeviceManager { + pub const fn new() -> Self { + Self { + devices: Vec::new(), + } + } + + pub fn register(&mut self, device: &'static dyn Device) -> DeviceId { + let id = DeviceId::from(self.devices.len()); + self.devices.push(device); + id + } + + pub fn devices(&self) -> impl Iterator + '_ { + self.devices.iter().copied() + } +} diff --git a/lib/device-api/src/serial.rs b/lib/device-api/src/serial.rs new file mode 100644 index 00000000..3a78fb0d --- /dev/null +++ b/lib/device-api/src/serial.rs @@ -0,0 +1,7 @@ +use yggdrasil_abi::error::Error; + +use crate::Device; + +pub trait SerialDevice: Device { + fn send(&self, byte: u8) -> Result<(), Error>; +} diff --git a/lib/device-api/src/timer.rs b/lib/device-api/src/timer.rs new file mode 100644 index 00000000..3fe2ab66 --- /dev/null +++ b/lib/device-api/src/timer.rs @@ -0,0 +1,31 @@ +//! Interfaces for time-providing devices + +use core::time::Duration; + +use yggdrasil_abi::error::Error; + +use crate::Device; + +/// Interface for precise timing devices +pub trait MonotonicTimestampProviderDevice: Device { + /// Provides a timestamp value of the timer. The value: + /// + /// * Represents monotonically increasing clock time since some arbitrary point in the past. + /// * Can be used for delays and measuring time passed between two measurements. + /// + /// * Is not an accurate wall-clock time or real-world time. + /// * Cannot be used for date/time managament purposes. + fn monotonic_timestamp(&self) -> Result; +} + +/// Interface for real-world time-telling devices +pub trait RealTimeProviderDevice: Device { + /// Provides a real-time clock value of the timer. The value: + /// + /// * Represents a real-world time since TODO TODO TODO. + /// * Can be used for rough measurements of duration passed between two points in time. + /// * Can be used for delays, but precision is not guaranteed. + /// * Can be used for date/time management. + // TODO actual type for time + fn real_timestamp(&self) -> Result; +} diff --git a/src/arch/aarch64/boot/entry.S b/src/arch/aarch64/boot/entry.S index 8bf12667..7cdfcc48 100644 --- a/src/arch/aarch64/boot/entry.S +++ b/src/arch/aarch64/boot/entry.S @@ -13,7 +13,32 @@ movk \reg, #((\value) >> 16), lsl #16 .endm +.macro LEAVE_EL2, ret_label + mrs x8, CNTHCTL_EL2 + orr x8, x8, #(CNTHCTL_EL2_EL1PCTEN | CNTHCTL_EL2_EL1PCEN) + msr CNTHCTL_EL2, x8 + msr CNTVOFF_EL2, xzr + + MOV_L x8, SCTLR_EL2_RES1 + msr SCTLR_EL2, x8 + + mov x8, #HCR_EL2_RW_EL1IsAArch64 + msr HCR_EL2, x8 + + mov x8, #SPSR_EL2_EL1h + orr x8, x8, #SPSR_EL2_MASK_DAIF + msr SPSR_EL2, x8 + + adr x8, \ret_label + msr ELR_EL2, x8 + + isb + eret +.endm + .global __aarch64_entry +.global __aarch64_ap_entry + .section .text.entry __aarch64_entry: // x0 -- dtb_phys @@ -30,26 +55,7 @@ __aarch64_entry: // Leave EL2 .el2: - mrs x8, CNTHCTL_EL2 - orr x8, x8, #(CNTHCTL_EL2_EL1PCTEN | CNTHCTL_EL2_EL1PCEN) - msr CNTHCTL_EL2, x8 - msr CNTVOFF_EL2, xzr - - MOV_L x8, SCTLR_EL2_RES1 - msr SCTLR_EL2, x8 - - mov x8, #HCR_EL2_RW_EL1IsAArch64 - msr HCR_EL2, x8 - - mov x8, #SPSR_EL2_EL1h - orr x8, x8, #SPSR_EL2_MASK_DAIF - msr SPSR_EL2, x8 - - adr x8, .el1 - msr ELR_EL2, x8 - - isb - eret + LEAVE_EL2 .el1 .el1: dsb sy isb @@ -75,3 +81,24 @@ __aarch64_entry: // TODO spin loop for this method of init 1: b . + +.section .text +__aarch64_ap_entry: + // x0 -- stack pointer (lower address space) + + mrs x8, CurrentEL + lsr x8, x8, #2 + cmp x8, #2 + bne .ap_el1 + +.ap_el2: + LEAVE_EL2 .ap_el1 + +.ap_el1: + dsb sy + isb + + mov sp, x0 + bl {kernel_ap_lower_entry} - {kernel_virt_offset} + + b . diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index b00b4c39..881102e4 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -7,20 +7,15 @@ use core::{ use aarch64_cpu::{asm::barrier, registers::CPACR_EL1}; use tock_registers::interfaces::ReadWriteable; -use super::{ - cpu::Cpu, exception, kernel_main, smp::CPU_COUNT, AArch64, KernelStack, ARCHITECTURE, - BOOT_STACK_SIZE, -}; +use super::{cpu::Cpu, exception, kernel_main, KernelStack, ARCHITECTURE, BOOT_STACK_SIZE}; use crate::{ absolute_address, - arch::{Architecture, PLATFORM}, - device::platform::Platform, + arch::{aarch64::smp::CPU_COUNT, Architecture, ArchitectureImpl}, mem::{ConvertAddress, KERNEL_VIRT_OFFSET}, sync::SpinFence, task, }; -#[allow(dead_code)] pub(super) static CPU_INIT_FENCE: SpinFence = SpinFence::new(); extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { @@ -39,6 +34,22 @@ extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { enter_higher_half(sp as usize, elr, dtb_phys); } +unsafe extern "C" fn el1_ap_lower_entry(sp: usize) -> ! { + ArchitectureImpl::set_interrupt_mask(true); + + // Unmask FP operations + CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); + + unsafe { + ARCHITECTURE.init_mmu(false); + } + + let sp = sp.virtualize(); + let elr = absolute_address!(__aarch64_ap_upper_entry); + + enter_higher_half(sp, elr, 0); +} + fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { unsafe { asm!(r#" @@ -49,29 +60,12 @@ fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { } } -pub(super) extern "C" fn __aarch64_ap_lower_entry(_sp: usize) -> ! { - loop { - core::hint::spin_loop(); - } - // __aarch64_common_lower_entry(); - - // unsafe { - // ARCHITECTURE.init_mmu(false); - // } - - // let sp = unsafe { sp.virtualize() }; - // let elr = absolute_address!(__aarch64_ap_upper_entry); - // enter_higher_half(sp, elr, 0); -} - extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { kernel_main(dtb_phys); } extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { - unsafe { - AArch64::set_interrupt_mask(true); - } + assert!(ArchitectureImpl::interrupt_mask()); // Signal to BSP that we're up CPU_COUNT.fetch_add(1, Ordering::SeqCst); @@ -81,7 +75,7 @@ extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { // Initialize CPU-local GIC and timer unsafe { - PLATFORM.init(false).expect("AP platform init failed"); + ARCHITECTURE.init_platform(false); Cpu::init_local(); @@ -100,6 +94,7 @@ static BSP_STACK: KernelStack = KernelStack { global_asm!( include_str!("entry.S"), kernel_lower_entry = sym el1_bsp_lower_entry, + kernel_ap_lower_entry = sym el1_ap_lower_entry, stack_bottom = sym BSP_STACK, stack_size = const BOOT_STACK_SIZE, kernel_virt_offset = const KERNEL_VIRT_OFFSET diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index fdd9c612..3f4d128e 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -88,7 +88,7 @@ impl TaskContextImpl for TaskContext { const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { - const KERNEL_TASK_PAGES: usize = 4; + const KERNEL_TASK_PAGES: usize = 8; let stack_base = unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() }; diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index dab7db23..3a3f8cec 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -5,10 +5,14 @@ use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; use tock_registers::interfaces::{Readable, Writeable}; -use crate::{arch::CpuMessage, sync::IrqSafeSpinlock, task::sched::CpuQueue, util::OneTimeInit}; +use crate::{ + arch::CpuMessage, panic, sync::IrqSafeSpinlock, task::sched::CpuQueue, util::OneTimeInit, +}; use super::smp::CPU_COUNT; +// use super::smp::CPU_COUNT; + /// Per-CPU private data structure #[repr(C, align(0x10))] pub struct Cpu { @@ -108,4 +112,17 @@ impl Cpu { (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), )); } + + /// Gets an IPI message from the processor's queue and takes corresponding actions. If there is + /// none, this is treated as a spurious IPI and ignored. See [CpuMessage]. + pub fn handle_ipi(&self) { + let Some(msg) = self.get_ipi() else { + warnln!("Spurious IPI in cpu{}", self.id); + return; + }; + + match msg { + CpuMessage::Panic => panic::panic_secondary(), + } + } } diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index c16670b7..d0e8c99c 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -40,10 +40,49 @@ pub trait DevTreeIndexPropExt { pub trait DevTreeIndexNodeExt { /// Returns the root node's `#address-cells` property, or the default value defined by the /// specification if it's absent - fn address_cells(&self) -> usize; + fn address_cells(&self) -> usize { + self.get_address_cells().unwrap_or(2) + } /// Returns the root node's `#size-cells` property, or the default value defined by the /// specification if it's absent - fn size_cells(&self) -> usize; + fn size_cells(&self) -> usize { + self.get_size_cells().unwrap_or(1) + } + + /// Returns the #address-cells property of the node, if there is one + fn get_address_cells(&self) -> Option; + /// Returns the #size-cells property of the node, if there is one + fn get_size_cells(&self) -> Option; +} + +/// Extension trait for [DevTreeIndexNode] to obtain typed property values +pub trait DevTreeIndexNodePropGet { + /// Returns a property value of given type, if it exists + fn prop(&self, name: &str) -> Option; +} + +impl<'a, 'i, 'dt> DevTreeIndexNodePropGet for DevTreeIndexNode<'a, 'i, 'dt> { + fn prop(&self, name: &str) -> Option { + self.props().find_map(|prop| { + if prop.name().ok()? == name { + prop.u32(0).ok() + } else { + None + } + }) + } +} + +impl<'a, 'i, 'dt> DevTreeIndexNodePropGet<&'a str> for DevTreeIndexNode<'a, 'i, 'dt> { + fn prop(&self, name: &str) -> Option<&'a str> { + self.props().find_map(|prop| { + if prop.name().ok()? == name { + prop.str().ok() + } else { + None + } + }) + } } /// Iterator for physical memory regions present in the device tree @@ -98,21 +137,24 @@ impl<'a> DeviceTree<'a> { pub fn size_cells(&self) -> usize { self.index.root().size_cells() } + + /// Returns the root node of the device tree + pub fn root(&self) -> DevTreeIndexNode { + self.index.root() + } } impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { - fn address_cells(&self) -> usize { + fn get_address_cells(&self) -> Option { self.props() .find(|p| p.name().unwrap_or("") == "#address-cells") .map(|p| p.u32(0).unwrap() as usize) - .unwrap_or(2) } - fn size_cells(&self) -> usize { + fn get_size_cells(&self) -> Option { self.props() .find(|p| p.name().unwrap_or("") == "#size-cells") .map(|p| p.u32(0).unwrap() as usize) - .unwrap_or(1) } } diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index eb5c8417..18676de9 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -10,14 +10,14 @@ use abi::{ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{aarch64::cpu::Cpu, CpuMessage, PLATFORM}, + arch::{aarch64::cpu::Cpu, Architecture}, debug::LogLevel, - device::interrupt::IrqContext, - panic::panic_secondary, syscall::raw_syscall_handler, task::{context::TaskFrame, process::Process}, }; +use super::ARCHITECTURE; + /// Struct for register values saved when taking an exception #[repr(C)] pub struct ExceptionFrame { @@ -202,8 +202,19 @@ extern "C" fn __aa64_el0_serror_handler() { // EL1 #[no_mangle] -extern "C" fn __aa64_el1_sync_handler(_frame: *mut ExceptionFrame) { - todo!(); +extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) { + let frame = unsafe { &*frame }; + let esr_el1 = ESR_EL1.get(); + let ec = (esr_el1 >> 26) & 0x3F; + let iss = esr_el1 & 0x1FFFFFF; + + unsafe { + crate::sync::hack_locks(); + } + + dump_irrecoverable_exception(frame, ec, iss); + + panic!("Irrecoverable exception in kernel mode"); } #[no_mangle] @@ -270,10 +281,9 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { } fn irq_common() { - unsafe { - let ic = IrqContext::new(); - PLATFORM.gic.handle_pending_irqs(&ic); - } + ARCHITECTURE + .external_interrupt_controller() + .handle_pending_irqs(); } unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) { @@ -288,15 +298,4 @@ unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) { frame.restore(&saved_data.frame); } -pub(super) fn ipi_handler(msg: Option) { - if let Some(msg) = msg { - match msg { - CpuMessage::Panic => panic_secondary(), - } - } else { - warnln!("Spurious IPI received by cpu{}", Cpu::local_id()); - todo!(); - } -} - global_asm!(include_str!("vectors.S")); diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs index 4a591f67..cd55b3a6 100644 --- a/src/arch/aarch64/gic/gicc.rs +++ b/src/arch/aarch64/gic/gicc.rs @@ -5,7 +5,7 @@ use tock_registers::{ registers::ReadWrite, }; -use crate::{device::interrupt::IrqContext, mem::device::DeviceMemoryIo}; +use crate::mem::device::DeviceMemoryIo; register_bitfields! { u32, @@ -50,11 +50,11 @@ impl Gicc { self.regs.PMR.write(PMR::Priority.val(0xFF)); } - pub fn pending_irq_number<'irq>(&'irq self, _ic: &IrqContext<'irq>) -> usize { + pub fn pending_irq_number(&self) -> usize { self.regs.IAR.read(IAR::InterruptID) as usize } - pub fn clear_irq<'irq>(&'irq self, irq: usize, _ic: &IrqContext<'irq>) { + pub fn clear_irq(&self, irq: usize) { self.regs.EOIR.write(EOIR::EOINTID.val(irq as u32)); } } diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index 5ed010cd..373839b5 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -1,4 +1,5 @@ //! ARM GICv2 Distributor registers +use device_api::interrupt::IpiDeliveryTarget; use spinning_top::Spinlock; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -6,9 +7,7 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use crate::{device::interrupt::IpiDeliveryTarget, mem::device::DeviceMemoryIo}; - -use super::IrqNumber; +use crate::mem::device::DeviceMemoryIo; register_bitfields! { u32, @@ -100,11 +99,12 @@ impl Gicd { pub unsafe fn set_sgir(&self, target: IpiDeliveryTarget, interrupt_id: u64) { assert_eq!(interrupt_id & !0xF, 0); let value = match target { - IpiDeliveryTarget::AllExceptLocal => SGIR::TargetListFilter::AllExceptLocal, - IpiDeliveryTarget::Specified(_mask) => { + IpiDeliveryTarget::OtherCpus => SGIR::TargetListFilter::AllExceptLocal, + IpiDeliveryTarget::Specific(_mask) => { // TODO: need to handle self-ipi case, releasing the lock somehow todo!(); } + IpiDeliveryTarget::ThisCpu => todo!(), } + SGIR::INTID.val(interrupt_id as u32); self.shared_regs.lock().SGIR.write(value); @@ -114,7 +114,7 @@ impl Gicd { self.banked_regs.ITARGETSR[0].read(ITARGETSR::Offset0) } - fn enable_irq_inner(&self, irq: usize) { + pub fn enable_irq(&self, irq: usize) { let reg = irq >> 5; let bit = 1u32 << (irq & 0x1F); @@ -135,12 +135,6 @@ impl Gicd { } } - pub fn enable_irq(&self, irq: IrqNumber) { - let irq = irq.get(); - - self.enable_irq_inner(irq); - } - pub unsafe fn init(&self) { let mask = self.local_gic_target_mask(); let regs = self.shared_regs.lock(); diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index e9161f0e..19bf8837 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -1,20 +1,32 @@ //! ARM Generic Interrupt Controller v2 driver +use core::sync::atomic::Ordering; + +use aarch64_cpu::asm::barrier; use abi::error::Error; -use spinning_top::Spinlock; +use alloc::boxed::Box; +use device_api::{ + interrupt::{ + ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, + IpiDeliveryTarget, IrqNumber, IrqOptions, LocalInterruptController, + }, + Device, +}; use crate::{ - device::{ - interrupt::{ExternalInterruptController, InterruptSource, IrqContext}, - Device, InitializableDevice, + arch::{ + aarch64::devtree::{self, DevTreeIndexPropExt}, + Architecture, CpuMessage, }, + device_tree_driver, mem::device::{DeviceMemory, DeviceMemoryIo}, + sync::IrqSafeSpinlock, util::OneTimeInit, }; use self::{gicc::Gicc, gicd::Gicd}; -use super::{cpu::Cpu, exception::ipi_handler}; +use super::{cpu::Cpu, smp::CPU_COUNT, ARCHITECTURE}; const MAX_IRQ: usize = 300; const IPI_VECTOR: u64 = 1; @@ -22,49 +34,21 @@ const IPI_VECTOR: u64 = 1; pub mod gicc; pub mod gicd; -/// Wrapper type for ARM interrupt vector -#[repr(transparent)] -#[derive(Clone, Copy)] -pub struct IrqNumber(usize); - /// ARM Generic Interrupt Controller v2 pub struct Gic { gicc: OneTimeInit, gicd: OneTimeInit, gicd_base: usize, gicc_base: usize, - irq_table: Spinlock<[Option<&'static (dyn InterruptSource + Sync)>; MAX_IRQ]>, -} - -impl IrqNumber { - /// Returns the underlying vector number - #[inline(always)] - pub const fn get(self) -> usize { - self.0 - } - - /// Wraps the interrupt vector value in the [IrqNumber] type. - /// - /// # Panics - /// - /// Will panic if `v` is not a valid interrupt number. - #[inline(always)] - pub const fn new(v: usize) -> Self { - assert!(v < MAX_IRQ); - Self(v) - } + table: IrqSafeSpinlock>, } impl Device for Gic { - fn name(&self) -> &'static str { + fn display_name(&self) -> &'static str { "ARM Generic Interrupt Controller v2" } -} -impl InitializableDevice for Gic { - type Options = (); - - unsafe fn init(&self, _opts: ()) -> Result<(), Error> { + unsafe fn init(&'static self) -> Result<(), Error> { let gicd_mmio = DeviceMemory::map("GICv2 Distributor registers", self.gicd_base, 0x1000)?; let gicd_mmio_shared = DeviceMemoryIo::new(gicd_mmio.clone()); let gicd_mmio_banked = DeviceMemoryIo::new(gicd_mmio); @@ -79,66 +63,104 @@ impl InitializableDevice for Gic { self.gicd.init(gicd); self.gicc.init(gicc); + ARCHITECTURE.register_external_interrupt_controller(self)?; + ARCHITECTURE.register_local_interrupt_controller(self)?; + Ok(()) } } impl ExternalInterruptController for Gic { - type IrqNumber = IrqNumber; + fn register_irq( + &self, + irq: IrqNumber, + _options: IrqOptions, + handler: &'static dyn InterruptHandler, + ) -> Result<(), Error> { + let mut table = self.table.lock(); + + let index = match irq { + IrqNumber::Shared(i) => i + 32, + IrqNumber::Private(i) => i + 16, + } as usize; + + debugln!("Bound irq{} to {:?}", index, handler.display_name()); + table.insert(index, handler)?; - fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { - self.gicd.get().enable_irq(irq); Ok(()) } - fn register_handler( - &self, - irq: Self::IrqNumber, - handler: &'static (dyn InterruptSource + Sync), - ) -> Result<(), Error> { - let mut table = self.irq_table.lock(); - let irq = irq.get(); - if table[irq].is_some() { - return Err(Error::AlreadyExists); + fn enable_irq(&self, irq: IrqNumber) -> Result<(), Error> { + let gicd = self.gicd.get(); + let index = match irq { + IrqNumber::Shared(i) => i + 32, + IrqNumber::Private(i) => i + 16, + } as usize; + gicd.enable_irq(index); + Ok(()) + } + + fn handle_pending_irqs(&self) { + let gicc = self.gicc.get(); + let irq_number = gicc.pending_irq_number(); + if irq_number >= MAX_IRQ { + return; } - debugln!("Bound irq{} to {:?}", irq, Device::name(handler)); - table[irq] = Some(handler); + if irq_number == IPI_VECTOR as usize { + gicc.clear_irq(irq_number); + Cpu::local().handle_ipi(); + return; + } + + gicc.clear_irq(irq_number); + + { + let table = self.table.lock(); + match table.handler(irq_number) { + Some(handler) => { + drop(table); + handler.handle_irq(); + } + None => warnln!("No handler for irq{}", irq_number), + } + } + } +} + +impl LocalInterruptController for Gic { + type IpiMessage = CpuMessage; + + fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> { + // TODO message queue insertion should be moved + match target { + IpiDeliveryTarget::OtherCpus => { + let local = Cpu::local_id(); + for i in 0..CPU_COUNT.load(Ordering::Acquire) { + if i != local as usize { + Cpu::push_ipi_queue(i as u32, msg); + } + } + } + IpiDeliveryTarget::Specific(_) => todo!(), + IpiDeliveryTarget::ThisCpu => todo!(), + } + + // Issue a memory barrier + barrier::dsb(barrier::ISH); + barrier::isb(barrier::SY); + + unsafe { + self.gicd.get().set_sgir(target, IPI_VECTOR); + } Ok(()) } - fn configure_irq( - &self, - _irq: Self::IrqNumber, - _active_low: bool, - _level_triggered: bool, - ) -> Result<(), Error> { - todo!() + unsafe fn init_ap(&self) -> Result<(), Error> { + self.gicc.get().init(); + Ok(()) } - - // unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { - // // TODO message queue insertion should be moved - // match target { - // IpiDeliveryTarget::AllExceptLocal => { - // let local = Cpu::local_id(); - // for i in 0..CPU_COUNT.load(Ordering::Acquire) { - // if i != local as usize { - // Cpu::push_ipi_queue(i as u32, msg); - // } - // } - // } - // IpiDeliveryTarget::Specified(_) => todo!(), - // } - - // // Issue a memory barrier - // barrier::dsb(barrier::ISH); - // barrier::isb(barrier::SY); - - // self.gicd.get().set_sgir(target, IPI_VECTOR); - - // Ok(()) - // } } impl Gic { @@ -153,46 +175,19 @@ impl Gic { gicd: OneTimeInit::new(), gicd_base, gicc_base, - irq_table: Spinlock::new([None; MAX_IRQ]), - } - } - - /// Initializes GICv2 for an application processor. - /// - /// # Safety - /// - /// Must not be called more than once per each AP. Must not be called from BSP. - pub unsafe fn init_smp_ap(&self) -> Result<(), Error> { - self.gicc.get().init(); - Ok(()) - } - - /// Handles a single pending IRQ (if there're many, the CPU will just get here again) - pub fn handle_pending_irqs<'irq>(&'irq self, ic: &IrqContext<'irq>) { - let gicc = self.gicc.get(); - let irq_number = gicc.pending_irq_number(ic); - if irq_number >= MAX_IRQ { - return; - } - - gicc.clear_irq(irq_number, ic); - - if irq_number as u64 == IPI_VECTOR { - // IPI received - let msg = Cpu::local().get_ipi(); - ipi_handler(msg); - return; - } - - { - let table = self.irq_table.lock(); - match table[irq_number] { - None => panic!("No IRQ handler registered for irq{}", irq_number), - Some(handler) => { - drop(table); - handler.handle_irq().expect("IRQ handler failed"); - } - } + table: IrqSafeSpinlock::new(FixedInterruptTable::new()), } } } + +device_tree_driver! { + compatible: ["arm,cortex-a15-gic", "arm,gic-400"], + probe(dt) => { + let reg = devtree::find_prop(&dt.node, "reg")?; + + let (gicc_base, _) = reg.cell2_array_item(0, dt.address_cells, dt.size_cells)?; + let (gicd_base, _) = reg.cell2_array_item(1, dt.address_cells, dt.size_cells)?; + + Some(Box::new(unsafe { Gic::new(gicc_base as usize, gicd_base as usize) })) + } +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index e2c23c05..534ff6ef 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -2,18 +2,27 @@ use core::sync::atomic::Ordering; -use aarch64_cpu::registers::{DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1}; +use aarch64_cpu::registers::{ + CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1, +}; use abi::error::Error; -use cfg_if::cfg_if; +use device_api::{ + interrupt::{ + ExternalInterruptController, IpiDeliveryTarget, IrqNumber, LocalInterruptController, + }, + timer::MonotonicTimestampProviderDevice, +}; +use fdt_rs::prelude::PropReader; +use git_version::git_version; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ arch::{ - aarch64::{cpu::Cpu, devtree::FdtMemoryRegionIter, smp::CPU_COUNT}, + aarch64::{cpu::Cpu, devtree::FdtMemoryRegionIter}, Architecture, }, - debug, - device::platform::Platform, + debug::{self}, + device::{self, power::arm_psci::Psci, DevTreeNodeInfo}, fs::{devfs, Initrd, INITRD_DATA}, mem::{ heap, @@ -25,23 +34,13 @@ use crate::{ }; use self::{ + boot::CPU_INIT_FENCE, devtree::{DevTreeIndexPropExt, DeviceTree}, + smp::CPU_COUNT, table::{init_fixed_tables, KERNEL_TABLES}, }; -pub mod plat_qemu; - -cfg_if! { - if #[cfg(feature = "aarch64_qemu")] { - pub use plat_qemu::{PlatformImpl, PLATFORM}; - } else if #[cfg(feature = "aarch64_orange_pi3")] { - pub mod plat_orange_pi3; - - pub use plat_orange_pi3::{PlatformImpl, PLATFORM}; - } else { - compile_error!("No platform selected for AArch64"); - } -} +use super::CpuMessage; pub mod boot; pub mod context; @@ -64,11 +63,24 @@ struct KernelStack { /// AArch64 platform interface pub struct AArch64 { dt: OneTimeInit>, + ext_intc: OneTimeInit<&'static dyn ExternalInterruptController>, + local_intc: OneTimeInit<&'static dyn LocalInterruptController>, + mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, + + // ARM-only devices + /// ARM PSCI instance on this system (there may not be one) + pub psci: OneTimeInit<&'static Psci>, } /// Global platform handle pub static ARCHITECTURE: AArch64 = AArch64 { dt: OneTimeInit::new(), + ext_intc: OneTimeInit::new(), + local_intc: OneTimeInit::new(), + mtimer: OneTimeInit::new(), + + // ARM-only devices + psci: OneTimeInit::new(), }; impl Architecture for AArch64 { @@ -123,6 +135,56 @@ impl Architecture for AArch64 { fn cpu_count() -> usize { CPU_COUNT.load(Ordering::Acquire) } + + fn register_external_interrupt_controller( + &self, + intc: &'static dyn ExternalInterruptController, + ) -> Result<(), Error> { + self.ext_intc.init(intc); + Ok(()) + } + + fn register_local_interrupt_controller( + &self, + intc: &'static dyn LocalInterruptController, + ) -> Result<(), Error> { + self.local_intc.init(intc); + Ok(()) + } + + fn register_monotonic_timer( + &self, + timer: &'static dyn MonotonicTimestampProviderDevice, + ) -> Result<(), Error> { + self.mtimer.init(timer); + Ok(()) + } + + fn external_interrupt_controller(&self) -> &'static dyn ExternalInterruptController { + *self.ext_intc.get() + } + + fn local_interrupt_controller( + &self, + ) -> &'static dyn LocalInterruptController { + *self.local_intc.get() + } + + fn monotonic_timer(&self) -> &'static dyn MonotonicTimestampProviderDevice { + *self.mtimer.get() + } + + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { + if let Some(local_intc) = self.local_intc.try_get() { + local_intc.send_ipi(target, msg) + } else { + Ok(()) + } + } + + unsafe fn reset(&self) -> ! { + todo!() + } } impl AArch64 { @@ -170,6 +232,109 @@ impl AArch64 { phys::init_from_iter(regions) } + + fn chosen_stdout_path<'a>(dt: &'a DeviceTree) -> Option<&'a str> { + let chosen = dt.node_by_path("/chosen")?; + let prop = devtree::find_prop(&chosen, "stdout-path")?; + prop.str().ok() + } + + fn init_platform(&self, bsp: bool) { + if bsp { + let dt = self.device_tree(); + + let address_cells = dt.address_cells(); + let size_cells = dt.size_cells(); + + let chosen_stdout_path = Self::chosen_stdout_path(dt); + let chosen_stdout = chosen_stdout_path.and_then(|path| dt.node_by_path(path)); + + // Probe and initialize the /chosen.stdout-path device first + if let Some(node) = chosen_stdout.clone() { + let probe = DevTreeNodeInfo { + address_cells, + size_cells, + node, + }; + + if let Some((device, _)) = device::probe_dt_node(&probe) { + unsafe { + device.init().unwrap(); + } + } + }; + + debug::reset(); + + // Print some stuff now that the output is initialized + infoln!( + "Yggdrasil v{} ({})", + env!("CARGO_PKG_VERSION"), + git_version!() + ); + infoln!("Initializing aarch64 platform"); + + // Probe and initialize the rest of devices + let nodes = dt.root().children(); + if let Err(error) = device::enumerate_dt( + address_cells, + size_cells, + nodes, + |_, probe| { + // Ignore /chosen/stdout-path node + if let Some(ref chosen_stdout) = chosen_stdout && chosen_stdout.name() == probe.node.name() { + return Ok(()); + } + + if let Some((device, _)) = device::probe_dt_node(&probe) { + unsafe { + device.init()?; + } + } + + Ok(()) + }, + ) { + warnln!( + "{} errors encountered when initializing platform devices", + error + ); + } + + // Initialize IRQs for the devices + device::manager_lock().devices().for_each(|dev| unsafe { + if let Err(error) = dev.init_irq() { + errorln!( + "Could not init interrupts for {:?}: {:?}", + dev.display_name(), + error + ); + } + }); + + // Print the device list + infoln!("Enumerated devices:"); + device::manager_lock().devices().for_each(|dev| { + infoln!("* {:?}", dev.display_name()); + }); + } else { + // BSP already initialized everything needed + // Setup timer and local interrupt controller + let intc = self.local_intc.get(); + + unsafe { + intc.init_ap().unwrap(); + } + + // TODO device-tree initialization for this + CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); + CNTP_TVAL_EL0.set(10000000); + self.ext_intc + .get() + .enable_irq(IrqNumber::Private(14)) + .unwrap(); + } + } } fn setup_initrd() { @@ -228,21 +393,15 @@ pub fn kernel_main(dtb_phys: usize) -> ! { TTBR0_EL1.set(0); unsafe { + ARCHITECTURE.init_device_tree(dtb_phys); + AArch64::set_interrupt_mask(true); - PLATFORM.init_debug(); - ARCHITECTURE.init_device_tree(dtb_phys); - } - debug::reset(); + exception::init_exceptions(); - exception::init_exceptions(); + // Setup initrd + setup_initrd(); - // Setup initrd - setup_initrd(); - - debugln!("Initializing {} platform", PLATFORM.name()); - - unsafe { ARCHITECTURE .init_physical_memory(dtb_phys) .expect("Failed to initialize the physical memory manager"); @@ -252,27 +411,32 @@ pub fn kernel_main(dtb_phys: usize) -> ! { .expect("Could not allocate a block for heap"); heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + devfs::init(); + + // Enumerate the device tree + ARCHITECTURE.init_platform(true); + + debugln!("Heap: {:#x?}", heap::heap_range()); + Cpu::init_local(); - devfs::init(); - PLATFORM.init(true).unwrap(); + let dt = ARCHITECTURE.dt.get(); + if let Err(e) = smp::start_ap_cores(dt) { + errorln!( + "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", + e + ); + } - // let dt = ARCHITECTURE.dt.get(); - // if let Err(e) = smp::start_ap_cores(dt) { - // errorln!( - // "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", - // e - // ); - // } + Cpu::init_ipi_queues(); - // Cpu::init_ipi_queues(); - - // CPU_INIT_FENCE.signal(); - // CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); + CPU_INIT_FENCE.signal(); + CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); task::init().expect("Failed to initialize the scheduler"); - // Initialize and enter the scheduler + infoln!("All cpus ready"); + task::enter(); } } diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs index 7135a4ea..eb89a64d 100644 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ b/src/arch/aarch64/plat_qemu/mod.rs @@ -35,29 +35,31 @@ impl Platform for PlatformImpl { const KERNEL_PHYS_BASE: usize = 0x40080000; unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { - if is_bsp { - self.gic.init(())?; + loop {} + // if is_bsp { + // self.gic.init(())?; - self.pl011.init_irq()?; - devfs::add_char_device(&self.pl011, CharDeviceType::TtySerial)?; + // self.pl011.init_irq()?; + // devfs::add_char_device(&self.pl011, CharDeviceType::TtySerial)?; - self.local_timer.init(())?; - self.local_timer.init_irq()?; - } else { - self.gic.init_smp_ap()?; + // self.local_timer.init(())?; + // self.local_timer.init_irq()?; + // } else { + // self.gic.init_smp_ap()?; - // TODO somehow merge this with the rest of the code - CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); - CNTP_TVAL_EL0.set(10000000); - self.gic.enable_irq(IrqNumber::new(30))?; - } + // // TODO somehow merge this with the rest of the code + // CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); + // CNTP_TVAL_EL0.set(10000000); + // self.gic.enable_irq(IrqNumber::new(30))?; + // } - Ok(()) + // Ok(()) } unsafe fn init_debug(&'static self) { - self.pl011.init(()).ok(); - debug::add_sink(&self.pl011, LogLevel::Debug); + loop {} + // self.pl011.init(()).ok(); + // debug::add_sink(&self.pl011, LogLevel::Debug); } fn interrupt_controller( diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index 4e63d68e..36366aed 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -1,60 +1,79 @@ //! Simultaneous multiprocessing support for aarch64 -use core::{ - arch::asm, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicUsize, Ordering}; use abi::error::Error; +use device_api::CpuBringupDevice; use fdt_rs::prelude::PropReader; use crate::{ absolute_address, - arch::aarch64::boot::__aarch64_ap_lower_entry, + arch::ARCHITECTURE, mem::{ phys::{self, PageUsage}, - ConvertAddress, KERNEL_VIRT_OFFSET, + ConvertAddress, }, }; -use super::devtree::{self, DeviceTree}; +use super::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; -/// ARM Power State Coordination Interface -pub struct Psci {} +#[derive(Debug)] +enum CpuEnableMethod { + Psci, + // Not currently supported + #[allow(dead_code)] + SpinTable { + release_addr: usize, + }, +} + +struct CpuInfo<'a> { + id: u32, + compatible: &'a str, + enable_method: CpuEnableMethod, +} + +fn enumerate_cpus<'a>(dt: &'a DeviceTree) -> impl Iterator> { + let cpus = dt.node_by_path("/cpus").unwrap(); + + cpus.children().filter_map(|cpu_node| { + let compatible = devtree::find_prop(&cpu_node, "compatible").and_then(|p| p.str().ok())?; + let id = cpu_node.prop("reg")?; // devtree::find_prop(&cpu_node, "reg").and_then(|p| p.u32(0).ok())?; + let enable_method_str: &str = cpu_node.prop("enable-method")?; + let enable_method = match enable_method_str { + "psci" => CpuEnableMethod::Psci, + _ => todo!(), + }; + + Some(CpuInfo { + id, + compatible, + enable_method, + }) + }) +} + +impl CpuEnableMethod { + unsafe fn start_cpu(&self, id: usize, ip: usize, sp: usize) -> Result<(), Error> { + match self { + Self::Psci => { + let psci = ARCHITECTURE.psci.try_get().ok_or_else(|| { + warnln!( + "cpu{} has to be enabled through PSCI, but no PSCI found", + id + ); + Error::InvalidArgument + })?; + + psci.start_cpu(id, ip, sp) + } + _ => todo!(), + } + } +} /// Number of online CPUs, initially set to 1 (BSP processor is up) pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); -impl Psci { - /// Function ID for CPU startup request - const CPU_ON: u32 = 0xC4000003; - - /// Constructs an interface instance for PSCI - pub const fn new() -> Self { - Self {} - } - - #[inline] - unsafe fn call(&self, mut x0: u64, x1: u64, x2: u64, x3: u64) -> u64 { - asm!("hvc #0", inout("x0") x0, in("x1") x1, in("x2") x2, in("x3") x3); - x0 - } - - /// Enables a single processor through a hvc/svc call. - /// - /// # Safety - /// - /// Calling this outside of initialization sequence or more than once may lead to unexpected - /// behavior. - pub unsafe fn cpu_on(&self, target_cpu: usize, entry_point_address: usize, context_id: usize) { - self.call( - Self::CPU_ON as _, - target_cpu as _, - entry_point_address as _, - context_id as _, - ); - } -} - /// Starts application processors using the method specified in the device tree. /// /// TODO: currently does not handle systems where APs are already started before entry. @@ -64,68 +83,42 @@ impl Psci { /// The caller must ensure the physical memory manager was initialized, virtual memory tables are /// set up and the function has not been called before. pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> { - let cpus = dt.node_by_path("/cpus").unwrap(); - let psci = Psci::new(); - - for cpu in cpus.children() { - let Some(compatible) = devtree::find_prop(&cpu, "compatible") else { - continue; - }; - let Ok(compatible) = compatible.str() else { - continue; - }; - if !compatible.starts_with("arm,cortex-a") { - continue; - } - - let reg = devtree::find_prop(&cpu, "reg").unwrap().u32(0).unwrap(); - if reg == 0 { - continue; - } + extern "C" { + fn __aarch64_ap_entry(); + } + for cpu in enumerate_cpus(dt).filter(|cpu| cpu.id != 0) { debugln!( - "Will start {}, compatible={:?}, reg={}", - cpu.name().unwrap(), - compatible, - reg + "cpu{}: enable-method={:?}, compatible={:?}", + cpu.id, + cpu.enable_method, + cpu.compatible ); const AP_STACK_PAGES: usize = 4; let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used)?; debugln!( - "{} stack: {:#x}..{:#x}", - cpu.name().unwrap(), + "cpu{} stack: {:#x}..{:#x}", + cpu.id, stack_pages, stack_pages + AP_STACK_PAGES * 0x1000 ); // Wait for the CPU to come up let old_count = CPU_COUNT.load(Ordering::Acquire); - psci.cpu_on( - reg as usize, - absolute_address!(__aarch64_ap_entry).physicalize(), - stack_pages + AP_STACK_PAGES * 0x1000, - ); + let ip = absolute_address!(__aarch64_ap_entry).physicalize(); + let sp = stack_pages + AP_STACK_PAGES * 0x1000; + if let Err(error) = cpu.enable_method.start_cpu(cpu.id as usize, ip, sp) { + errorln!("Couldn't start cpu{} up: {:?}", cpu.id, error); + continue; + } while CPU_COUNT.load(Ordering::Acquire) == old_count { aarch64_cpu::asm::wfe(); } - debugln!("{} is up", cpu.name().unwrap()); + debugln!("cpu{} is up", cpu.id); } Ok(()) } - -#[naked] -unsafe extern "C" fn __aarch64_ap_entry() -> ! { - asm!( - r#" - mov sp, x0 - bl {entry} - {kernel_virt_offset} - "#, - entry = sym __aarch64_ap_lower_entry, - kernel_virt_offset = const KERNEL_VIRT_OFFSET, - options(noreturn) - ); -} diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index 01e6156a..a9e8105f 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -362,7 +362,10 @@ impl VirtualMemoryManager for AddressSpace { for page in (addr..addr + len).step_by(0x1000) { let Some(phys) = self.translate(page) else { - todo!(); + todo!( + "Tried to deallocate address not present in the table: {:#x}", + addr + ); }; self.write_entry(page, PageEntry::INVALID, true)?; diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 62b69964..fc00a0c7 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -4,14 +4,17 @@ use core::time::Duration; use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; +use alloc::boxed::Box; +use device_api::{ + interrupt::{InterruptHandler, IrqNumber}, + timer::MonotonicTimestampProviderDevice, + Device, +}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::{aarch64::gic::IrqNumber, PLATFORM}, - device::{ - interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device, - InitializableDevice, - }, + arch::{Architecture, ARCHITECTURE}, + device_tree_driver, proc::wait, task::tasklet, }; @@ -26,23 +29,24 @@ pub struct ArmTimer { /// ARM timer tick interval (in some time units?) pub const TICK_INTERVAL: u64 = 1000000; -impl Device for ArmTimer { - fn name(&self) -> &'static str { - "ARM Generic Timer" +impl InterruptHandler for ArmTimer { + fn handle_irq(&self) -> bool { + CNTP_TVAL_EL0.set(TICK_INTERVAL); + let now = self.monotonic_timestamp().unwrap(); + + wait::tick(now); + tasklet::tick(now); + + unsafe { + Cpu::local().queue().yield_cpu(); + } + + true } } -impl InitializableDevice for ArmTimer { - type Options = (); - - unsafe fn init(&self, _opts: ()) -> Result<(), Error> { - CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); - Ok(()) - } -} - -impl TimestampSource for ArmTimer { - fn timestamp(&self) -> Result { +impl MonotonicTimestampProviderDevice for ArmTimer { + fn monotonic_timestamp(&self) -> Result { let count = CNTPCT_EL0.get() * 1_000_000; let freq = CNTFRQ_EL0.get(); @@ -50,33 +54,61 @@ impl TimestampSource for ArmTimer { } } -impl InterruptSource for ArmTimer { - fn handle_irq(&self) -> Result { - CNTP_TVAL_EL0.set(TICK_INTERVAL); - let t = self.timestamp()?; +impl Device for ArmTimer { + fn display_name(&self) -> &'static str { + "ARM Generic Timer" + } - wait::tick(t); - tasklet::tick(t); - - unsafe { - Cpu::local().queue().yield_cpu(); - } - - Ok(true) + unsafe fn init(&'static self) -> Result<(), Error> { + CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); + ARCHITECTURE.register_monotonic_timer(self)?; + Ok(()) } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.interrupt_controller(); + let intc = ARCHITECTURE.external_interrupt_controller(); + + intc.register_irq(self.irq, Default::default(), self)?; - intc.register_handler(self.irq, self)?; CNTP_CTL_EL0.modify(CNTP_CTL_EL0::IMASK::CLEAR); CNTP_TVAL_EL0.set(TICK_INTERVAL); + intc.enable_irq(self.irq)?; Ok(()) } } +// impl TimestampSource for ArmTimer { +// fn timestamp(&self) -> Result { +// } +// } + +// impl InterruptSource for ArmTimer { +// fn handle_irq(&self) -> Result { +// CNTP_TVAL_EL0.set(TICK_INTERVAL); +// let t = self.timestamp()?; +// +// wait::tick(t); +// tasklet::tick(t); +// +// unsafe { +// Cpu::local().queue().yield_cpu(); +// } +// +// Ok(true) +// } +// +// unsafe fn init_irq(&'static self) -> Result<(), Error> { +// let intc = PLATFORM.interrupt_controller(); +// +// intc.register_handler(self.irq, self)?; +// intc.enable_irq(self.irq)?; +// +// Ok(()) +// } +// } + impl ArmTimer { /// Constructs an instance of ARM generic timer. /// @@ -87,3 +119,11 @@ impl ArmTimer { Self { irq } } } + +device_tree_driver! { + compatible: ["arm,armv8-timer"], + probe(_dt) => { + // TODO actually get info from the dt + Some(Box::new(unsafe { ArmTimer::new(IrqNumber::Private(14)) })) + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 82a08831..a3b0f803 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,7 +1,6 @@ //! Provides architecture/platform-specific implementation details use abi::error::Error; -use cfg_if::cfg_if; /// Returns an absolute address to the given symbol #[macro_export] @@ -21,23 +20,29 @@ macro_rules! absolute_address { } pub mod aarch64; -cfg_if! { - if #[cfg(target_arch = "aarch64")] { - pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE, PlatformImpl, PLATFORM}; - } else if #[cfg(target_arch = "x86_64")] { - pub mod x86_64; - - pub use x86_64::{ - X86_64 as ArchitectureImpl, - X86_64 as PlatformImpl, - ARCHITECTURE, - ARCHITECTURE as PLATFORM - }; - } else { - compile_error!("Architecture is not supported"); - } -} +pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; +use device_api::{ + interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, + timer::MonotonicTimestampProviderDevice, +}; +// cfg_if! { +// if #[cfg(target_arch = "aarch64")] { +// +// pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE, PlatformImpl, PLATFORM}; +// } else if #[cfg(target_arch = "x86_64")] { +// pub mod x86_64; +// +// pub use x86_64::{ +// X86_64 as ArchitectureImpl, +// X86_64 as PlatformImpl, +// ARCHITECTURE, +// ARCHITECTURE as PLATFORM +// }; +// } else { +// compile_error!("Architecture is not supported"); +// } +// } /// Describes messages sent from some CPU to others #[derive(Clone, Copy, PartialEq, Debug)] @@ -82,4 +87,54 @@ pub trait Architecture { /// Returns the count of present CPUs, including the BSP fn cpu_count() -> usize; + + /// Adds an external interrupt controller to the system + fn register_external_interrupt_controller( + &self, + intc: &'static dyn ExternalInterruptController, + ) -> Result<(), Error>; + + /// Adds a local interrupt controller to the system + fn register_local_interrupt_controller( + &self, + intc: &'static dyn LocalInterruptController, + ) -> Result<(), Error>; + + /// Adds a monotonic timer to the system + fn register_monotonic_timer( + &self, + timer: &'static dyn MonotonicTimestampProviderDevice, + ) -> Result<(), Error>; + + // TODO only supports 1 extintc per system + /// Returns the primary external interrupt controller + fn external_interrupt_controller(&self) -> &'static dyn ExternalInterruptController; + + /// Returns the local interrupt controller + fn local_interrupt_controller( + &self, + ) -> &'static dyn LocalInterruptController; + + /// Returns the monotonic timer + fn monotonic_timer(&self) -> &'static dyn MonotonicTimestampProviderDevice; + + /// Sends a message to the requested set of CPUs through an interprocessor interrupt. + /// + /// # Note + /// + /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations + /// may impose narrower restrictions. + /// + /// # Safety + /// + /// As the call may alter the flow of execution on CPUs, this function is unsafe. + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; + + /// Performs a CPU reset. + /// + /// # Safety + /// + /// The caller must ensure it is actually safe to reset, i.e. no critical processes will be + /// aborted and no data will be lost. + unsafe fn reset(&self) -> !; } diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index cbaf5c22..f28ce514 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -20,7 +20,7 @@ use crate::{ task, }; -use super::{smp::CPU_COUNT, ARCHITECTURE}; +use super::ARCHITECTURE; const BOOT_STACK_SIZE: usize = 65536; @@ -81,39 +81,40 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { /// Application processor entry point pub extern "C" fn __x86_64_ap_entry() -> ! { - let cpu_id = CPU_COUNT.load(Ordering::Acquire); + loop {} + // let cpu_id = CPU_COUNT.load(Ordering::Acquire); - MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); - unsafe { - core::arch::asm!("swapgs"); - } - MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); - unsafe { - core::arch::asm!("swapgs"); - } + // MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + // unsafe { + // core::arch::asm!("swapgs"); + // } + // MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + // unsafe { + // core::arch::asm!("swapgs"); + // } - // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base - cpuid::feature_gate(); + // // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base + // cpuid::feature_gate(); - infoln!("cpu{} initializing", cpu_id); - unsafe { - ARCHITECTURE.init_mmu(false); - core::arch::asm!("wbinvd"); + // infoln!("cpu{} initializing", cpu_id); + // unsafe { + // ARCHITECTURE.init_mmu(false); + // core::arch::asm!("wbinvd"); - Cpu::init_local(LocalApic::new(), cpu_id as u32); - syscall::init_syscall(); - exception::init_exceptions(cpu_id); + // Cpu::init_local(LocalApic::new(), cpu_id as u32); + // syscall::init_syscall(); + // exception::init_exceptions(cpu_id); - ARCHITECTURE.init(false).unwrap(); - } + // ARCHITECTURE.init(false).unwrap(); + // } - CPU_COUNT.fetch_add(1, Ordering::Release); + // CPU_COUNT.fetch_add(1, Ordering::Release); - CPU_INIT_FENCE.wait_one(); + // CPU_INIT_FENCE.wait_one(); - infoln!("cpu{} initialized", cpu_id); + // infoln!("cpu{} initialized", cpu_id); - unsafe { task::enter() } + // unsafe { task::enter() } } global_asm!( diff --git a/src/device/bus/mod.rs b/src/device/bus/mod.rs new file mode 100644 index 00000000..00a9c9d6 --- /dev/null +++ b/src/device/bus/mod.rs @@ -0,0 +1,3 @@ +//! Bus devices + +pub mod simple_bus; diff --git a/src/device/bus/simple_bus.rs b/src/device/bus/simple_bus.rs new file mode 100644 index 00000000..a5782d31 --- /dev/null +++ b/src/device/bus/simple_bus.rs @@ -0,0 +1,27 @@ +//! Simple "passthrough" bus device + +use crate::{arch::aarch64::devtree::DevTreeIndexNodeExt, device, device_tree_driver}; + +device_tree_driver! { + compatible: ["simple-bus"], + probe(dt) => { + let address_cells = dt.node.address_cells(); + let size_cells = dt.node.size_cells(); + + let nodes = dt.node.children(); + + // Iterate devices on the bus + device::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { + if let Some((device, _)) = device::probe_dt_node(&probe) { + unsafe { + device.init()?; + } + } + + Ok(()) + }).ok(); + + // Don't yield any devices + None + } +} diff --git a/src/device/interrupt.rs b/src/device/interrupt.rs deleted file mode 100644 index b361e267..00000000 --- a/src/device/interrupt.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Interrupt-related interfaces -use core::marker::PhantomData; - -use abi::error::Error; - -use super::Device; - -/// Convenience type alias for a static IRQ handler -pub type IrqHandler = &'static (dyn InterruptSource + Sync); - -/// Specifies the target(s) of interprocessor interrupt delivery -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum IpiDeliveryTarget { - /// IPI will be delivered to every CPU except the local one - AllExceptLocal, - /// IPI will only be sent to CPUs specified in the mask - Specified(u64), -} - -/// Interface for a device capable of emitting interrupts -pub trait InterruptSource: Device { - /// Initializes and enables IRQs for the device. - /// - /// # Safety - /// - /// The caller must ensure the function hasn't been called before. - unsafe fn init_irq(&'static self) -> Result<(), Error>; - - /// Handles the interrupt raised by the device. Returns `true` if the interrupt was handled, - /// `false` if device itself sent no interrupt (may be possible if several interrupt handlers - /// share the same IRQ vector). - fn handle_irq(&self) -> Result; -} - -// TODO not yet ready for this -// /// Controls per-CPU interrupt delivery -// pub trait LocalInterruptController { -// type Vector; -// type Id; -// -// /// Allocates a vector for external IRQ -// unsafe fn allocate_irq_vector(&self) -> Result; -// -// /// Returns the unique ID of this local interrupt controller -// fn id(&self) -> Self::Id; -// -// } - -/// Interface for a device responsible for routing and handling IRQs from external sources -pub trait ExternalInterruptController: Device { - /// Interrupt number wrapper type - type IrqNumber; - - /// Binds an interrupt number to its handler implementation - fn register_handler( - &self, - irq: Self::IrqNumber, - handler: &'static (dyn InterruptSource + Sync), - ) -> Result<(), Error>; - - /// Configures how an interrupt should be handled by the controller - fn configure_irq( - &self, - irq: Self::IrqNumber, - active_low: bool, - level_triggered: bool, - ) -> Result<(), Error>; - - /// Enables given interrupt number/vector - fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error>; -} - -/// Token type to indicate that the code is being run from an interrupt handler -pub struct IrqContext<'irq> { - _0: PhantomData<&'irq ()>, -} - -impl<'irq> IrqContext<'irq> { - /// Constructs an IRQ context token - /// - /// # Safety - /// - /// Only allowed to be constructed in top-level IRQ handlers - #[inline(always)] - pub const unsafe fn new() -> Self { - Self { _0: PhantomData } - } -} diff --git a/src/device/mod.rs b/src/device/mod.rs index b0eee8b7..d7722ae7 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,31 +1,185 @@ //! Device management and interfaces -use abi::error::Error; +use core::mem::size_of; -#[cfg(feature = "fb_console")] -pub mod display; -pub mod input; -pub mod interrupt; -pub mod platform; +use abi::error::Error; +use alloc::boxed::Box; +use device_api::{manager::DeviceManager, Device, DeviceId}; +use fdt_rs::{index::DevTreeIndexNode, prelude::PropReader}; + +use crate::{ + arch::aarch64::devtree, + sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, +}; + +pub mod bus; +pub mod power; pub mod serial; -pub mod timer; pub mod tty; -/// General device interface -pub trait Device { - /// Returns a display name for the device - fn name(&self) -> &'static str; +// pub mod display; + +static DEVICE_MANAGER: IrqSafeSpinlock = IrqSafeSpinlock::new(DeviceManager::new()); + +/// Helper macro to return the count of expressions supplied to it +#[macro_export] +macro_rules! count { + () => (0usize); + ($x:tt $($xs:tt)*) => (1usize + $crate::count!($($xs)*)); } -/// Interface for device initialization -pub trait InitializableDevice { - /// Options provided when initializing the device - type Options; +/// Registers a device driver for compatible device tree nodes +/// +/// # Usage example +/// +/// ``` +/// device_tree_driver! { +/// compatible: ["arm,pl011"], +/// probe(of) => { +/// let my_device = ...; // ... extract some info about the device ... +/// Some(Box::new(my_device)) +/// } +/// } +/// ``` +#[macro_export] +macro_rules! device_tree_driver { + ( + compatible: [$($compatible:literal),+], + probe ($node:ident) => $probe_body:block $(,)? + ) => { + const __COMPATIBLE_LEN: usize = $crate::count!($($compatible )+); + static __COMPATIBLE: [&str; __COMPATIBLE_LEN] = [$($compatible),+]; - /// Initializes the device. - /// - /// # Safety - /// - /// Unsafe: only meant to be called once for each device. - unsafe fn init(&self, opts: Self::Options) -> Result<(), Error>; + fn __probe($node: &$crate::device::DevTreeNodeInfo) -> + Option> $probe_body + + core::arch::global_asm!(r#" + .pushsection .dt_probes, "a" + .quad {compatible} + .quad {compatible_len} + .quad {probe_func} + .popsection + "#, + compatible = sym __COMPATIBLE, + compatible_len = const __COMPATIBLE_LEN, + probe_func = sym __probe + ); + }; +} + +struct DevTreeProbe<'a> { + compatible: &'static [&'static str], + probe_func: fn(&'a DevTreeNodeInfo<'a, 'a, 'a>) -> Option>, +} + +/// Provides information about a device tree node to device driver's "probe" function +pub struct DevTreeNodeInfo<'a, 'i, 'dt> { + /// #address-cells property of the parent bus/system + pub address_cells: usize, + /// #size-cells property of the parent bus/system + pub size_cells: usize, + /// Device tree node being probed + pub node: DevTreeIndexNode<'a, 'i, 'dt>, +} + +fn iter_dt_probes<'a>() -> impl Iterator> { + extern "C" { + static __dt_probes_start: u64; + static __dt_probes_end: u64; + } + + unsafe { + let base = &__dt_probes_start as *const u64; + let end = &__dt_probes_end as *const u64; + let len = (end as usize - base as usize) / (size_of::() * 3); + + (0..len).map(move |i| { + let compatible_ptr = *base.add(i * 3); + let compatible_len = *base.add(i * 3 + 1); + let probe_func_ptr = *base.add(i * 3 + 2); + + let compatible = + core::slice::from_raw_parts(compatible_ptr as *const &str, compatible_len as usize); + let probe_func = core::mem::transmute(probe_func_ptr); + + DevTreeProbe { + compatible, + probe_func, + } + }) + } +} + +fn dt_match_compatible(compatible: &str) -> Option { + iter_dt_probes().find(|probe| probe.compatible.contains(&compatible)) +} + +/// "Probes" a device tree node for any matching device, registering it if a compatible driver is +/// found +pub fn probe_dt_node(dt: &DevTreeNodeInfo) -> Option<(&'static dyn Device, DeviceId)> { + // TODO use list, not just the first item + let Some(compatible) = + devtree::find_prop(&dt.node, "compatible").and_then(|prop| prop.str().ok()) + else { + return None; + }; + + let probe = dt_match_compatible(compatible)?; + let device = Box::leak((probe.probe_func)(dt)?); + let id = register_device(device); + Some((device, id)) +} + +/// Performs shallow walk of a device tree node and executes the visitor function on each node +pub fn enumerate_dt< + 'a, + I: Iterator>, + F: Fn(&str, DevTreeNodeInfo) -> Result<(), Error>, +>( + address_cells: usize, + size_cells: usize, + nodes: I, + f: F, +) -> Result<(), usize> { + let mut failed_count = 0; + + for node in nodes { + // Skip /cpus and /memory* + let probe = DevTreeNodeInfo { + address_cells, + size_cells, + node, + }; + + let Ok(name) = probe.node.name() else { + continue; + }; + let Some(compatible) = + devtree::find_prop(&probe.node, "compatible").and_then(|prop| prop.str().ok()) + else { + continue; + }; + + if let Err(error) = f(compatible, probe) { + warnln!("{}: {:?}", name, error); + failed_count += 1; + } + } + + if failed_count == 0 { + Ok(()) + } else { + Err(failed_count) + } +} + +/// Adds a device to the kernel's device table and returns the ID assigned to it +pub fn register_device(device: &'static dyn Device) -> DeviceId { + debugln!("Register {:?}", device.display_name()); + DEVICE_MANAGER.lock().register(device) +} + +/// Returns a safe reference to the kernel's [DeviceManager] instance +pub fn manager_lock<'a>() -> IrqSafeSpinlockGuard<'a, DeviceManager> { + DEVICE_MANAGER.lock() } diff --git a/src/device/platform.rs b/src/device/platform.rs index 2d59fd76..414c99ec 100644 --- a/src/device/platform.rs +++ b/src/device/platform.rs @@ -11,61 +11,10 @@ use super::{ /// Platform interface for interacting with a general hardware set pub trait Platform { - /// Interrupt number type for the platform - type IrqNumber; - - /// Address, to which the kernel is expected to be loaded for this platform - const KERNEL_PHYS_BASE: usize; - - /// Initializes the platform devices to their usable state. - /// - /// # Safety - /// - /// Unsafe to call if the platform has already been initialized. - unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error>; - - /// Initializes the debug devices to provide early debug output. - /// - /// # Safety - /// - /// Unsafe to call if the device has already been initialized. - unsafe fn init_debug(&'static self); - - /// Returns a display name for the platform - fn name(&self) -> &'static str; - - /// Returns a reference to the platform's interrupt controller. - /// - /// # Note - /// - /// May not be initialized at the moment of calling. - fn interrupt_controller(&self) - -> &dyn ExternalInterruptController; - /// Returns the platform's primary timestamp source. /// /// # Note /// /// May not be initialized at the moment of calling. fn timestamp_source(&self) -> &dyn TimestampSource; - - /// Sends a message to the requested set of CPUs through an interprocessor interrupt. - /// - /// # Note - /// - /// u64 limits the number of targetable CPUs to (only) 64. Platform-specific implementations - /// may impose narrower restrictions. - /// - /// # Safety - /// - /// As the call may alter the flow of execution on CPUs, this function is unsafe. - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; - - /// Performs a CPU reset. - /// - /// # Safety - /// - /// The caller must ensure it is actually safe to reset, i.e. no critical processes will be - /// aborted and no data will be lost. - unsafe fn reset(&self) -> !; } diff --git a/src/device/power/arm_psci.rs b/src/device/power/arm_psci.rs new file mode 100644 index 00000000..05f31d3c --- /dev/null +++ b/src/device/power/arm_psci.rs @@ -0,0 +1,81 @@ +//! ARM PSCI driver implementation + +use abi::error::Error; +use alloc::boxed::Box; +use device_api::{CpuBringupDevice, Device}; +use fdt_rs::prelude::PropReader; + +use crate::{ + arch::{aarch64::devtree, ARCHITECTURE}, + device_tree_driver, +}; + +enum CallMethod { + Hvc, + Smc, +} + +/// ARM Power State Coordination Interface driver +pub struct Psci { + method: CallMethod, + cpu_on: u32, + #[allow(dead_code)] + cpu_off: u32, + #[allow(dead_code)] + cpu_suspend: u32, +} + +impl Device for Psci { + fn display_name(&self) -> &'static str { + "ARM PSCI" + } + + unsafe fn init(&'static self) -> Result<(), Error> { + ARCHITECTURE.psci.init(self); + Ok(()) + } +} + +impl CpuBringupDevice for Psci { + unsafe fn start_cpu(&self, id: usize, ip: usize, arg0: usize) -> Result<(), Error> { + self.call(self.cpu_on as _, id as _, ip as _, arg0 as _); + Ok(()) + } +} + +impl Psci { + #[inline] + unsafe fn call(&self, mut x0: u64, x1: u64, x2: u64, x3: u64) -> u64 { + match self.method { + CallMethod::Hvc => { + core::arch::asm!("hvc #0", inlateout("x0") x0, in("x1") x1, in("x2") x2, in("x3") x3) + } + CallMethod::Smc => { + core::arch::asm!("smc #0", inlateout("x0") x0, in("x1") x1, in("x2") x2, in("x3") x3) + } + } + x0 + } +} + +device_tree_driver! { + compatible: ["arm,psci-1.0", "arm,psci"], + probe(dt) => { + let method = devtree::find_prop(&dt.node, "method").and_then(|prop| prop.str().ok())?; + let method = match method { + "hvc" => CallMethod::Hvc, + "smc" => CallMethod::Smc, + _ => panic!("Unknown PSCI call method: {:?}", method) + }; + let cpu_on = devtree::find_prop(&dt.node, "cpu_on").and_then(|prop| prop.u32(0).ok())?; + let cpu_off = devtree::find_prop(&dt.node, "cpu_off").and_then(|prop| prop.u32(0).ok())?; + let cpu_suspend = devtree::find_prop(&dt.node, "cpu_suspend").and_then(|prop| prop.u32(0).ok())?; + + Some(Box::new(Psci { + method, + cpu_on, + cpu_off, + cpu_suspend + })) + } +} diff --git a/src/device/power/mod.rs b/src/device/power/mod.rs new file mode 100644 index 00000000..78fd5dc3 --- /dev/null +++ b/src/device/power/mod.rs @@ -0,0 +1,3 @@ +//! Power-management related device drivers + +pub mod arm_psci; diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index b195d1e1..757e4894 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -1,16 +1,4 @@ //! Serial device interfaces -use abi::error::Error; -use super::Device; - -#[cfg(all(target_arch = "aarch64", not(feature = "aarch64_orange_pi3")))] pub mod pl011; - -/// Generic serial device interface -pub trait SerialDevice: Device { - /// Sends (blocking) a single byte into the serial port - fn send(&self, byte: u8) -> Result<(), Error>; - - /// Receive a single byte from the serial port, blocking if necessary - fn receive(&self, blocking: bool) -> Result; -} +pub mod sunxi_uart; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 4144310c..987eb1e2 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -1,5 +1,11 @@ //! ARM PL011 driver use abi::{error::Error, io::DeviceRequest}; +use alloc::boxed::Box; +use device_api::{ + interrupt::{InterruptHandler, IrqNumber}, + serial::SerialDevice, + Device, +}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -7,16 +13,15 @@ use tock_registers::{ }; use vfs::CharDevice; -use super::SerialDevice; use crate::{ - arch::{aarch64::gic::IrqNumber, PLATFORM}, - debug::DebugSink, - device::{ - interrupt::InterruptSource, - platform::Platform, - tty::{CharRing, TtyDevice}, - Device, InitializableDevice, + arch::{ + aarch64::devtree::{self, DevTreeIndexPropExt}, + Architecture, ARCHITECTURE, }, + debug::{self, DebugSink, LogLevel}, + device::tty::{CharRing, TtyDevice}, + device_tree_driver, + fs::devfs::{self, CharDeviceType}, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, util::OneTimeInit, @@ -81,19 +86,6 @@ impl Pl011Inner { Ok(()) } - fn recv_byte(&mut self, blocking: bool) -> Result { - if self.regs.FR.matches_all(FR::RXFE::SET) { - if !blocking { - todo!(); - } - while self.regs.FR.matches_all(FR::RXFE::SET) { - core::hint::spin_loop(); - } - } - - Ok(self.regs.DR.get() as u8) - } - unsafe fn init(&mut self) { self.regs.CR.set(0); self.regs.ICR.write(ICR::ALL::CLEAR); @@ -141,68 +133,82 @@ impl SerialDevice for Pl011 { fn send(&self, byte: u8) -> Result<(), Error> { self.inner.get().lock().send_byte(byte) } - - fn receive(&self, blocking: bool) -> Result { - self.inner.get().lock().recv_byte(blocking) - } } -impl InterruptSource for Pl011 { - unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.interrupt_controller(); - - intc.register_handler(self.irq, self)?; - self.inner.get().lock().regs.IMSC.modify(IMSC::RXIM::SET); - intc.enable_irq(self.irq)?; - - Ok(()) - } - - fn handle_irq(&self) -> Result { +impl InterruptHandler for Pl011 { + fn handle_irq(&self) -> bool { let inner = self.inner.get().lock(); inner.regs.ICR.write(ICR::ALL::CLEAR); let byte = inner.regs.DR.get(); drop(inner); - self.recv_byte(byte as u8); + if byte == b'\x1b' as u32 { + use crate::task::sched::CpuQueue; - Ok(true) + for (i, queue) in CpuQueue::all().enumerate() { + log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); + let lock = unsafe { queue.grab() }; + for item in lock.iter() { + log_print_raw!( + LogLevel::Fatal, + "* {} {:?} {:?}\n", + item.id(), + item.name(), + item.state() + ); + } + } + } else { + self.recv_byte(byte as u8); + } + + true } } -impl InitializableDevice for Pl011 { - type Options = (); +impl Device for Pl011 { + fn display_name(&self) -> &'static str { + "Primecell PL011 UART" + } - unsafe fn init(&self, _opts: ()) -> Result<(), Error> { + unsafe fn init(&'static self) -> Result<(), Error> { let mut inner = Pl011Inner { regs: DeviceMemoryIo::map("pl011 UART", self.base)?, }; inner.init(); self.inner.init(IrqSafeSpinlock::new(inner)); + + debug::add_sink(self, LogLevel::Debug); + devfs::add_char_device(self, CharDeviceType::TtySerial)?; + + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = ARCHITECTURE.external_interrupt_controller(); + + intc.register_irq(self.irq, Default::default(), self)?; + self.inner.get().lock().regs.IMSC.modify(IMSC::RXIM::SET); + intc.enable_irq(self.irq)?; + Ok(()) } } -impl Device for Pl011 { - fn name(&self) -> &'static str { - "pl011" - } -} +device_tree_driver! { + compatible: ["arm,pl011"], + probe(of) => { + let reg = devtree::find_prop(&of.node, "reg")?; + let (base, _) = reg.cell2_array_item(0, of.address_cells, of.size_cells)?; -impl Pl011 { - /// Constructs an instance of the device at `base`. - /// - /// # Safety - /// - /// The caller must ensure the address is valid. - pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self { - Self { + Some(Box::new(Pl011 { inner: OneTimeInit::new(), + // TODO obtain IRQ from dt + irq: IrqNumber::Shared(1), ring: CharRing::new(), - base, - irq, - } + base: base as usize + })) } } diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs new file mode 100644 index 00000000..e57fa8c5 --- /dev/null +++ b/src/device/serial/sunxi_uart.rs @@ -0,0 +1,154 @@ +//! Allwinner (H6) UART implementation + +use abi::{error::Error, io::DeviceRequest}; +use alloc::boxed::Box; +use device_api::{ + interrupt::{InterruptHandler, IrqNumber}, + serial::SerialDevice, + Device, +}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite}, +}; +use vfs::CharDevice; + +use crate::{ + arch::{ + aarch64::devtree::{self, DevTreeIndexPropExt}, + Architecture, ARCHITECTURE, + }, + debug::{self, DebugSink, LogLevel}, + device::tty::{CharRing, TtyDevice}, + device_tree_driver, + fs::devfs::{self, CharDeviceType}, + mem::device::DeviceMemoryIo, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +register_bitfields! { + u32, + USR [ + TFE OFFSET(2) NUMBITS(1) [], + TFNF OFFSET(1) NUMBITS(1) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x00 => DLL: ReadWrite), + (0x04 => _0), + (0x7C => USR: ReadOnly), + (0x80 => @END), + } +} + +struct Inner { + regs: DeviceMemoryIo, +} + +struct SunxiUart { + inner: OneTimeInit>, + base: usize, + irq: IrqNumber, + ring: CharRing<16>, +} + +impl DebugSink for SunxiUart { + fn putc(&self, c: u8) -> Result<(), Error> { + self.send(c) + } +} + +impl CharDevice for SunxiUart { + fn read(&'static self, _blocking: bool, data: &mut [u8]) -> Result { + self.line_read(data) + } + + fn write(&self, _blocking: bool, data: &[u8]) -> Result { + self.line_write(data) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(id as _); + Ok(()) + } + _ => Err(Error::InvalidArgument), + } + } +} + +impl TtyDevice<16> for SunxiUart { + fn ring(&self) -> &CharRing<16> { + &self.ring + } +} + +impl InterruptHandler for SunxiUart { + fn handle_irq(&self) -> bool { + todo!() + } +} + +impl SerialDevice for SunxiUart { + fn send(&self, byte: u8) -> Result<(), Error> { + let inner = self.inner.get().lock(); + if byte == b'\n' { + while inner.regs.USR.matches_all(USR::TFE::CLEAR) { + core::hint::spin_loop(); + } + inner.regs.DLL.set(b'\r' as u32); + } + while inner.regs.USR.matches_all(USR::TFE::CLEAR) { + core::hint::spin_loop(); + } + inner.regs.DLL.set(byte as u32); + Ok(()) + } +} + +impl Device for SunxiUart { + fn display_name(&self) -> &'static str { + "Allwinner UART" + } + + unsafe fn init(&'static self) -> Result<(), Error> { + let regs = DeviceMemoryIo::::map("sunxi-uart", self.base)?; + self.inner.init(IrqSafeSpinlock::new(Inner { regs })); + debug::add_sink(self, LogLevel::Debug); + devfs::add_char_device(self, CharDeviceType::TtySerial)?; + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = ARCHITECTURE.external_interrupt_controller(); + intc.register_irq(self.irq, Default::default(), self)?; + intc.enable_irq(self.irq)?; + Ok(()) + } +} + +device_tree_driver! { + compatible: ["snps,dw-apb-uart"], + probe(of) => { + let reg = devtree::find_prop(&of.node, "reg")?; + let (base, _) = reg.cell2_array_item(0, of.address_cells, of.size_cells)?; + + if base == 0x05000000 { + Some(Box::new(SunxiUart { + inner: OneTimeInit::new(), + ring: CharRing::new(), + irq: IrqNumber::Shared(0), + base: base as usize + })) + } else { + // TODO don't just hardcode and ignore other UARTs + None + } + } +} diff --git a/src/device/tty.rs b/src/device/tty.rs index 22f0ffef..44673f98 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -4,6 +4,7 @@ use abi::{ io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, process::Signal, }; +use device_api::serial::SerialDevice; use crate::{ proc::wait::Wait, @@ -11,8 +12,6 @@ use crate::{ task::{process::Process, ProcessId}, }; -use super::serial::SerialDevice; - #[cfg(feature = "fb_console")] pub mod combined { use abi::{error::Error, io::DeviceRequest}; diff --git a/src/main.rs b/src/main.rs index 784597b2..7fdb32e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -56,6 +56,8 @@ fn setup_root() -> Result { /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kernel_main() { + infoln!("In main"); + #[cfg(feature = "fb_console")] { use core::time::Duration; diff --git a/src/mem/device.rs b/src/mem/device.rs index d7530111..f9b9f36b 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -36,6 +36,8 @@ impl DeviceMemory { let base = ARCHITECTURE.map_device_pages(aligned_base, aligned_size / 0x1000)?; let base = base + base_offset; + debugln!("Mapped {}@{:#x} to {:#x}", name, phys, base); + Ok(Self { name, base, size }) } diff --git a/src/mem/heap.rs b/src/mem/heap.rs index e8fa441e..2c805182 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -1,6 +1,7 @@ //! Kernel's global heap allocator use core::{ alloc::{GlobalAlloc, Layout}, + ops::Range, ptr::{null_mut, NonNull}, }; @@ -21,13 +22,21 @@ impl KernelAllocator { unsafe fn init(&self, base: usize, size: usize) { self.inner.lock().init(base as _, size); } + + fn range(&self) -> Range { + let lock = self.inner.lock(); + lock.bottom() as usize..lock.top() as usize + } } unsafe impl GlobalAlloc for KernelAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { match self.inner.lock().allocate_first_fit(layout) { Ok(v) => v.as_ptr(), - Err(_) => null_mut(), + Err(e) => { + errorln!("Failed to allocate {:?}: {:?}", layout, e); + null_mut() + } } } @@ -46,6 +55,10 @@ static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); /// /// The caller must ensure the range is valid and mapped virtual memory. pub unsafe fn init_heap(heap_base: usize, heap_size: usize) { - debugln!("Heap: {:#x}..{:#x}", heap_base, heap_base + heap_size); GLOBAL_HEAP.init(heap_base, heap_size); } + +/// Returns the heap address range +pub fn heap_range() -> Range { + GLOBAL_HEAP.range() +} diff --git a/src/mem/mod.rs b/src/mem/mod.rs index a3f3f062..83e941c9 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -7,10 +7,7 @@ use abi::error::Error; // use abi::error::Error; // -use crate::{ - arch::{Architecture, ArchitectureImpl, PlatformImpl}, - device::platform::Platform, -}; +use crate::arch::{Architecture, ArchitectureImpl /*, PlatformImpl*/}; use self::table::AddressSpace; // @@ -22,7 +19,7 @@ pub mod phys; pub mod table; /// Kernel's physical load address -pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; +// pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; /// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + /// [KERNEL_VIRT_OFFSET]) pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index c79e681d..06b8b77d 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -7,7 +7,7 @@ use crate::{ debug::LogLevel, mem::{ phys::reserved::{is_reserved, reserve_region}, - ConvertAddress, KERNEL_PHYS_BASE, + ConvertAddress, /*, KERNEL_PHYS_BASE */ }, sync::IrqSafeSpinlock, util::OneTimeInit, @@ -258,12 +258,12 @@ pub unsafe fn init_from_iter + Clone>( fn kernel_physical_memory_region() -> PhysicalMemoryRegion { extern "C" { + static __kernel_phys_start: u8; static __kernel_size: u8; } + + let base = absolute_address!(__kernel_phys_start); let size = absolute_address!(__kernel_size); - PhysicalMemoryRegion { - base: KERNEL_PHYS_BASE, - size, - } + PhysicalMemoryRegion { base, size } } diff --git a/src/panic.rs b/src/panic.rs index e003320d..b3e3beab 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -1,10 +1,12 @@ //! Kernel panic handler code use core::sync::atomic::{AtomicBool, Ordering}; +use device_api::interrupt::IpiDeliveryTarget; + use crate::{ - arch::{Architecture, ArchitectureImpl}, + arch::{Architecture, ArchitectureImpl, CpuMessage, ARCHITECTURE}, debug::{debug_internal, LogLevel}, - sync::SpinFence, + sync::{hack_locks, SpinFence}, task::{sched::CpuQueue, Cpu}, }; @@ -41,19 +43,19 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() { - // // Let other CPUs know we're screwed - // unsafe { - // PLATFORM - // .send_ipi(IpiDeliveryTarget::AllExceptLocal, CpuMessage::Panic) - // .ok(); - // } + // Let other CPUs know we're screwed + unsafe { + ARCHITECTURE + .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) + .ok(); + } - // let ap_count = ArchitectureImpl::cpu_count() - 1; - // PANIC_HANDLED_FENCE.wait_all(ap_count); + let ap_count = ArchitectureImpl::cpu_count() - 1; + PANIC_HANDLED_FENCE.wait_all(ap_count); - // unsafe { - // hack_locks(); - // } + unsafe { + hack_locks(); + } log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); diff --git a/src/proc/wait.rs b/src/proc/wait.rs index b8b5579b..28772ce9 100644 --- a/src/proc/wait.rs +++ b/src/proc/wait.rs @@ -5,8 +5,7 @@ use abi::error::Error; use alloc::{collections::LinkedList, rc::Rc}; use crate::{ - arch::PLATFORM, - device::platform::Platform, + arch::{Architecture, ARCHITECTURE}, sync::IrqSafeSpinlock, task::process::{Process, ProcessState}, }; @@ -124,7 +123,7 @@ impl Wait { queue_lock = self.queue.lock(); if let Some(deadline) = deadline { - let now = PLATFORM.timestamp_source().timestamp()?; + let now = ARCHITECTURE.monotonic_timer().monotonic_timestamp()?; if now > deadline { let mut cursor = queue_lock.cursor_front_mut(); @@ -150,7 +149,7 @@ static TICK_LIST: IrqSafeSpinlock> = IrqSafeSpinlock::new(Li /// Suspends current task until given deadline pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { static SLEEP_NOTIFY: Wait = Wait::new("sleep"); - let now = PLATFORM.timestamp_source().timestamp()?; + let now = ARCHITECTURE.monotonic_timer().monotonic_timestamp()?; let deadline = now + timeout; match SLEEP_NOTIFY.wait(Some(deadline)) { diff --git a/src/task/mod.rs b/src/task/mod.rs index 2d71df64..82dd505a 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -79,7 +79,7 @@ pub fn init() -> Result<(), Error> { let cpu_count = ArchitectureImpl::cpu_count(); // Create a queue for each CPU - sched::init_queues(Vec::from_iter((0..cpu_count).map(|_| CpuQueue::new()))); + sched::init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); spawn_kernel_closure("[kmain]", kernel_main)?; // spawn_kernel_closure(move || loop { diff --git a/src/task/process.rs b/src/task/process.rs index cfc85081..911cf8b8 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -59,6 +59,8 @@ struct ProcessInner { session_terminal: Option, signal_entry: Option, signal_stack: VecDeque, + + queue: Option<&'static CpuQueue>, } /// Process data and state structure @@ -110,6 +112,8 @@ impl Process { session_terminal: None, signal_entry: None, signal_stack: VecDeque::new(), + + queue: None, }), io: IrqSafeSpinlock::new(ProcessIo::new()), }); @@ -230,8 +234,14 @@ impl Process { /// # Panics /// /// Currently, the code will panic if the process is queued/executing on any queue. - pub fn enqueue_to(self: Rc, queue: &CpuQueue) { + pub fn enqueue_to(self: Rc, queue: &'static CpuQueue) { let _irq = IrqGuard::acquire(); + + { + let mut inner = self.inner.lock(); + let old_queue = inner.queue.replace(queue); + assert!(old_queue.is_none()); + } let current_state = self.state.swap(ProcessState::Ready, Ordering::SeqCst); if current_state == ProcessState::Terminated { @@ -243,15 +253,16 @@ impl Process { } } - /// Marks the process as suspended, blocking it from being run until it's resumed. - /// - /// # Note - /// - /// The process may not halt its execution immediately when this function is called, only when - /// this function is called targeting the *current process* running on *local* CPU. - pub fn suspend(&self) { + fn dequeue(&self, new_state: ProcessState) { let _irq = IrqGuard::acquire(); - let current_state = self.state.swap(ProcessState::Suspended, Ordering::SeqCst); + assert_ne!(new_state, ProcessState::Ready); + assert_ne!(new_state, ProcessState::Running); + + let mut inner = self.inner.lock(); + let current_state = self.state.swap(new_state, Ordering::SeqCst); + let proc_queue = inner.queue.take().unwrap(); + + proc_queue.dequeue(self.id()); match current_state { // NOTE: I'm not sure if the process could've been queued between the store and this @@ -267,6 +278,8 @@ impl Process { let queue = Cpu::local().queue(); if cpu_id == local_cpu_id { + assert_eq!(queue as *const _, proc_queue as *const _, "Process queue mismatch: process says cpu{}, queue {:p}, actual cpu{}, queue {:p}", cpu_id, proc_queue, local_cpu_id, queue); + drop(inner); // Suspending a process running on local CPU unsafe { queue.yield_cpu() } } else { @@ -276,6 +289,16 @@ impl Process { } } + /// Marks the process as suspended, blocking it from being run until it's resumed. + /// + /// # Note + /// + /// The process may not halt its execution immediately when this function is called, only when + /// this function is called targeting the *current process* running on *local* CPU. + pub fn suspend(&self) { + self.dequeue(ProcessState::Suspended); + } + /// Returns current wait status of the task pub fn wait_status(&self) -> WaitStatus { self.inner.lock().wait_status @@ -319,9 +342,7 @@ impl Process { /// Handles the cleanup of an exited process pub fn handle_exit(&self) { - // Queue lock is still held - assert_eq!(self.state(), ProcessState::Terminated); - + // Scheduler still holds a lock of this process? // TODO cancel Wait if a process was killed while suspended? { let inner = self.inner.lock(); @@ -422,21 +443,25 @@ impl CurrentProcess { /// Terminate the current process pub fn exit(&self, status: ExitCode) { self.inner.lock().exit_status = status.into(); - let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); - assert_eq!(current_state, ProcessState::Running); debugln!("Process {} exited with code {:?}", self.id(), status); - match current_state { - ProcessState::Suspended => { - todo!(); - } - ProcessState::Ready => todo!(), - ProcessState::Running => { - self.handle_exit(); - unsafe { Cpu::local().queue().yield_cpu() } - } - ProcessState::Terminated => todo!(), - } + self.handle_exit(); + self.dequeue(ProcessState::Terminated); + + // let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); + // assert_eq!(current_state, ProcessState::Running); + + // match current_state { + // ProcessState::Suspended => { + // todo!(); + // } + // ProcessState::Ready => todo!(), + // ProcessState::Running => { + // self.handle_exit(); + // unsafe { Cpu::local().queue().yield_cpu() } + // } + // ProcessState::Terminated => todo!(), + // } } /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. diff --git a/src/task/sched.rs b/src/task/sched.rs index 7bde8732..f8040c31 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -44,6 +44,7 @@ pub struct CpuQueueInner { pub struct CpuQueue { inner: IrqSafeSpinlock, idle: TaskContext, + index: usize, } static QUEUES: OneTimeInit> = OneTimeInit::new(); @@ -87,18 +88,13 @@ impl CpuQueueInner { ProcessState::Ready => { return Some(task); } - // Drop suspended tasks from the queue - ProcessState::Suspended | ProcessState::Terminated => (), - ProcessState::Running => { - // TODO fix this finally - } - // e => panic!( - // "Unexpected process state in CpuQueue: {:?} ({} {:?}, cpu_id={})", - // e, - // task.id(), - // task.name(), - // task.cpu_id() - // ), + e => panic!( + "Unexpected process state in CpuQueue: {:?} ({} {:?}, cpu_id={})", + e, + task.id(), + task.name(), + task.cpu_id() + ), } } @@ -114,7 +110,7 @@ impl CpuQueueInner { impl CpuQueue { /// Constructs an empty queue with its own idle task - pub fn new() -> Self { + pub fn new(index: usize) -> Self { let idle = TaskContext::kernel(__idle, Cpu::local_id() as usize) .expect("Could not construct an idle task"); @@ -127,6 +123,7 @@ impl CpuQueue { }) }, idle, + index, } } @@ -164,8 +161,8 @@ impl CpuQueue { if let Some(current) = current.as_ref() { if current.state() == ProcessState::Running { current.set_state(ProcessState::Ready); + inner.queue.push_back(current.clone()); } - inner.queue.push_back(current.clone()); // inner.stats.cpu_time += delta; } else { @@ -209,6 +206,7 @@ impl CpuQueue { // log_print_raw!(crate::debug::LogLevel::Info, "\n"); + assert!(ArchitectureImpl::interrupt_mask()); to.switch(from) } @@ -219,13 +217,19 @@ impl CpuQueue { /// Only meant to be called from Process impl. The function does not set any process accounting /// information, which may lead to invalid states. pub unsafe fn enqueue(&self, p: Rc) { + let mut inner = self.inner.lock(); + assert!(ArchitectureImpl::interrupt_mask()); assert_eq!(p.state(), ProcessState::Ready); - self.inner.lock().queue.push_back(p); + + inner.queue.push_back(p); } /// Removes process with given PID from the exeuction queue. - pub fn dequeue(&self, _pid: ProcessId) { - todo!(); + pub fn dequeue(&self, pid: ProcessId) { + assert!(ArchitectureImpl::interrupt_mask()); + + let mut inner = self.inner.lock(); + inner.queue.retain(|p| p.id() != pid) } /// Returns the queue length at this moment. From bbf30b21a6fa96b8f2078ed3a9236559fa0219d3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 14 Aug 2023 14:31:44 +0300 Subject: [PATCH 049/211] aarch64: use appropriate MAIR for device/normal memory --- src/arch/aarch64/context.S | 3 ++ src/arch/aarch64/context.rs | 2 +- src/arch/aarch64/exception.rs | 30 +++++++++++++- src/arch/aarch64/mod.rs | 36 ++++++++++++++-- src/arch/aarch64/table.rs | 77 ++++++++++++++++++++++++++++++++--- src/proc/exec.rs | 2 +- 6 files changed, 137 insertions(+), 13 deletions(-) diff --git a/src/arch/aarch64/context.S b/src/arch/aarch64/context.S index 2b918c12..3008c2aa 100644 --- a/src/arch/aarch64/context.S +++ b/src/arch/aarch64/context.S @@ -82,6 +82,9 @@ __aarch64_task_enter_user: mov lr, xzr + dmb ish + isb sy + eret __aarch64_switch_task: diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 3f4d128e..1da9a20d 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -113,7 +113,7 @@ impl TaskContextImpl for TaskContext { } fn user(entry: usize, arg: usize, ttbr0: usize, user_stack_sp: usize) -> Result { - const USER_TASK_PAGES: usize = 8; + const USER_TASK_PAGES: usize = 16; let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 18676de9..d67d7ae0 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -1,7 +1,13 @@ //! Exception and interrupt management functions use core::{arch::global_asm, fmt}; -use aarch64_cpu::registers::{ELR_EL1, ESR_EL1, FAR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; +use aarch64_cpu::{ + asm::barrier, + registers::{ + ELR_EL1, ESR_EL1, FAR_EL1, SCTLR_EL1, TCR_EL1, TPIDR_EL0, TPIDR_EL1, TTBR0_EL1, TTBR1_EL1, + VBAR_EL1, + }, +}; use abi::{ arch::SavedFrame, process::{Signal, SignalEntryData}, @@ -10,8 +16,12 @@ use abi::{ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{aarch64::cpu::Cpu, Architecture}, + arch::{ + aarch64::{cpu::Cpu, table::AddressSpace}, + Architecture, + }, debug::LogLevel, + mem::KERNEL_VIRT_OFFSET, syscall::raw_syscall_handler, task::{context::TaskFrame, process::Process}, }; @@ -107,6 +117,7 @@ pub fn init_exceptions() { } let vbar = unsafe { &__aarch64_el1_vectors as *const _ }; VBAR_EL1.set(vbar as u64); + barrier::isb(barrier::SY); } fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { @@ -168,6 +179,21 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { _ => (), } + + unsafe { + let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); + let far = FAR_EL1.get() as usize; + space.walk(far, |level, raw| { + log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); + true + }); + } + + log_print_raw!(LogLevel::Fatal, "System register dump:\n"); + log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); + log_print_raw!(LogLevel::Fatal, "TCR_EL1 = {:#x}\n", TCR_EL1.get()); + log_print_raw!(LogLevel::Fatal, "TPIDR_EL1 = {:#x}\n", TPIDR_EL1.get()); + log_print_raw!(LogLevel::Fatal, "TPIDR_EL0 = {:#x}\n", TPIDR_EL0.get()); } #[no_mangle] diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 534ff6ef..a89fe599 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -2,8 +2,12 @@ use core::sync::atomic::Ordering; -use aarch64_cpu::registers::{ - CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF, ID_AA64MMFR0_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1, TTBR1_EL1, +use aarch64_cpu::{ + asm::barrier, + registers::{ + CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, + TTBR0_EL1, TTBR1_EL1, + }, }; use abi::error::Error; use device_api::{ @@ -97,6 +101,14 @@ impl Architecture for AArch64 { todo!(); } + MAIR_EL1.write( + // Attribute 0 -- normal memory + MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + // Attribute 1 -- device memory + MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, + ); + TCR_EL1.modify( // General TCR_EL1::IPS::Bits_48 + @@ -106,10 +118,18 @@ impl Architecture for AArch64 { TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, ); + barrier::dmb(barrier::ISH); + TTBR0_EL1.set_baddr(tables_phys); TTBR1_EL1.set_baddr(tables_phys); - SCTLR_EL1.modify(SCTLR_EL1::M::Enable); + barrier::isb(barrier::SY); + + // Enable instruction cache, data cache and translation + SCTLR_EL1 + .modify(SCTLR_EL1::M::Enable + SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable); + + barrier::isb(barrier::SY); } fn map_device_pages(&self, phys: usize, count: usize) -> Result { @@ -391,6 +411,7 @@ pub fn kernel_main(dtb_phys: usize) -> ! { // Unmap TTBR0 TTBR0_EL1.set(0); + barrier::isb(barrier::SY); unsafe { ARCHITECTURE.init_device_tree(dtb_phys); @@ -416,6 +437,15 @@ pub fn kernel_main(dtb_phys: usize) -> ! { // Enumerate the device tree ARCHITECTURE.init_platform(true); + barrier::isb(barrier::SY); + SCTLR_EL1.modify( + SCTLR_EL1::SA::Enable + + SCTLR_EL1::A::Enable + + SCTLR_EL1::I::NonCacheable + + SCTLR_EL1::C::NonCacheable, + ); + barrier::isb(barrier::SY); + debugln!("Heap: {:#x?}", heap::heap_range()); Cpu::init_local(); diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index a9e8105f..723fd497 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -76,6 +76,12 @@ bitflags! { /// For page/block mappings, only allows read access for EL0/EL1 const AP_BOTH_READONLY = 3 << 6; + const SH_OUTER = 2 << 8; + const SH_INNER = 3 << 8; + + const PAGE_ATTR_NORMAL = 0 << 2; + const PAGE_ATTR_DEVICE = 1 << 2; + /// Indicates the mapping is unique to a specific ASID (important for proper TLB /// maintenance) const NON_GLOBAL = 1 << 11; @@ -126,10 +132,29 @@ pub struct FixedTables { impl PageEntry { /// Creates a 4KiB page mapping - pub fn page(phys: usize, attrs: PageAttributes) -> Self { + pub fn normal_page(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) - | (PageAttributes::PAGE | PageAttributes::PRESENT | PageAttributes::ACCESS | attrs) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_page(phys: usize, attrs: PageAttributes) -> Self { + Self( + (phys as u64) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | attrs) .bits(), PhantomData, ) @@ -150,12 +175,14 @@ impl PageEntry { impl PageEntry { /// Creates a 2MiB page mapping - pub fn block(phys: usize, attrs: PageAttributes) -> Self { + pub fn normal_block(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) | (PageAttributes::BLOCK | PageAttributes::PRESENT | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL | attrs) .bits(), PhantomData, @@ -299,7 +326,7 @@ impl FixedTables { let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12); for i in 0..count { self.device_l3[self.device_l3i + i] = - PageEntry::page(phys + i * 0x1000, PageAttributes::empty()); + PageEntry::device_page(phys + i * 0x1000, PageAttributes::empty()); } self.device_l3i += count; @@ -378,7 +405,7 @@ impl VirtualMemoryManager for AddressSpace { } fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> { - self.write_entry(virt, PageEntry::page(phys, attrs.into()), true) + self.write_entry(virt, PageEntry::normal_page(phys, attrs.into()), true) } } @@ -400,6 +427,44 @@ impl AddressSpace { Ok(Self { l1, asid }) } + pub unsafe fn from_phys_raw(value: usize) -> Self { + let value = value.virtualize(); + Self { + l1: value as *mut PageTable, + asid: 0, + } + } + + pub fn walk bool>(&self, vaddr: usize, f: F) { + let l1i = L1::index(vaddr); + let l2i = L2::index(vaddr); + let l3i = L3::index(vaddr); + + let l1 = unsafe { self.as_mut() }; + let l1e = l1[l1i]; + if !f(1, l1e.0 as _) { + return; + } + + let Some(l2) = l1.get_mut(l1i) else { + f(1, 0); + return; + }; + let l2e = l2[l2i]; + if !f(2, l2e.0 as _) { + return; + } + + let Some(l3) = l2.get_mut(l2i) else { + f(2, 0); + return; + }; + let l3e = l3[l3i]; + if !f(3, l3e.0 as _) { + return; + } + } + unsafe fn as_mut(&self) -> &'static mut PageTable { self.l1.as_mut().unwrap() } @@ -458,7 +523,7 @@ pub fn tlb_flush_vaae1(mut page: usize) { pub unsafe fn init_fixed_tables() { // Map first 256GiB for i in 0..256 { - KERNEL_TABLES.l1[i] = PageEntry::::block(i << 30, PageAttributes::empty()); + KERNEL_TABLES.l1[i] = PageEntry::::normal_block(i << 30, PageAttributes::empty()); } KERNEL_TABLES.l1[256] = PageEntry::::table( diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 4803d636..c110c7a7 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -76,7 +76,7 @@ fn setup_binary>( entry: usize, args: &[&str], ) -> Result, Error> { - const USER_STACK_PAGES: usize = 8; + const USER_STACK_PAGES: usize = 16; let virt_stack_base = 0x3000000; // 0x1000 of guard page From 887d65f89023a141cf837a01916df4a1b3fc1e13 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 15 Aug 2023 13:32:48 +0300 Subject: [PATCH 050/211] dev: sunxi-uart interrupts --- src/arch/aarch64/context.S | 2 +- src/arch/aarch64/exception.rs | 52 +++++++++++++++- src/arch/aarch64/plat_qemu/mod.rs | 98 ------------------------------- src/arch/aarch64/table.rs | 11 ++-- src/device/serial/sunxi_uart.rs | 54 ++++++++++++++++- src/proc/exec.rs | 4 +- src/syscall/mod.rs | 9 ++- 7 files changed, 117 insertions(+), 113 deletions(-) delete mode 100644 src/arch/aarch64/plat_qemu/mod.rs diff --git a/src/arch/aarch64/context.S b/src/arch/aarch64/context.S index 3008c2aa..83d9ef03 100644 --- a/src/arch/aarch64/context.S +++ b/src/arch/aarch64/context.S @@ -15,7 +15,7 @@ mrs x19, tpidr_el0 mrs x20, ttbr0_el1 - stp x29, x20, [sp, #16 * 6] + stp x19, x20, [sp, #16 * 6] .endm .macro LOAD_TASK_STATE diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index d67d7ae0..26aae24b 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -18,7 +18,7 @@ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ arch::{ aarch64::{cpu::Cpu, table::AddressSpace}, - Architecture, + Architecture, ArchitectureImpl, }, debug::LogLevel, mem::KERNEL_VIRT_OFFSET, @@ -198,20 +198,66 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { #[no_mangle] extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) { + assert!(ArchitectureImpl::interrupt_mask()); let frame = unsafe { &mut *frame }; - let process = Process::current(); + + // let current_id = Process::get_current().map(|p| p.id()); + // log_print_raw!( + // LogLevel::Fatal, + // "sync_entry {:?}: x30 = {:#x} @ {:p}\n", + // current_id, + // frame.r[30], + // &frame.r[30] + // ); + el0_sync_inner(frame); + + // let current_id = Process::get_current().map(|p| p.id()); + // log_print_raw!( + // LogLevel::Fatal, + // "sync_exit {:?}: x30 = {:#x} @ {:p}\n", + // current_id, + // frame.r[30], + // &frame.r[30] + // ); + unsafe { + let process = Process::current(); process.handle_signal(frame); } } #[no_mangle] extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) { + assert!(ArchitectureImpl::interrupt_mask()); let frame = unsafe { &mut *frame }; - let process = Process::current(); + + // let current_id = Process::get_current().map(|p| p.id()); + // log_print_raw!(LogLevel::Fatal, "+++ irq_handler ENTRY +++\n"); + // log_print_raw!( + // LogLevel::Fatal, + // "{:?}: x30 = {:#x} @ {:p}\n", + // current_id, + // frame.r[30], + // &frame.r[30] + // ); + // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); + irq_common(); + + // let current_id = Process::get_current().map(|p| p.id()); + // log_print_raw!(LogLevel::Fatal, "+++ irq_handler EXIT +++\n"); + // log_print_raw!( + // LogLevel::Fatal, + // "{:?}: x30 = {:#x} @ {:p}\n", + // current_id, + // frame.r[30], + // &frame.r[30] + // ); + // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); + unsafe { + let process = Process::current(); process.handle_signal(frame); } } diff --git a/src/arch/aarch64/plat_qemu/mod.rs b/src/arch/aarch64/plat_qemu/mod.rs deleted file mode 100644 index eb89a64d..00000000 --- a/src/arch/aarch64/plat_qemu/mod.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! Qemu's "virt" platform implementation for AArch64 -use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; -use abi::error::Error; -use tock_registers::interfaces::Writeable; - -use crate::{ - arch::CpuMessage, - debug::{self, LogLevel}, - device::{ - interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, - platform::Platform, - serial::pl011::Pl011, - timer::TimestampSource, - InitializableDevice, - }, - fs::devfs::{self, CharDeviceType}, -}; - -use super::{ - gic::{Gic, IrqNumber}, - timer::ArmTimer, -}; - -/// AArch64 "virt" platform implementation -pub struct PlatformImpl { - /// ... - pub gic: Gic, - pl011: Pl011, - local_timer: ArmTimer, -} - -impl Platform for PlatformImpl { - type IrqNumber = IrqNumber; - - const KERNEL_PHYS_BASE: usize = 0x40080000; - - unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { - loop {} - // if is_bsp { - // self.gic.init(())?; - - // self.pl011.init_irq()?; - // devfs::add_char_device(&self.pl011, CharDeviceType::TtySerial)?; - - // self.local_timer.init(())?; - // self.local_timer.init_irq()?; - // } else { - // self.gic.init_smp_ap()?; - - // // TODO somehow merge this with the rest of the code - // CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); - // CNTP_TVAL_EL0.set(10000000); - // self.gic.enable_irq(IrqNumber::new(30))?; - // } - - // Ok(()) - } - - unsafe fn init_debug(&'static self) { - loop {} - // self.pl011.init(()).ok(); - // debug::add_sink(&self.pl011, LogLevel::Debug); - } - - fn interrupt_controller( - &self, - ) -> &dyn ExternalInterruptController { - &self.gic - } - - fn name(&self) -> &'static str { - "qemu" - } - - fn timestamp_source(&self) -> &dyn TimestampSource { - &self.local_timer - } - - unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { - infoln!("TODO send ipi"); - loop { - core::hint::spin_loop(); - } - } - - unsafe fn reset(&self) -> ! { - todo!() - } -} - -/// AArch64 "virt" platform -pub static PLATFORM: PlatformImpl = unsafe { - PlatformImpl { - pl011: Pl011::new(0x09000000, IrqNumber::new(33)), - gic: Gic::new(0x08000000, 0x08010000), - local_timer: ArmTimer::new(IrqNumber::new(30)), - } -}; diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index 723fd497..384c64a6 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -442,23 +442,24 @@ impl AddressSpace { let l1 = unsafe { self.as_mut() }; let l1e = l1[l1i]; - if !f(1, l1e.0 as _) { - return; - } let Some(l2) = l1.get_mut(l1i) else { f(1, 0); return; }; - let l2e = l2[l2i]; - if !f(2, l2e.0 as _) { + if !f(1, l1e.0 as _) { return; } + let l2e = l2[l2i]; let Some(l3) = l2.get_mut(l2i) else { f(2, 0); return; }; + if !f(2, l2e.0 as _) { + return; + } + let l3e = l3[l3i]; if !f(3, l3e.0 as _) { return; diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs index e57fa8c5..70d2c1b1 100644 --- a/src/device/serial/sunxi_uart.rs +++ b/src/device/serial/sunxi_uart.rs @@ -8,7 +8,7 @@ use device_api::{ Device, }; use tock_registers::{ - interfaces::{Readable, Writeable}, + interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, registers::{ReadOnly, ReadWrite}, }; @@ -33,6 +33,14 @@ register_bitfields! { USR [ TFE OFFSET(2) NUMBITS(1) [], TFNF OFFSET(1) NUMBITS(1) [] + ], + IER [ + ERBFI OFFSET(0) NUMBITS(1) [], + ], + IIR [ + IID OFFSET(0) NUMBITS(4) [ + RecvDataAvailable = 0b0100, + ] ] } @@ -40,7 +48,9 @@ register_structs! { #[allow(non_snake_case)] Regs { (0x00 => DLL: ReadWrite), - (0x04 => _0), + (0x04 => IER: ReadWrite), + (0x08 => IIR: ReadWrite), + (0x0C => _0), (0x7C => USR: ReadOnly), (0x80 => @END), } @@ -91,7 +101,40 @@ impl TtyDevice<16> for SunxiUart { impl InterruptHandler for SunxiUart { fn handle_irq(&self) -> bool { - todo!() + let inner = self.inner.get().lock(); + + if inner.regs.IIR.matches_all(IIR::IID::RecvDataAvailable) { + let byte = inner.regs.DLL.get(); + drop(inner); + + self.recv_byte(byte as u8); + } + // inner.regs.ICR.write(ICR::ALL::CLEAR); + + // let byte = inner.regs.DR.get(); + // drop(inner); + + // if byte == b'\x1b' as u32 { + // use crate::task::sched::CpuQueue; + + // for (i, queue) in CpuQueue::all().enumerate() { + // log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); + // let lock = unsafe { queue.grab() }; + // for item in lock.iter() { + // log_print_raw!( + // LogLevel::Fatal, + // "* {} {:?} {:?}\n", + // item.id(), + // item.name(), + // item.state() + // ); + // } + // } + // } else { + // self.recv_byte(byte as u8); + // } + + true } } @@ -127,8 +170,13 @@ impl Device for SunxiUart { unsafe fn init_irq(&'static self) -> Result<(), Error> { let intc = ARCHITECTURE.external_interrupt_controller(); + intc.register_irq(self.irq, Default::default(), self)?; intc.enable_irq(self.irq)?; + + let inner = self.inner.get().lock(); + inner.regs.IER.modify(IER::ERBFI::SET); + Ok(()) } } diff --git a/src/proc/exec.rs b/src/proc/exec.rs index c110c7a7..13856795 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -34,7 +34,7 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er space.map_page( virt, phys_page, - MapAttributes::USER_READ | MapAttributes::USER_WRITE, // PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, // PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, )?; let write = unsafe { phys_page.virtualize() }; @@ -87,7 +87,7 @@ fn setup_binary>( space.map_page( virt_stack_base + i * 0x1000, phys, - MapAttributes::USER_WRITE | MapAttributes::USER_READ, // PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index c4c53d4e..01fc5348 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -415,6 +415,13 @@ pub fn raw_syscall_handler(func: u64, args: &[u64]) -> u64 { todo!("Undefined syscall: {}", func); }; - // debugln!("raw_syscall_handler {:?}, {:x?}", func, args); + // let proc = Process::current(); + // debugln!( + // "{} {:?}: raw_syscall_handler {:?}, {:x?}", + // proc.id(), + // proc.name(), + // func, + // args + // ); syscall_handler(func, args).into_syscall_result() as u64 } From f6b27eb42f7a925bae48e53701eb9cbbf6bbdb1c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 15 Aug 2023 18:38:54 +0300 Subject: [PATCH 051/211] sunxi-h6: reset through watchdog --- lib/device-api/src/lib.rs | 4 ++ src/arch/aarch64/mod.rs | 11 +++- src/arch/mod.rs | 3 + src/device/power/mod.rs | 1 + src/device/power/sunxi_rwdog.rs | 110 ++++++++++++++++++++++++++++++++ src/device/serial/sunxi_uart.rs | 4 ++ src/panic.rs | 5 ++ 7 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 src/device/power/sunxi_rwdog.rs diff --git a/lib/device-api/src/lib.rs b/lib/device-api/src/lib.rs index d73fceb6..32427ea0 100644 --- a/lib/device-api/src/lib.rs +++ b/lib/device-api/src/lib.rs @@ -22,3 +22,7 @@ pub trait CpuBringupDevice: Device { /// misused. unsafe fn start_cpu(&self, id: usize, ip: usize, arg0: usize) -> Result<(), Error>; } + +pub trait ResetDevice: Device { + unsafe fn reset(&self) -> !; +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index a89fe599..506ca1aa 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -15,6 +15,7 @@ use device_api::{ ExternalInterruptController, IpiDeliveryTarget, IrqNumber, LocalInterruptController, }, timer::MonotonicTimestampProviderDevice, + ResetDevice, }; use fdt_rs::prelude::PropReader; use git_version::git_version; @@ -70,6 +71,7 @@ pub struct AArch64 { ext_intc: OneTimeInit<&'static dyn ExternalInterruptController>, local_intc: OneTimeInit<&'static dyn LocalInterruptController>, mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, + reset: OneTimeInit<&'static dyn ResetDevice>, // ARM-only devices /// ARM PSCI instance on this system (there may not be one) @@ -82,6 +84,7 @@ pub static ARCHITECTURE: AArch64 = AArch64 { ext_intc: OneTimeInit::new(), local_intc: OneTimeInit::new(), mtimer: OneTimeInit::new(), + reset: OneTimeInit::new(), // ARM-only devices psci: OneTimeInit::new(), @@ -180,6 +183,11 @@ impl Architecture for AArch64 { Ok(()) } + fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { + self.reset.init(reset); + Ok(()) + } + fn external_interrupt_controller(&self) -> &'static dyn ExternalInterruptController { *self.ext_intc.get() } @@ -203,7 +211,8 @@ impl Architecture for AArch64 { } unsafe fn reset(&self) -> ! { - todo!() + let reset = self.reset.get(); + reset.reset(); } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index a3b0f803..b30e5437 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -25,6 +25,7 @@ pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; use device_api::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, + ResetDevice, }; // cfg_if! { // if #[cfg(target_arch = "aarch64")] { @@ -106,6 +107,8 @@ pub trait Architecture { timer: &'static dyn MonotonicTimestampProviderDevice, ) -> Result<(), Error>; + fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error>; + // TODO only supports 1 extintc per system /// Returns the primary external interrupt controller fn external_interrupt_controller(&self) -> &'static dyn ExternalInterruptController; diff --git a/src/device/power/mod.rs b/src/device/power/mod.rs index 78fd5dc3..8b87a8c6 100644 --- a/src/device/power/mod.rs +++ b/src/device/power/mod.rs @@ -1,3 +1,4 @@ //! Power-management related device drivers pub mod arm_psci; +pub mod sunxi_rwdog; diff --git a/src/device/power/sunxi_rwdog.rs b/src/device/power/sunxi_rwdog.rs new file mode 100644 index 00000000..7d47e3de --- /dev/null +++ b/src/device/power/sunxi_rwdog.rs @@ -0,0 +1,110 @@ +//! Allwinner (H6) R Watchdog driver + +use abi::error::Error; +use alloc::boxed::Box; +use device_api::{Device, ResetDevice}; +use tock_registers::{ + interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, +}; + +use crate::{ + arch::{ + aarch64::devtree::{self, DevTreeIndexNodePropGet, DevTreeIndexPropExt}, + Architecture, ARCHITECTURE, + }, + device_tree_driver, + mem::device::DeviceMemoryIo, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; + +register_bitfields! { + u32, + CTRL [ + KEY OFFSET(1) NUMBITS(12) [ + Value = 0xA57 + ], + RESTART OFFSET(0) NUMBITS(1) [] + ], + CFG [ + CONFIG OFFSET(0) NUMBITS(2) [ + System = 1, + ] + ], + MODE [ + EN OFFSET(0) NUMBITS(1) [], + ] +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x00 => IRQ_EN: ReadWrite), + (0x04 => IRQ_STA: ReadWrite), + (0x08 => _0), + (0x10 => CTRL: ReadWrite), + (0x14 => CFG: ReadWrite), + (0x18 => MODE: ReadWrite), + (0x1C => @END), + } +} + +struct Inner { + regs: DeviceMemoryIo, +} + +struct RWdog { + inner: OneTimeInit>, + base: usize, +} + +impl ResetDevice for RWdog { + unsafe fn reset(&self) -> ! { + // TODO disable IRQs + let inner = self.inner.get().lock(); + + inner.regs.CFG.write(CFG::CONFIG::System); + inner.regs.MODE.write(MODE::EN::SET); + inner.regs.CTRL.write(CTRL::KEY::Value + CTRL::RESTART::SET); + + loop { + core::arch::asm!("wfe"); + } + } +} + +impl Device for RWdog { + fn display_name(&self) -> &'static str { + "Allwinner H6 Watchdog" + } + + unsafe fn init(&'static self) -> Result<(), Error> { + let regs = DeviceMemoryIo::map("r_wdog", self.base)?; + + self.inner.init(IrqSafeSpinlock::new(Inner { regs })); + + ARCHITECTURE.register_reset_device(self)?; + + Ok(()) + } +} + +device_tree_driver! { + compatible: ["allwinner,sun50i-h6-wdt"], + probe(of) => { + let reg = devtree::find_prop(&of.node, "reg")?; + let status: &str = of.node.prop("status").unwrap_or("enabled"); + + if status == "disabled" { + return None; + } + + let (base, _) = reg.cell2_array_item(0, of.address_cells, of.size_cells)?; + let base = base as usize; + + Some(Box::new(RWdog { + inner: OneTimeInit::new(), + base + })) + } +} diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs index 70d2c1b1..d1d24862 100644 --- a/src/device/serial/sunxi_uart.rs +++ b/src/device/serial/sunxi_uart.rs @@ -107,6 +107,10 @@ impl InterruptHandler for SunxiUart { let byte = inner.regs.DLL.get(); drop(inner); + if byte == b'\x1b' as u32 { + panic!("RESET TRIGGERED"); + } + self.recv_byte(byte as u8); } // inner.regs.ICR.write(ICR::ALL::CLEAR); diff --git a/src/panic.rs b/src/panic.rs index b3e3beab..ef3e9962 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -97,6 +97,11 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { log_print_raw!(LogLevel::Fatal, "X"); PANIC_FINISHED_FENCE.signal(); + + #[cfg(feature = "aarch64_orange_pi3")] + unsafe { + ARCHITECTURE.reset(); + } } loop { From 13aa4402390fbf53ba1d45aa9a1cca65f0b5550e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 18 Aug 2023 09:58:56 +0300 Subject: [PATCH 052/211] aarch64: reset through PSCI as a fallback --- src/arch/aarch64/devtree.rs | 82 ++++++++++++++++++------------------ src/arch/aarch64/mod.rs | 10 +++-- src/device/power/arm_psci.rs | 21 ++++++++- src/main.rs | 2 + src/panic.rs | 1 - 5 files changed, 70 insertions(+), 46 deletions(-) diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index d0e8c99c..0fdc95b4 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -53,6 +53,8 @@ pub trait DevTreeIndexNodeExt { fn get_address_cells(&self) -> Option; /// Returns the #size-cells property of the node, if there is one fn get_size_cells(&self) -> Option; + + fn dump(&self, level: LogLevel, depth: usize); } /// Extension trait for [DevTreeIndexNode] to obtain typed property values @@ -118,7 +120,7 @@ impl<'a> DeviceTree<'a> { /// Prints the device tree to log output pub fn dump(&self, level: LogLevel) { - dump_node(&self.index.root(), 0, level) + self.index.root().dump(level, 0) } /// Returns the total size of the device tree in memory @@ -156,6 +158,45 @@ impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { .find(|p| p.name().unwrap_or("") == "#size-cells") .map(|p| p.u32(0).unwrap() as usize) } + + fn dump(&self, level: LogLevel, depth: usize) { + fn indent(level: LogLevel, depth: usize) { + for _ in 0..depth { + log_print_raw!(level, " "); + } + } + + let node_name = self.name().unwrap(); + + // Don't dump these + if node_name.starts_with("virtio_mmio@") { + return; + } + + indent(level, depth); + log_print_raw!(level, "{:?} {{\n", node_name); + for prop in self.props() { + indent(level, depth + 1); + let name = prop.name().unwrap_or(""); + log_print_raw!(level, "{name:?} = "); + + match name { + "compatible" | "stdout-path" => { + log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) + } + _ => log_print_raw!(level, "{:x?}", prop.raw()), + } + + log_print_raw!(level, "\n"); + } + + for child in self.children() { + child.dump(level, depth + 1); + } + + indent(level, depth); + log_print_raw!(level, "}}\n"); + } } impl<'a, 'i, 'dt> DevTreeIndexPropExt for DevTreeIndexProp<'a, 'i, 'dt> { @@ -255,42 +296,3 @@ fn find_node<'a>(at: TNode<'a>, path: &str) -> Option> { } } } - -fn dump_node(node: &TNode, depth: usize, level: LogLevel) { - fn indent(level: LogLevel, depth: usize) { - for _ in 0..depth { - log_print_raw!(level, " "); - } - } - - let node_name = node.name().unwrap(); - - // Don't dump these - if node_name.starts_with("virtio_mmio@") { - return; - } - - indent(level, depth); - log_print_raw!(level, "{:?} {{\n", node_name); - for prop in node.props() { - indent(level, depth + 1); - let name = prop.name().unwrap_or(""); - log_print_raw!(level, "{name:?} = "); - - match name { - "compatible" | "stdout-path" => { - log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) - } - _ => log_print_raw!(level, "{:x?}", prop.raw()), - } - - log_print_raw!(level, "\n"); - } - - for child in node.children() { - dump_node(&child, depth + 1, level); - } - - indent(level, depth); - log_print_raw!(level, "}}\n"); -} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 506ca1aa..c6ade41c 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -15,7 +15,7 @@ use device_api::{ ExternalInterruptController, IpiDeliveryTarget, IrqNumber, LocalInterruptController, }, timer::MonotonicTimestampProviderDevice, - ResetDevice, + CpuBringupDevice, ResetDevice, }; use fdt_rs::prelude::PropReader; use git_version::git_version; @@ -211,8 +211,12 @@ impl Architecture for AArch64 { } unsafe fn reset(&self) -> ! { - let reset = self.reset.get(); - reset.reset(); + if let Some(reset) = self.reset.try_get() { + reset.reset(); + } else { + let psci = self.psci.get(); + psci.reset(); + } } } diff --git a/src/device/power/arm_psci.rs b/src/device/power/arm_psci.rs index 05f31d3c..d7397796 100644 --- a/src/device/power/arm_psci.rs +++ b/src/device/power/arm_psci.rs @@ -2,11 +2,14 @@ use abi::error::Error; use alloc::boxed::Box; -use device_api::{CpuBringupDevice, Device}; +use device_api::{CpuBringupDevice, Device, ResetDevice}; use fdt_rs::prelude::PropReader; use crate::{ - arch::{aarch64::devtree, ARCHITECTURE}, + arch::{ + aarch64::devtree::{self}, + Architecture, ArchitectureImpl, ARCHITECTURE, + }, device_tree_driver, }; @@ -43,7 +46,21 @@ impl CpuBringupDevice for Psci { } } +impl ResetDevice for Psci { + unsafe fn reset(&self) -> ! { + ArchitectureImpl::set_interrupt_mask(true); + + self.call(Self::SYSTEM_RESET as _, 0, 0, 0); + + loop { + ArchitectureImpl::wait_for_interrupt(); + } + } +} + impl Psci { + const SYSTEM_RESET: u32 = 0x84000009; + #[inline] unsafe fn call(&self, mut x0: u64, x1: u64, x2: u64, x3: u64) -> u64 { match self.method { diff --git a/src/main.rs b/src/main.rs index 7fdb32e8..603b4466 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,8 @@ fn setup_root() -> Result { pub fn kernel_main() { infoln!("In main"); + panic!(); + #[cfg(feature = "fb_console")] { use core::time::Duration; diff --git a/src/panic.rs b/src/panic.rs index ef3e9962..21a4d425 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -98,7 +98,6 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { log_print_raw!(LogLevel::Fatal, "X"); PANIC_FINISHED_FENCE.signal(); - #[cfg(feature = "aarch64_orange_pi3")] unsafe { ARCHITECTURE.reset(); } From cf53f5c48582caf041e7d7e5ee0ee2b188ea52af Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 18 Aug 2023 18:05:07 +0300 Subject: [PATCH 053/211] fs: copy-on-write for memfs --- lib/memfs/src/block.rs | 151 +++++++++++++++++++++++++++++++++-------- lib/memfs/src/bvec.rs | 28 ++++++-- lib/memfs/src/lib.rs | 43 +++++++++++- src/fs/mod.rs | 2 +- src/main.rs | 2 - 5 files changed, 186 insertions(+), 40 deletions(-) diff --git a/lib/memfs/src/block.rs b/lib/memfs/src/block.rs index ec68ef93..0b732bd1 100644 --- a/lib/memfs/src/block.rs +++ b/lib/memfs/src/block.rs @@ -31,10 +31,14 @@ pub unsafe trait BlockAllocator: 'static { unsafe fn dealloc(block: NonNull); } +struct BlockRef<'a, A: BlockAllocator> { + ptr: usize, + _pd: PhantomData<&'a A>, +} + #[repr(transparent)] struct BlockRaw<'a, A: BlockAllocator> { - inner: Option<&'a mut [u8; SIZE]>, - _pd: PhantomData, + inner: BlockRef<'a, A>, // inner: Option<&'a mut [u8; SIZE]>, } /// Block containing file data @@ -49,45 +53,122 @@ pub struct BlockIndirect<'a, A: BlockAllocator> { inner: BlockRaw<'a, A>, } +impl<'a, A: BlockAllocator> BlockRef<'a, A> { + const fn null() -> Self { + Self { + ptr: 0, + _pd: PhantomData, + } + } + + unsafe fn from_allocated(address: usize) -> Self { + debug_assert_eq!(address & 1, 0); + Self { + ptr: address, + _pd: PhantomData, + } + } + + unsafe fn copy_on_write(address: usize) -> Self { + debug_assert_eq!(address & 1, 0); + Self { + ptr: address | 1, + _pd: PhantomData, + } + } + + #[inline] + fn is_allocated(&self) -> bool { + self.ptr & 1 == 0 + } + + #[inline] + fn is_null(&self) -> bool { + self.ptr == 0 + } + + #[inline] + fn as_mut(&mut self) -> &'a mut [u8; SIZE] { + if self.is_null() { + panic!("Null block dereference"); + } + + // FIXME: if a non-full block has been marked as CoW, the function will overrun the file + // boundary + if !self.is_allocated() { + // Allocate the block + let ptr = A::alloc().expect("Could not allocate a block").as_ptr() as usize; + // Clone data + let src = self.as_ref(); + let dst = unsafe { core::slice::from_raw_parts_mut(ptr as *mut u8, SIZE) }; + + dst.copy_from_slice(src); + + self.ptr = ptr; + } + + unsafe { &mut *((self.ptr & !1) as *mut [u8; SIZE]) } + } + + #[inline] + fn as_ref(&self) -> &'a [u8; SIZE] { + if self.is_null() { + panic!("Null block dereference"); + } + + unsafe { &*((self.ptr & !1) as *const [u8; SIZE]) } + } +} + +impl<'a, A: BlockAllocator> Drop for BlockRef<'a, A> { + fn drop(&mut self) { + if self.is_allocated() && !self.is_null() { + unsafe { + A::dealloc(NonNull::new_unchecked(self.ptr as *mut _)); + } + } + } +} + impl<'a, A: BlockAllocator> BlockRaw<'a, A> { const fn null() -> Self { Self { - inner: None, - _pd: PhantomData, + inner: BlockRef::null(), } } fn new() -> Result { let ptr = A::alloc()?; - unsafe { Ok(Self::from_raw(ptr)) } - } - - unsafe fn from_raw(ptr: NonNull) -> Self { - Self { - inner: Some(&mut *(ptr.as_ptr() as *mut _)), - _pd: PhantomData, + unsafe { + Ok(Self { + inner: BlockRef::from_allocated(ptr.as_ptr() as _), + }) } } unsafe fn as_uninit_indirect_mut( &mut self, ) -> &'a mut [MaybeUninit>; ENTRY_COUNT] { - unsafe { &mut *(self.inner.as_ref().unwrap().as_ptr() as *mut _) } + if self.inner.is_null() { + panic!("Null block dereference"); + } + + &mut *(self.inner.ptr as *mut _) + } + + #[inline] + unsafe fn as_data_ref(&self) -> &'a [u8; SIZE] { + self.inner.as_ref() + } + + #[inline] + unsafe fn as_data_mut(&mut self) -> &'a mut [u8; SIZE] { + self.inner.as_mut() } #[inline] fn is_null(&self) -> bool { - self.inner.is_none() - } -} - -impl Drop for BlockRaw<'_, A> { - fn drop(&mut self) { - if let Some(inner) = self.inner.take() { - unsafe { - A::dealloc(NonNull::new_unchecked(inner as *mut _)); - } - } + self.inner.is_null() } } @@ -100,6 +181,21 @@ impl<'a, A: BlockAllocator> BlockData<'a, A> { } } + /// Create a Copy-on-Write data block from existing data. + /// + /// # Safety + /// + /// This function is unsafe as it accepts arbitrary pointers. The caller must ensure the + /// address is properly aligned (at least to a u16 boundary), does not cross any device memory + /// and the pointer outlives the block reference. + pub unsafe fn copy_on_write(address: usize) -> Self { + Self { + inner: BlockRaw { + inner: BlockRef::copy_on_write(address), + }, + } + } + /// Allocates a new block for data pub fn new() -> Result { Ok(Self { @@ -123,13 +219,14 @@ impl Deref for BlockData<'_, A> { type Target = [u8; SIZE]; fn deref(&self) -> &Self::Target { - self.inner.inner.as_ref().unwrap() + unsafe { self.inner.as_data_ref() } } } impl DerefMut for BlockData<'_, A> { + #[inline] fn deref_mut(&mut self) -> &mut Self::Target { - self.inner.inner.as_mut().unwrap() + unsafe { self.inner.as_data_mut() } } } @@ -162,13 +259,13 @@ impl<'a, A: BlockAllocator> Deref for BlockIndirect<'a, A> { type Target = [BlockData<'a, A>; ENTRY_COUNT]; fn deref(&self) -> &Self::Target { - unsafe { &*(self.inner.inner.as_ref().unwrap().as_ptr() as *const _) } + unsafe { &*(self.inner.inner.as_ref() as *const _ as *const _) } } } impl<'a, A: BlockAllocator> DerefMut for BlockIndirect<'a, A> { fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { &mut *(self.inner.inner.as_mut().unwrap().as_mut_ptr() as *mut _) } + unsafe { &mut *(self.inner.inner.as_mut() as *mut _ as *mut _) } } } diff --git a/lib/memfs/src/bvec.rs b/lib/memfs/src/bvec.rs index f731115b..92c05adb 100644 --- a/lib/memfs/src/bvec.rs +++ b/lib/memfs/src/bvec.rs @@ -47,6 +47,24 @@ impl<'a, A: BlockAllocator> BVec<'a, A> { } } + /// Initializes the block vector with existing data, marking all blocks as Copy-on-Write + pub fn init_with_cow(&mut self, data: &'static [u8]) -> Result<(), Error> { + let data_ptr = data.as_ptr() as usize; + assert_eq!(data_ptr & 1, 0); + + let blocks = (data.len() + block::SIZE - 1) / block::SIZE; + self.resize(blocks)?; + + for i in 0..blocks { + let src = data_ptr + i * block::SIZE; + let block = unsafe { BlockData::copy_on_write(src) }; + self[i] = block; + } + + self.size = data.len(); + Ok(()) + } + /// Returns the size of the data inside this vector #[inline] pub const fn size(&self) -> usize { @@ -142,10 +160,6 @@ impl<'a, A: BlockAllocator> BVec<'a, A> { /// Writes data to the vector, growing it if needed pub fn write(&mut self, pos: u64, data: &[u8]) -> Result { let mut pos = pos as usize; - // if pos > self.size { - // return Err(Error::InvalidFile); - // } - let mut rem = data.len(); let mut doff = 0usize; @@ -269,12 +283,12 @@ impl<'a, A: BlockAllocator> IndexMut for BVec<'a, A> { } } -impl<'a, A: BlockAllocator> TryFrom<&[u8]> for BVec<'a, A> { +impl<'a, A: BlockAllocator> TryFrom<&'static [u8]> for BVec<'a, A> { type Error = Error; - fn try_from(value: &[u8]) -> Result { + fn try_from(value: &'static [u8]) -> Result { let mut res = Self::new(); - res.write(0, value)?; + res.init_with_cow(value)?; assert_eq!(res.size(), value.len()); Ok(res) } diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index 1215211e..853193c4 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -197,12 +197,24 @@ impl MemoryFilesystem { #[cfg(test)] mod tests { + use core::sync::atomic::Ordering; use std::rc::Rc; use vfs::{Filesystem, IoContext, Read, Seek, VnodeKind, Write}; use yggdrasil_abi::io::{FileMode, OpenOptions, SeekFrom}; use crate::MemoryFilesystem; + #[repr(C)] + struct AlignedTo { + _align: [Align; 0], + bytes: Bytes, + } + + static TEST_IMAGE: &'static AlignedTo = &AlignedTo { + _align: [], + bytes: *include_bytes!("../test/test_image.tar"), + }; + #[test] fn test_memfs_construction() { fn check_file(ioctx: &IoContext, path: &str, expected_data: &str) { @@ -222,12 +234,14 @@ mod tests { assert_eq!(&buf[..expected_data.len()], expected_data.as_bytes()); } - static TEST_IMAGE: &[u8] = include_bytes!("../test/test_image.tar"); test_allocator_with_counter!(A_COUNTER, A); - let fs = MemoryFilesystem::::from_slice(TEST_IMAGE).unwrap(); + let fs = MemoryFilesystem::::from_slice(&TEST_IMAGE.bytes).unwrap(); let root = fs.root().unwrap(); + // Files are small, so no indirect blocks allocated + assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); + let ioctx = IoContext::new(root.clone()); assert!(Rc::ptr_eq( @@ -235,7 +249,30 @@ mod tests { &ioctx.find(None, "/", false, false).unwrap() )); - check_file(&ioctx, "/test1.txt", include_str!("../test/test1.txt")); + let old_data = include_str!("../test/test1.txt"); + check_file(&ioctx, "/test1.txt", old_data); + + // Write to the file + { + let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); + let file = node.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); + + assert_eq!(file.borrow_mut().write(b"Hello").unwrap(), 5); + } + + assert_eq!(A_COUNTER.load(Ordering::Acquire), 1); + + // Read back + { + let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); + let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + + let mut buf = [0; 512]; + assert_eq!(file.borrow_mut().read(&mut buf).unwrap(), old_data.len()); + + assert_eq!(&buf[..5], b"Hello"); + assert_eq!(&buf[5..old_data.len()], &old_data.as_bytes()[5..]); + } } #[test] diff --git a/src/fs/mod.rs b/src/fs/mod.rs index ae792952..513231a2 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -44,7 +44,7 @@ unsafe impl BlockAllocator for FileBlockAllocator { unsafe fn dealloc(block: NonNull) { let page = block.as_ptr() as usize; assert!(page > mem::KERNEL_VIRT_OFFSET); - todo!("Release physical memory"); + phys::free_page(page.physicalize()); } } diff --git a/src/main.rs b/src/main.rs index 603b4466..7fdb32e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,8 +58,6 @@ fn setup_root() -> Result { pub fn kernel_main() { infoln!("In main"); - panic!(); - #[cfg(feature = "fb_console")] { use core::time::Duration; From b0379e398fecfd3c0cbbafcf0534fc2ca826ab27 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 18 Aug 2023 22:53:58 +0300 Subject: [PATCH 054/211] aarch64: move some code into "common" kernel --- src/arch/aarch64/boot/mod.rs | 65 ++++++++++++++----- src/arch/aarch64/devtree.rs | 1 + src/arch/aarch64/exception.rs | 1 - src/arch/aarch64/mod.rs | 91 ++++---------------------- src/arch/aarch64/table.rs | 16 ++++- src/arch/mod.rs | 8 +++ src/init.rs | 79 +++++++++++++++++++++++ src/main.rs | 118 ++++++++++++++-------------------- src/task/mod.rs | 2 - 9 files changed, 215 insertions(+), 166 deletions(-) create mode 100644 src/init.rs diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 881102e4..ac1ec5e3 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -4,20 +4,25 @@ use core::{ sync::atomic::Ordering, }; -use aarch64_cpu::{asm::barrier, registers::CPACR_EL1}; -use tock_registers::interfaces::ReadWriteable; +use aarch64_cpu::{ + asm::barrier, + registers::{CPACR_EL1, TTBR0_EL1}, +}; +use tock_registers::interfaces::{ReadWriteable, Writeable}; -use super::{cpu::Cpu, exception, kernel_main, KernelStack, ARCHITECTURE, BOOT_STACK_SIZE}; +use super::{cpu::Cpu, exception, AArch64, KernelStack, ARCHITECTURE, BOOT_STACK_SIZE}; use crate::{ absolute_address, arch::{aarch64::smp::CPU_COUNT, Architecture, ArchitectureImpl}, - mem::{ConvertAddress, KERNEL_VIRT_OFFSET}, - sync::SpinFence, - task, + fs::devfs, + kernel_main, kernel_secondary_main, + mem::{ + heap, + phys::{self, PageUsage}, + ConvertAddress, KERNEL_VIRT_OFFSET, + }, }; -pub(super) static CPU_INIT_FENCE: SpinFence = SpinFence::new(); - extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { // Unmask FP operations CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); @@ -60,8 +65,42 @@ fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { } } -extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { - kernel_main(dtb_phys); +unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { + // NOTE it is critical that the code does not panic until the debug is set up, otherwise no + // message will be displayed + + // Unmap TTBR0 + TTBR0_EL1.set(0); + barrier::isb(barrier::SY); + + ARCHITECTURE.init_device_tree(dtb_phys); + + AArch64::set_interrupt_mask(true); + + exception::init_exceptions(); + + // Setup initrd + super::setup_initrd(); + + ARCHITECTURE + .init_physical_memory(dtb_phys) + .expect("Failed to initialize the physical memory manager"); + + // Setup heap + let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) + .expect("Could not allocate a block for heap"); + + heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + + devfs::init(); + + // Enumerate the device tree + ARCHITECTURE.init_platform(true); + + Cpu::init_local(); + + kernel_main() + // kernel_main(dtb_phys) } extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { @@ -79,11 +118,7 @@ extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { Cpu::init_local(); - // Synchronize the CPUs to this point - CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); - - task::enter(); + kernel_secondary_main() } } diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index 0fdc95b4..19cf91b4 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -54,6 +54,7 @@ pub trait DevTreeIndexNodeExt { /// Returns the #size-cells property of the node, if there is one fn get_size_cells(&self) -> Option; + /// Prints the node contents to debug output fn dump(&self, level: LogLevel, depth: usize); } diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 26aae24b..4bd89603 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -21,7 +21,6 @@ use crate::{ Architecture, ArchitectureImpl, }, debug::LogLevel, - mem::KERNEL_VIRT_OFFSET, syscall::raw_syscall_handler, task::{context::TaskFrame, process::Process}, }; diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index c6ade41c..b2834c36 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -15,31 +15,25 @@ use device_api::{ ExternalInterruptController, IpiDeliveryTarget, IrqNumber, LocalInterruptController, }, timer::MonotonicTimestampProviderDevice, - CpuBringupDevice, ResetDevice, + ResetDevice, }; use fdt_rs::prelude::PropReader; use git_version::git_version; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::{ - aarch64::{cpu::Cpu, devtree::FdtMemoryRegionIter}, - Architecture, - }, + arch::{aarch64::devtree::FdtMemoryRegionIter, Architecture}, debug::{self}, device::{self, power::arm_psci::Psci, DevTreeNodeInfo}, - fs::{devfs, Initrd, INITRD_DATA}, + fs::{Initrd, INITRD_DATA}, mem::{ - heap, - phys::{self, reserved::reserve_region, PageUsage, PhysicalMemoryRegion}, + phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, ConvertAddress, }, - task, util::OneTimeInit, }; use self::{ - boot::CPU_INIT_FENCE, devtree::{DevTreeIndexPropExt, DeviceTree}, smp::CPU_COUNT, table::{init_fixed_tables, KERNEL_TABLES}, @@ -135,6 +129,16 @@ impl Architecture for AArch64 { barrier::isb(barrier::SY); } + unsafe fn start_application_processors(&self) { + let dt = self.dt.get(); + if let Err(e) = smp::start_ap_cores(dt) { + errorln!( + "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", + e + ); + } + } + fn map_device_pages(&self, phys: usize, count: usize) -> Result { unsafe { KERNEL_TABLES.map_device_pages(phys, count) } } @@ -416,70 +420,3 @@ fn setup_initrd() { INITRD_DATA.init(initrd); } - -/// AArch64 kernel main entry point -pub fn kernel_main(dtb_phys: usize) -> ! { - // NOTE it is critical that the code does not panic until the debug is set up, otherwise no - // message will be displayed - - // Unmap TTBR0 - TTBR0_EL1.set(0); - barrier::isb(barrier::SY); - - unsafe { - ARCHITECTURE.init_device_tree(dtb_phys); - - AArch64::set_interrupt_mask(true); - - exception::init_exceptions(); - - // Setup initrd - setup_initrd(); - - ARCHITECTURE - .init_physical_memory(dtb_phys) - .expect("Failed to initialize the physical memory manager"); - - // Setup heap - let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) - .expect("Could not allocate a block for heap"); - heap::init_heap(heap_base.virtualize(), 16 * 0x1000); - - devfs::init(); - - // Enumerate the device tree - ARCHITECTURE.init_platform(true); - - barrier::isb(barrier::SY); - SCTLR_EL1.modify( - SCTLR_EL1::SA::Enable - + SCTLR_EL1::A::Enable - + SCTLR_EL1::I::NonCacheable - + SCTLR_EL1::C::NonCacheable, - ); - barrier::isb(barrier::SY); - - debugln!("Heap: {:#x?}", heap::heap_range()); - - Cpu::init_local(); - - let dt = ARCHITECTURE.dt.get(); - if let Err(e) = smp::start_ap_cores(dt) { - errorln!( - "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", - e - ); - } - - Cpu::init_ipi_queues(); - - CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); - - task::init().expect("Failed to initialize the scheduler"); - - infoln!("All cpus ready"); - - task::enter(); - } -} diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs index 384c64a6..1690ea38 100644 --- a/src/arch/aarch64/table.rs +++ b/src/arch/aarch64/table.rs @@ -51,8 +51,6 @@ impl NonTerminalEntryLevel for L2 { } bitflags! { - /// TODO split attrs for different translation levels - /// /// Describes how each page is mapped: access, presence, type of the mapping. #[derive(Clone, Copy)] pub struct PageAttributes: u64 { @@ -76,10 +74,14 @@ bitflags! { /// For page/block mappings, only allows read access for EL0/EL1 const AP_BOTH_READONLY = 3 << 6; + /// Indicates outer shareability domain const SH_OUTER = 2 << 8; + /// Indicates inner shareability domain const SH_INNER = 3 << 8; + /// For page/block mappings, indicates regular physical memory const PAGE_ATTR_NORMAL = 0 << 2; + /// For page/block mappings, indicates device memory const PAGE_ATTR_DEVICE = 1 << 2; /// Indicates the mapping is unique to a specific ASID (important for proper TLB @@ -131,7 +133,7 @@ pub struct FixedTables { } impl PageEntry { - /// Creates a 4KiB page mapping + /// Creates a 4KiB normal memory page mapping pub fn normal_page(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) @@ -146,6 +148,7 @@ impl PageEntry { ) } + /// Creates a 4KiB device memory page mapping pub fn device_page(phys: usize, attrs: PageAttributes) -> Self { Self( (phys as u64) @@ -427,6 +430,11 @@ impl AddressSpace { Ok(Self { l1, asid }) } + /// Interprets a physical address as an address space structure pointer. + /// + /// # Safety + /// + /// Unsafe: accepts arbitrary addresses and ignores ASIDs. pub unsafe fn from_phys_raw(value: usize) -> Self { let value = value.virtualize(); Self { @@ -435,6 +443,8 @@ impl AddressSpace { } } + /// Iterates through all the translation levels for a given address, invoking the callback + /// function on each of them pub fn walk bool>(&self, vaddr: usize, f: F) { let l1i = L1::index(vaddr); let l2i = L2::index(vaddr); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index b30e5437..34d4b2bc 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -67,6 +67,13 @@ pub trait Architecture { /// Unsafe to call if the MMU has already been initialized. unsafe fn init_mmu(&self, bsp: bool); + /// Starts up the application processors that may be present in the system. + /// + /// # Safety + /// + /// Only safe to call once during system init. + unsafe fn start_application_processors(&self); + /// Allocates a virtual mapping for the specified physical memory region fn map_device_pages(&self, phys: usize, count: usize) -> Result; @@ -107,6 +114,7 @@ pub trait Architecture { timer: &'static dyn MonotonicTimestampProviderDevice, ) -> Result<(), Error>; + /// Adds a reset device to the system fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error>; // TODO only supports 1 extintc per system diff --git a/src/init.rs b/src/init.rs new file mode 100644 index 00000000..b61f7456 --- /dev/null +++ b/src/init.rs @@ -0,0 +1,79 @@ +//! Kernel main process implementation: filesystem initialization and userspace init start +use abi::{ + error::Error, + io::{FileMode, OpenOptions, RawFd}, +}; +use memfs::MemoryFilesystem; +use vfs::{Filesystem, IoContext, VnodeRef}; + +use crate::{ + fs::{devfs, FileBlockAllocator, INITRD_DATA}, + proc, +}; + +fn setup_root() -> Result { + let initrd_data = INITRD_DATA.get(); + let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); + fs.root() +} + +/// Kernel's "main" process function. +/// +/// # Note +/// +/// This function is meant to be used as a kernel-space process after all the platform-specific +/// initialization has finished. +pub fn kinit() { + infoln!("In main"); + + #[cfg(feature = "fb_console")] + { + use core::time::Duration; + + use device::display::console::task_update_consoles; + use task::tasklet; + + tasklet::add_periodic( + "update-console", + Duration::from_millis(15), + task_update_consoles, + ); + } + + let root = match setup_root() { + Ok(root) => root, + Err(err) => { + warnln!("Could not setup root from initrd: {:?}", err); + return; + } + }; + + let ioctx = IoContext::new(root); + let node = ioctx.find(None, "/init", true, true).unwrap(); + let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + + let devfs = devfs::root(); + #[cfg(target_arch = "x86_64")] + let console = ioctx.find(Some(devfs.clone()), "tty0", true, true).unwrap(); + #[cfg(target_arch = "aarch64")] + let console = ioctx + .find(Some(devfs.clone()), "ttyS0", true, true) + .unwrap(); + let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); + let stderr = stdout.clone(); + + { + let user_init = proc::exec::load_elf("init", file, &["/init", "xxx"]).unwrap(); + let mut io = user_init.io.lock(); + io.set_ioctx(ioctx); + io.set_file(RawFd::STDIN, stdin).unwrap(); + io.set_file(RawFd::STDOUT, stdout).unwrap(); + io.set_file(RawFd::STDERR, stderr).unwrap(); + drop(io); + + user_init.set_session_terminal(console); + + user_init.enqueue_somewhere(); + } +} diff --git a/src/main.rs b/src/main.rs index 7fdb32e8..6e7511fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,13 +16,13 @@ #![no_std] #![no_main] -use abi::{ - error::Error, - io::{FileMode, OpenOptions, RawFd}, +use sync::SpinFence; +use task::{context::Cpu, spawn_kernel_closure}; + +use crate::{ + arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, + mem::heap, }; -use fs::{devfs, FileBlockAllocator, INITRD_DATA}; -use memfs::MemoryFilesystem; -use vfs::{Filesystem, IoContext, VnodeRef}; extern crate yggdrasil_abi as abi; @@ -35,6 +35,7 @@ pub mod arch; pub mod device; pub mod fs; +pub mod init; pub mod mem; pub mod panic; pub mod proc; @@ -43,69 +44,50 @@ pub mod syscall; pub mod task; pub mod util; -fn setup_root() -> Result { - let initrd_data = INITRD_DATA.get(); - let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); - fs.root() -} +static CPU_INIT_FENCE: SpinFence = SpinFence::new(); -/// Entry point for common kernel code. -/// -/// # Note -/// -/// This function is meant to be used as a kernel-space process after all the platform-specific -/// initialization has finished. -pub fn kernel_main() { - infoln!("In main"); +/// Common kernel main function for application processors +pub fn kernel_secondary_main() -> ! { + // Synchronize the CPUs to this point + CPU_INIT_FENCE.signal(); + CPU_INIT_FENCE.wait_all(ArchitectureImpl::cpu_count()); - #[cfg(feature = "fb_console")] - { - use core::time::Duration; - - use device::display::console::task_update_consoles; - use task::tasklet; - - tasklet::add_periodic( - "update-console", - Duration::from_millis(15), - task_update_consoles, - ); - } - - let root = match setup_root() { - Ok(root) => root, - Err(err) => { - warnln!("Could not setup root from initrd: {:?}", err); - return; - } - }; - - let ioctx = IoContext::new(root); - let node = ioctx.find(None, "/init", true, true).unwrap(); - let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); - - let devfs = devfs::root(); - #[cfg(target_arch = "x86_64")] - let console = ioctx.find(Some(devfs.clone()), "tty0", true, true).unwrap(); - #[cfg(target_arch = "aarch64")] - let console = ioctx - .find(Some(devfs.clone()), "ttyS0", true, true) - .unwrap(); - let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); - let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); - let stderr = stdout.clone(); - - { - let user_init = proc::exec::load_elf("init", file, &["/init", "xxx"]).unwrap(); - let mut io = user_init.io.lock(); - io.set_ioctx(ioctx); - io.set_file(RawFd::STDIN, stdin).unwrap(); - io.set_file(RawFd::STDOUT, stdout).unwrap(); - io.set_file(RawFd::STDERR, stderr).unwrap(); - drop(io); - - user_init.set_session_terminal(console); - - user_init.enqueue_somewhere(); + unsafe { + task::enter(); + } +} + +/// Common kernel main function. Must be called for BSP processor only. +/// +/// # Prerequisites +/// +/// Before the function can be called, the following preparations must be made: +/// +/// * Virtual memory set up according to the architecture's memory map +/// * Physical memory +/// * Heap +/// * Basic debugging facilities +/// * Initrd +pub fn kernel_main() -> ! { + debugln!("Heap: {:#x?}", heap::heap_range()); + + unsafe { + ARCHITECTURE.start_application_processors(); + } + + Cpu::init_ipi_queues(); + + // Wait until all APs initialize + CPU_INIT_FENCE.signal(); + CPU_INIT_FENCE.wait_all(ArchitectureImpl::cpu_count()); + + task::init().expect("Failed to initialize the scheduler"); + + spawn_kernel_closure("[kinit]", init::kinit).expect("Could not spawn [kinit]"); + + infoln!("All cpus ready"); + + unsafe { + task::enter(); } } diff --git a/src/task/mod.rs b/src/task/mod.rs index 82dd505a..51c44f02 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -7,7 +7,6 @@ use alloc::{rc::Rc, string::String, vec::Vec}; use crate::{ arch::{Architecture, ArchitectureImpl}, - kernel_main, sync::{IrqSafeSpinlock, SpinFence}, task::sched::CpuQueue, }; @@ -81,7 +80,6 @@ pub fn init() -> Result<(), Error> { // Create a queue for each CPU sched::init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); - spawn_kernel_closure("[kmain]", kernel_main)?; // spawn_kernel_closure(move || loop { // debugln!("B"); // for _ in 0..100000 { From af5529a84fccf84fa92a58101e37ad9d37ec372b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 21 Aug 2023 17:26:44 +0300 Subject: [PATCH 055/211] x86-64: sync x86-64 up with the new kernel structure --- Cargo.toml | 7 +- lib/device-api/src/input.rs | 11 + lib/device-api/src/interrupt.rs | 25 +- lib/device-api/src/lib.rs | 7 + lib/kernel-util/Cargo.toml | 8 + lib/kernel-util/src/lib.rs | 5 + lib/kernel-util/src/sync.rs | 0 {src => lib/kernel-util/src}/util.rs | 0 lib/memfs/src/dir.rs | 19 +- lib/memfs/src/file.rs | 26 +- lib/memfs/src/lib.rs | 4 +- lib/vfs/Cargo.toml | 1 + lib/vfs/src/char.rs | 17 +- lib/vfs/src/node.rs | 64 +-- src/arch/aarch64/boot/mod.rs | 1 - src/arch/aarch64/cpu.rs | 5 +- src/arch/aarch64/devtree.rs | 299 ------------- src/arch/aarch64/gic/mod.rs | 12 +- src/arch/aarch64/mod.rs | 46 +- src/arch/aarch64/smp.rs | 2 +- src/arch/aarch64/timer.rs | 8 +- src/arch/mod.rs | 49 ++- src/arch/x86_64/apic/ioapic.rs | 154 +++---- src/arch/x86_64/apic/local.rs | 77 ++-- src/arch/x86_64/apic/mod.rs | 45 +- src/arch/x86_64/boot/mod.rs | 90 ++-- src/arch/x86_64/cpu.rs | 6 +- src/arch/x86_64/mod.rs | 583 ++++++++++++++----------- src/arch/x86_64/peripherals/hpet.rs | 78 ++-- src/arch/x86_64/peripherals/ps2/mod.rs | 86 ++-- src/arch/x86_64/peripherals/serial.rs | 34 +- src/debug.rs | 3 +- src/device/bus/mod.rs | 1 + src/device/bus/simple_bus.rs | 6 +- src/device/devtree.rs | 454 +++++++++++++++++++ src/device/display/console.rs | 40 +- src/device/display/fb_console.rs | 13 +- src/device/display/linear_fb.rs | 5 +- src/device/display/mod.rs | 2 + src/device/mod.rs | 168 +------ src/device/power/arm_psci.rs | 6 +- src/device/power/mod.rs | 10 +- src/device/power/sunxi_rwdog.rs | 8 +- src/device/serial/mod.rs | 10 +- src/device/serial/pl011.rs | 18 +- src/device/serial/sunxi_uart.rs | 18 +- src/device/timer.rs | 13 - src/device/tty.rs | 25 +- src/fs/devfs.rs | 12 +- src/fs/mod.rs | 12 +- src/init.rs | 4 +- src/main.rs | 9 +- src/mem/mod.rs | 58 +-- src/mem/phys/mod.rs | 2 +- src/mem/phys/reserved.rs | 2 +- src/task/process.rs | 2 +- src/task/sched.rs | 2 +- 57 files changed, 1357 insertions(+), 1315 deletions(-) create mode 100644 lib/device-api/src/input.rs create mode 100644 lib/kernel-util/Cargo.toml create mode 100644 lib/kernel-util/src/lib.rs create mode 100644 lib/kernel-util/src/sync.rs rename {src => lib/kernel-util/src}/util.rs (100%) create mode 100644 src/device/devtree.rs delete mode 100644 src/device/timer.rs diff --git a/Cargo.toml b/Cargo.toml index 7d68103e..004e61c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } memfs = { path = "lib/memfs" } device-api = { path = "lib/device-api", features = ["derive"] } +kernel-util = { path = "lib/kernel-util" } atomic_enum = "0.2.0" bitflags = "2.3.3" @@ -21,12 +22,10 @@ tock-registers = "0.8.1" cfg-if = "1.0.0" git-version = "0.3.5" -aarch64-cpu = "9.3.1" - bitmap-font = { version = "0.3.0", optional = true } embedded-graphics = { version = "0.8.0", optional = true } -fdt-rs = { version = "0.4.3", default-features = false } +log = "0.4.20" [dependencies.elf] version = "0.7.2" @@ -35,6 +34,8 @@ default-features = false features = ["no_std_stream"] [target.'cfg(target_arch = "aarch64")'.dependencies] +aarch64-cpu = "9.3.1" +fdt-rs = { version = "0.4.3", default-features = false } [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } diff --git a/lib/device-api/src/input.rs b/lib/device-api/src/input.rs new file mode 100644 index 00000000..2df3fbc3 --- /dev/null +++ b/lib/device-api/src/input.rs @@ -0,0 +1,11 @@ +use yggdrasil_abi::error::Error; + +use crate::Device; + +pub trait KeyboardConsumer { + fn handle_character(&self, data: u8) -> Result<(), Error>; +} + +pub trait KeyboardProducer: Device { + fn connect(&self, to: &'static dyn KeyboardConsumer); +} diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index decc14c8..618f34dc 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -27,11 +27,11 @@ pub enum IpiDeliveryTarget { OtherCpus, } -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum IrqNumber { - Private(u32), - Shared(u32), -} +// #[derive(Clone, Copy, PartialEq, Eq, Debug)] +// pub enum IrqNumber { +// Private(u32), +// Shared(u32), +// } #[derive(Default, Clone, Copy, Debug)] pub struct IrqOptions { @@ -44,17 +44,19 @@ pub trait InterruptTable { } pub trait ExternalInterruptController { + type IrqNumber; + /// Performs IRQ delivery method configuration and registers a handler to execute when it is /// fired fn register_irq( &self, - irq: IrqNumber, + irq: Self::IrqNumber, options: IrqOptions, handler: &'static dyn InterruptHandler, ) -> Result<(), Error>; /// Enables the specified IRQ (unmasks it) - fn enable_irq(&self, irq: IrqNumber) -> Result<(), Error>; + fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error>; /// Handles a single pending interrupt on this controller. /// The function is intended for interrupt controllers which have internal registers to track @@ -107,6 +109,15 @@ impl FixedInterruptTable { self.entries[index] = Some(handler); Ok(()) } + + pub fn insert_least_loaded( + &mut self, + handler: &'static dyn InterruptHandler, + ) -> Result { + let index = self.entries.iter().position(|p| p.is_none()).unwrap(); + self.entries[index].replace(handler); + Ok(index) + } } impl InterruptTable for FixedInterruptTable { diff --git a/lib/device-api/src/lib.rs b/lib/device-api/src/lib.rs index 32427ea0..927b1f45 100644 --- a/lib/device-api/src/lib.rs +++ b/lib/device-api/src/lib.rs @@ -4,6 +4,7 @@ extern crate alloc; pub mod bus; pub mod device; +pub mod input; pub mod interrupt; pub mod manager; pub mod serial; @@ -24,5 +25,11 @@ pub trait CpuBringupDevice: Device { } pub trait ResetDevice: Device { + /// Performs a system reset. + /// + /// # Safety + /// + /// The kernel must ensure it is actually safe to perform a reset, no critical operations + /// are aborted and no data is lost. unsafe fn reset(&self) -> !; } diff --git a/lib/kernel-util/Cargo.toml b/lib/kernel-util/Cargo.toml new file mode 100644 index 00000000..496f8d0d --- /dev/null +++ b/lib/kernel-util/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "kernel-util" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs new file mode 100644 index 00000000..06f7b372 --- /dev/null +++ b/lib/kernel-util/src/lib.rs @@ -0,0 +1,5 @@ +#![no_std] +#![feature(maybe_uninit_slice)] + +pub mod sync; +pub mod util; diff --git a/lib/kernel-util/src/sync.rs b/lib/kernel-util/src/sync.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/util.rs b/lib/kernel-util/src/util.rs similarity index 100% rename from src/util.rs rename to lib/kernel-util/src/util.rs diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs index bb7571ba..c9280b88 100644 --- a/lib/memfs/src/dir.rs +++ b/lib/memfs/src/dir.rs @@ -1,4 +1,4 @@ -use core::marker::PhantomData; +use core::{cell::RefCell, marker::PhantomData}; use alloc::boxed::Box; @@ -15,12 +15,12 @@ pub(crate) struct DirectoryNode { } impl VnodeImpl for DirectoryNode { - fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + fn create(&self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { let child = Vnode::new(name, kind); match kind { VnodeKind::Directory => child.set_data(Box::new(Self { _pd: PhantomData })), VnodeKind::Regular => child.set_data(Box::new(FileNode { - data: BVec::::new(), + data: RefCell::new(BVec::::new()), })), _ => todo!(), } @@ -28,24 +28,19 @@ impl VnodeImpl for DirectoryNode { Ok(child) } - fn open( - &mut self, - _node: &VnodeRef, - _opts: OpenOptions, - _mode: FileMode, - ) -> Result { + fn open(&self, _node: &VnodeRef, _opts: OpenOptions, _mode: FileMode) -> Result { Ok(DIR_POSITION_FROM_CACHE) } - fn remove(&mut self, _at: &VnodeRef, _name: &str) -> Result<(), Error> { + fn remove(&self, _at: &VnodeRef, _name: &str) -> Result<(), Error> { Ok(()) } - fn size(&mut self, node: &VnodeRef) -> Result { + fn size(&self, node: &VnodeRef) -> Result { Ok(node.children().len() as u64) } - fn metadata(&mut self, _node: &VnodeRef) -> Result { + fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { size: 0, mode: FileMode::default_dir(), diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs index 2ac22277..bc02276b 100644 --- a/lib/memfs/src/file.rs +++ b/lib/memfs/src/file.rs @@ -1,3 +1,5 @@ +use core::cell::RefCell; + use vfs::{VnodeImpl, VnodeRef}; use yggdrasil_abi::{ error::Error, @@ -7,37 +9,37 @@ use yggdrasil_abi::{ use crate::{block::BlockAllocator, bvec::BVec}; pub(crate) struct FileNode { - pub(crate) data: BVec<'static, A>, + pub(crate) data: RefCell>, } impl VnodeImpl for FileNode { - fn open(&mut self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { + fn open(&self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { if opts.contains(OpenOptions::APPEND) { - Ok(self.data.size() as u64) + Ok(self.data.borrow().size() as u64) } else { Ok(0) } } - fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + fn close(&self, _node: &VnodeRef) -> Result<(), Error> { Ok(()) } - fn read(&mut self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { - self.data.read(pos, data) + fn read(&self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + self.data.borrow().read(pos, data) } - fn write(&mut self, _node: &VnodeRef, pos: u64, data: &[u8]) -> Result { - self.data.write(pos, data) + fn write(&self, _node: &VnodeRef, pos: u64, data: &[u8]) -> Result { + self.data.borrow_mut().write(pos, data) } - fn size(&mut self, _node: &VnodeRef) -> Result { - Ok(self.data.size() as u64) + fn size(&self, _node: &VnodeRef) -> Result { + Ok(self.data.borrow().size() as u64) } - fn metadata(&mut self, _node: &VnodeRef) -> Result { + fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { - size: self.data.size() as u64, + size: self.data.borrow().size() as u64, mode: FileMode::default_file(), ty: FileType::File, }) diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index 853193c4..94d2b5d9 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -162,7 +162,9 @@ impl MemoryFilesystem { let bvec = BVec::::try_from(data)?; assert_eq!(bvec.size(), data.len()); - node.set_data(Box::new(FileNode { data: bvec })); + node.set_data(Box::new(FileNode { + data: RefCell::new(bvec), + })); } } diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 42c6b09a..e0b452a7 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../kernel-util" } bitflags = "2.3.3" diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index 70774490..bb44a524 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -23,28 +23,23 @@ impl CharDeviceWrapper { } impl VnodeImpl for CharDeviceWrapper { - fn open( - &mut self, - _node: &VnodeRef, - _opts: OpenOptions, - _mode: FileMode, - ) -> Result { + fn open(&self, _node: &VnodeRef, _opts: OpenOptions, _mode: FileMode) -> Result { Ok(0) } - fn close(&mut self, _node: &VnodeRef) -> Result<(), Error> { + fn close(&self, _node: &VnodeRef) -> Result<(), Error> { Ok(()) } - fn read(&mut self, _node: &VnodeRef, _pos: u64, data: &mut [u8]) -> Result { + fn read(&self, _node: &VnodeRef, _pos: u64, data: &mut [u8]) -> Result { self.device.read(true, data) } - fn write(&mut self, _node: &VnodeRef, _pos: u64, data: &[u8]) -> Result { + fn write(&self, _node: &VnodeRef, _pos: u64, data: &[u8]) -> Result { self.device.write(true, data) } - fn metadata(&mut self, _node: &VnodeRef) -> Result { + fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { size: 0, ty: FileType::Char, @@ -52,7 +47,7 @@ impl VnodeImpl for CharDeviceWrapper { }) } - fn device_request(&mut self, _node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { + fn device_request(&self, _node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { self.device.device_request(req) } } diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index 53921e5f..d32735c4 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -9,6 +9,7 @@ use alloc::{ string::String, vec::Vec, }; +use kernel_util::util::OneTimeInit; use yggdrasil_abi::{ error::Error, io::{DeviceRequest, FileAttr, FileMode, FileType, OpenOptions}, @@ -44,46 +45,46 @@ pub struct Vnode { name: String, tree: RefCell, kind: VnodeKind, - data: RefCell>>, + data: OneTimeInit>, fs: RefCell>>, target: RefCell>, } #[allow(unused_variables)] pub trait VnodeImpl { - fn create(&mut self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + fn create(&self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { Err(Error::NotImplemented) } - fn remove(&mut self, at: &VnodeRef, name: &str) -> Result<(), Error> { + fn remove(&self, at: &VnodeRef, name: &str) -> Result<(), Error> { Err(Error::NotImplemented) } - fn open(&mut self, node: &VnodeRef, opts: OpenOptions, mode: FileMode) -> Result { + fn open(&self, node: &VnodeRef, opts: OpenOptions, mode: FileMode) -> Result { Err(Error::NotImplemented) } - fn close(&mut self, node: &VnodeRef) -> Result<(), Error> { + fn close(&self, node: &VnodeRef) -> Result<(), Error> { Err(Error::NotImplemented) } - fn read(&mut self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + fn read(&self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { Err(Error::NotImplemented) } - fn write(&mut self, node: &VnodeRef, pos: u64, data: &[u8]) -> Result { + fn write(&self, node: &VnodeRef, pos: u64, data: &[u8]) -> Result { Err(Error::NotImplemented) } - fn size(&mut self, node: &VnodeRef) -> Result { + fn size(&self, node: &VnodeRef) -> Result { Err(Error::NotImplemented) } - fn metadata(&mut self, node: &VnodeRef) -> Result { + fn metadata(&self, node: &VnodeRef) -> Result { Err(Error::NotImplemented) } - fn device_request(&mut self, node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { + fn device_request(&self, node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { Err(Error::NotImplemented) } } @@ -97,7 +98,7 @@ impl Vnode { children: Vec::new(), }), kind, - data: RefCell::new(None), + data: OneTimeInit::new(), fs: RefCell::new(None), target: RefCell::new(None), }) @@ -119,14 +120,18 @@ impl Vnode { } #[inline] - pub fn data(&self) -> RefMut>> { - match self.data.try_borrow_mut() { - Ok(r) => r, - Err(e) => { - panic!("{:?} on {:?}", e, self.name()) - } - } + pub fn data(&self) -> Option<&dyn VnodeImpl> { + self.data.try_get().map(Box::as_ref) } + // #[inline] + // pub fn data(&self) -> RefMut>> { + // match self.data.try_borrow_mut() { + // Ok(r) => r, + // Err(e) => { + // panic!("{:?} on {:?}", e, self.name()) + // } + // } + // } #[inline] pub fn fs(&self) -> Option> { @@ -151,7 +156,8 @@ impl Vnode { } pub fn set_data(&self, data: Box) { - self.data.borrow_mut().replace(data); + self.data.init(data); + // self.data.borrow_mut().replace(data); } pub fn set_fs(&self, data: Rc) { @@ -273,7 +279,7 @@ impl Vnode { return Err(Error::IsADirectory); } - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { let pos = data.open(self, flags, mode)?; Ok(File::normal(self.clone(), pos, open_flags)) } else { @@ -286,7 +292,7 @@ impl Vnode { return Err(Error::IsADirectory); } - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { let pos = data.open(self, OpenOptions::READ, FileMode::empty())?; Ok(File::directory(self.clone(), pos)) } else { @@ -296,7 +302,7 @@ impl Vnode { } pub fn close(self: &VnodeRef) -> Result<(), Error> { - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.close(self) } else { Ok(()) @@ -317,7 +323,7 @@ impl Vnode { e => return e, }; - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { let vnode = data.create(self, name, kind)?; if let Some(fs) = self.fs() { vnode.set_fs(fs); @@ -350,7 +356,7 @@ impl Vnode { } } - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.remove(self, name)?; } @@ -365,7 +371,7 @@ impl Vnode { todo!(); } - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.write(self, pos, buf) } else { todo!() @@ -377,7 +383,7 @@ impl Vnode { todo!(); } - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.read(self, pos, buf) } else { todo!() @@ -385,7 +391,7 @@ impl Vnode { } pub fn size(self: &VnodeRef) -> Result { - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.size(self) } else { todo!(); @@ -435,7 +441,7 @@ impl Vnode { } pub fn metadata(self: &VnodeRef) -> Result { - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.metadata(self) } else { todo!() @@ -443,7 +449,7 @@ impl Vnode { } pub fn device_request(self: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { - if let Some(ref mut data) = *self.data() { + if let Some(data) = self.data() { data.device_request(self, req) } else { todo!() diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index ac1ec5e3..c209e74b 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -100,7 +100,6 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { Cpu::init_local(); kernel_main() - // kernel_main(dtb_phys) } extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 3a3f8cec..8ab4463a 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -3,11 +3,10 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; +use kernel_util::util::OneTimeInit; use tock_registers::interfaces::{Readable, Writeable}; -use crate::{ - arch::CpuMessage, panic, sync::IrqSafeSpinlock, task::sched::CpuQueue, util::OneTimeInit, -}; +use crate::{arch::CpuMessage, panic, sync::IrqSafeSpinlock, task::sched::CpuQueue}; use super::smp::CPU_COUNT; diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs index 19cf91b4..e69de29b 100644 --- a/src/arch/aarch64/devtree.rs +++ b/src/arch/aarch64/devtree.rs @@ -1,299 +0,0 @@ -//! ARM device tree utlities -use fdt_rs::{ - base::DevTree, - index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, - prelude::PropReader, -}; - -use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; - -const INDEX_BUFFER_SIZE: usize = 65536; - -#[repr(C, align(0x10))] -struct FdtIndexBuffer([u8; INDEX_BUFFER_SIZE]); - -static mut FDT_INDEX_BUFFER: FdtIndexBuffer = FdtIndexBuffer::zeroed(); - -impl FdtIndexBuffer { - const fn zeroed() -> Self { - Self([0; INDEX_BUFFER_SIZE]) - } -} - -/// Device tree node -pub type TNode<'a> = DevTreeIndexNode<'a, 'a, 'a>; -/// Device tree property -pub type TProp<'a> = DevTreeIndexProp<'a, 'a, 'a>; - -/// Helper trait to provide extra functionality for [DevTreeIndexProp] -pub trait DevTreeIndexPropExt { - /// Reads a cell value from single-type cell array at given cell index - fn cell1_array_item(&self, index: usize, cells: usize) -> Option; - /// Reads a cell pair from cell pair array at given pair index - fn cell2_array_item(&self, index: usize, cells0: usize, cells1: usize) -> Option<(u64, u64)>; - - /// Reads a cell value from the property at given offset - fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option; -} - -/// Helper trait to provide extra functionality for [DevTreeIndexNode] -pub trait DevTreeIndexNodeExt { - /// Returns the root node's `#address-cells` property, or the default value defined by the - /// specification if it's absent - fn address_cells(&self) -> usize { - self.get_address_cells().unwrap_or(2) - } - /// Returns the root node's `#size-cells` property, or the default value defined by the - /// specification if it's absent - fn size_cells(&self) -> usize { - self.get_size_cells().unwrap_or(1) - } - - /// Returns the #address-cells property of the node, if there is one - fn get_address_cells(&self) -> Option; - /// Returns the #size-cells property of the node, if there is one - fn get_size_cells(&self) -> Option; - - /// Prints the node contents to debug output - fn dump(&self, level: LogLevel, depth: usize); -} - -/// Extension trait for [DevTreeIndexNode] to obtain typed property values -pub trait DevTreeIndexNodePropGet { - /// Returns a property value of given type, if it exists - fn prop(&self, name: &str) -> Option; -} - -impl<'a, 'i, 'dt> DevTreeIndexNodePropGet for DevTreeIndexNode<'a, 'i, 'dt> { - fn prop(&self, name: &str) -> Option { - self.props().find_map(|prop| { - if prop.name().ok()? == name { - prop.u32(0).ok() - } else { - None - } - }) - } -} - -impl<'a, 'i, 'dt> DevTreeIndexNodePropGet<&'a str> for DevTreeIndexNode<'a, 'i, 'dt> { - fn prop(&self, name: &str) -> Option<&'a str> { - self.props().find_map(|prop| { - if prop.name().ok()? == name { - prop.str().ok() - } else { - None - } - }) - } -} - -/// Iterator for physical memory regions present in the device tree -#[derive(Clone)] -pub struct FdtMemoryRegionIter<'a> { - inner: DevTreeIndexNodeSiblingIter<'a, 'a, 'a>, - address_cells: usize, - size_cells: usize, -} - -/// Device tree wrapper struct -pub struct DeviceTree<'a> { - tree: DevTree<'a>, - index: DevTreeIndex<'a, 'a>, -} - -impl<'a> DeviceTree<'a> { - /// Constructs a device tree wrapper from the DTB virtual address. - /// - /// # Safety - /// - /// The caller must ensure the validity of the address. - pub unsafe fn from_addr(virt: usize) -> Self { - let tree = DevTree::from_raw_pointer(virt as _).unwrap(); - let index = DevTreeIndex::new(tree, &mut FDT_INDEX_BUFFER.0).unwrap(); - Self { tree, index } - } - - /// Looks up a node for a given path - pub fn node_by_path(&self, path: &str) -> Option { - find_node(self.index.root(), path.trim_start_matches('/')) - } - - /// Prints the device tree to log output - pub fn dump(&self, level: LogLevel) { - self.index.root().dump(level, 0) - } - - /// Returns the total size of the device tree in memory - pub fn size(&self) -> usize { - self.tree.totalsize() - } - - /// Returns the root node's `#address-cells` property, or the default value defined by the - /// specification if it's absent - pub fn address_cells(&self) -> usize { - self.index.root().address_cells() - } - - /// Returns the root node's `#size-cells` property, or the default value defined by the - /// specification if it's absent - pub fn size_cells(&self) -> usize { - self.index.root().size_cells() - } - - /// Returns the root node of the device tree - pub fn root(&self) -> DevTreeIndexNode { - self.index.root() - } -} - -impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { - fn get_address_cells(&self) -> Option { - self.props() - .find(|p| p.name().unwrap_or("") == "#address-cells") - .map(|p| p.u32(0).unwrap() as usize) - } - - fn get_size_cells(&self) -> Option { - self.props() - .find(|p| p.name().unwrap_or("") == "#size-cells") - .map(|p| p.u32(0).unwrap() as usize) - } - - fn dump(&self, level: LogLevel, depth: usize) { - fn indent(level: LogLevel, depth: usize) { - for _ in 0..depth { - log_print_raw!(level, " "); - } - } - - let node_name = self.name().unwrap(); - - // Don't dump these - if node_name.starts_with("virtio_mmio@") { - return; - } - - indent(level, depth); - log_print_raw!(level, "{:?} {{\n", node_name); - for prop in self.props() { - indent(level, depth + 1); - let name = prop.name().unwrap_or(""); - log_print_raw!(level, "{name:?} = "); - - match name { - "compatible" | "stdout-path" => { - log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) - } - _ => log_print_raw!(level, "{:x?}", prop.raw()), - } - - log_print_raw!(level, "\n"); - } - - for child in self.children() { - child.dump(level, depth + 1); - } - - indent(level, depth); - log_print_raw!(level, "}}\n"); - } -} - -impl<'a, 'i, 'dt> DevTreeIndexPropExt for DevTreeIndexProp<'a, 'i, 'dt> { - fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option { - match cell_size { - 1 => self.u32(u32_offset).map(|x| x as u64).ok(), - 2 => { - let high = self.u32(u32_offset).ok()? as u64; - let low = self.u32(u32_offset + 1).ok()? as u64; - - Some((high << 32) | low) - } - _ => unimplemented!(), - } - } - - fn cell1_array_item(&self, index: usize, cells: usize) -> Option { - self.read_cell(index * cells, cells) - } - - fn cell2_array_item(&self, index: usize, cells0: usize, cells1: usize) -> Option<(u64, u64)> { - let u32_index = index * (cells0 + cells1); - let cell0 = self.read_cell(u32_index, cells0)?; - let cell1 = self.read_cell(u32_index + cells0, cells1)?; - Some((cell0, cell1)) - } -} - -impl<'a> FdtMemoryRegionIter<'a> { - /// Constructs a memory region iterator for given device tree - pub fn new(dt: &'a DeviceTree) -> Self { - let inner = dt.index.root().children(); - let address_cells = dt.address_cells(); - let size_cells = dt.size_cells(); - Self { - inner, - address_cells, - size_cells, - } - } -} - -impl Iterator for FdtMemoryRegionIter<'_> { - type Item = PhysicalMemoryRegion; - - fn next(&mut self) -> Option { - loop { - let Some(item) = self.inner.next() else { - break None; - }; - - let name = item.name().unwrap_or(""); - - if name.starts_with("memory@") || name == "memory" { - let reg = item - .props() - .find(|p| p.name().unwrap_or("") == "reg") - .unwrap(); - - let (base, size) = reg - .cell2_array_item(0, self.address_cells, self.size_cells) - .unwrap(); - - break Some(PhysicalMemoryRegion { - base: base as usize, - size: size as usize, - }); - } - } - } -} - -/// Looks up a property with given name in the node -pub fn find_prop<'a>(node: &TNode<'a>, name: &str) -> Option> { - node.props().find(|p| p.name().unwrap_or("") == name) -} - -fn path_component_left(path: &str) -> (&str, &str) { - if let Some((left, right)) = path.split_once('/') { - (left, right.trim_start_matches('/')) - } else { - (path, "") - } -} - -fn find_node<'a>(at: TNode<'a>, path: &str) -> Option> { - let (item, path) = path_component_left(path); - if item.is_empty() { - assert_eq!(path, ""); - Some(at) - } else { - let child = at.children().find(|c| c.name().unwrap() == item)?; - if path.is_empty() { - Some(child) - } else { - find_node(child, path) - } - } -} diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 19bf8837..0bb60616 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -8,20 +8,18 @@ use alloc::boxed::Box; use device_api::{ interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, - IpiDeliveryTarget, IrqNumber, IrqOptions, LocalInterruptController, + IpiDeliveryTarget, IrqOptions, LocalInterruptController, }, Device, }; +use kernel_util::util::OneTimeInit; use crate::{ - arch::{ - aarch64::devtree::{self, DevTreeIndexPropExt}, - Architecture, CpuMessage, - }, + arch::{aarch64::IrqNumber, Architecture, CpuMessage}, + device::devtree::{self, DevTreeIndexPropExt}, device_tree_driver, mem::device::{DeviceMemory, DeviceMemoryIo}, sync::IrqSafeSpinlock, - util::OneTimeInit, }; use self::{gicc::Gicc, gicd::Gicd}; @@ -71,6 +69,8 @@ impl Device for Gic { } impl ExternalInterruptController for Gic { + type IrqNumber = IrqNumber; + fn register_irq( &self, irq: IrqNumber, diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index b2834c36..407924fe 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -11,30 +11,34 @@ use aarch64_cpu::{ }; use abi::error::Error; use device_api::{ - interrupt::{ - ExternalInterruptController, IpiDeliveryTarget, IrqNumber, LocalInterruptController, - }, + interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; use fdt_rs::prelude::PropReader; use git_version::git_version; +use kernel_util::util::OneTimeInit; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::{aarch64::devtree::FdtMemoryRegionIter, Architecture}, - debug::{self}, - device::{self, power::arm_psci::Psci, DevTreeNodeInfo}, + arch::Architecture, + debug, + device::{ + self, + devtree::{ + self, DevTreeIndexNodePropGet, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, + FdtMemoryRegionIter, + }, + power::arm_psci::Psci, + }, fs::{Initrd, INITRD_DATA}, mem::{ phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, ConvertAddress, }, - util::OneTimeInit, }; use self::{ - devtree::{DevTreeIndexPropExt, DeviceTree}, smp::CPU_COUNT, table::{init_fixed_tables, KERNEL_TABLES}, }; @@ -44,7 +48,6 @@ use super::CpuMessage; pub mod boot; pub mod context; pub mod cpu; -pub mod devtree; pub mod exception; pub mod gic; pub mod smp; @@ -59,10 +62,16 @@ struct KernelStack { data: [u8; BOOT_STACK_SIZE], } +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum IrqNumber { + Private(u32), + Shared(u32), +} + /// AArch64 platform interface pub struct AArch64 { dt: OneTimeInit>, - ext_intc: OneTimeInit<&'static dyn ExternalInterruptController>, + ext_intc: OneTimeInit<&'static dyn ExternalInterruptController>, local_intc: OneTimeInit<&'static dyn LocalInterruptController>, mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, reset: OneTimeInit<&'static dyn ResetDevice>, @@ -85,6 +94,8 @@ pub static ARCHITECTURE: AArch64 = AArch64 { }; impl Architecture for AArch64 { + type IrqNumber = IrqNumber; + const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; unsafe fn init_mmu(&self, bsp: bool) { @@ -165,7 +176,7 @@ impl Architecture for AArch64 { fn register_external_interrupt_controller( &self, - intc: &'static dyn ExternalInterruptController, + intc: &'static dyn ExternalInterruptController, ) -> Result<(), Error> { self.ext_intc.init(intc); Ok(()) @@ -192,7 +203,9 @@ impl Architecture for AArch64 { Ok(()) } - fn external_interrupt_controller(&self) -> &'static dyn ExternalInterruptController { + fn external_interrupt_controller( + &self, + ) -> &'static dyn ExternalInterruptController { *self.ext_intc.get() } @@ -272,8 +285,7 @@ impl AArch64 { fn chosen_stdout_path<'a>(dt: &'a DeviceTree) -> Option<&'a str> { let chosen = dt.node_by_path("/chosen")?; - let prop = devtree::find_prop(&chosen, "stdout-path")?; - prop.str().ok() + chosen.prop("stdout-path") } fn init_platform(&self, bsp: bool) { @@ -294,7 +306,7 @@ impl AArch64 { node, }; - if let Some((device, _)) = device::probe_dt_node(&probe) { + if let Some((device, _)) = devtree::probe_dt_node(&probe) { unsafe { device.init().unwrap(); } @@ -313,7 +325,7 @@ impl AArch64 { // Probe and initialize the rest of devices let nodes = dt.root().children(); - if let Err(error) = device::enumerate_dt( + if let Err(error) = devtree::enumerate_dt( address_cells, size_cells, nodes, @@ -323,7 +335,7 @@ impl AArch64 { return Ok(()); } - if let Some((device, _)) = device::probe_dt_node(&probe) { + if let Some((device, _)) = devtree::probe_dt_node(&probe) { unsafe { device.init()?; } diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index 36366aed..424a8cf7 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -14,7 +14,7 @@ use crate::{ }, }; -use super::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; +use crate::device::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; #[derive(Debug)] enum CpuEnableMethod { diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index fc00a0c7..e4cc7c8f 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -5,15 +5,11 @@ use core::time::Duration; use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use alloc::boxed::Box; -use device_api::{ - interrupt::{InterruptHandler, IrqNumber}, - timer::MonotonicTimestampProviderDevice, - Device, -}; +use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::{Architecture, ARCHITECTURE}, + arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, device_tree_driver, proc::wait, task::tasklet, diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 34d4b2bc..ae4f3b11 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -19,31 +19,27 @@ macro_rules! absolute_address { }}; } -pub mod aarch64; - -pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; +use cfg_if::cfg_if; +// pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; use device_api::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; -// cfg_if! { -// if #[cfg(target_arch = "aarch64")] { -// -// pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE, PlatformImpl, PLATFORM}; -// } else if #[cfg(target_arch = "x86_64")] { -// pub mod x86_64; -// -// pub use x86_64::{ -// X86_64 as ArchitectureImpl, -// X86_64 as PlatformImpl, -// ARCHITECTURE, -// ARCHITECTURE as PLATFORM -// }; -// } else { -// compile_error!("Architecture is not supported"); -// } -// } + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub mod aarch64; + + pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; + } else if #[cfg(target_arch = "x86_64")] { + pub mod x86_64; + + pub use x86_64::{X86_64 as ArchitectureImpl, ARCHITECTURE}; + } else { + compile_error!("Architecture is not supported"); + } +} /// Describes messages sent from some CPU to others #[derive(Clone, Copy, PartialEq, Debug)] @@ -58,6 +54,9 @@ pub trait Architecture { /// Address, to which "zero" address is mapped in the virtual address space const KERNEL_VIRT_OFFSET: usize; + /// IRQ number type associated with the architecture + type IrqNumber; + /// Initializes the memory management unit and sets up virtual memory management. /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP /// system. @@ -99,7 +98,7 @@ pub trait Architecture { /// Adds an external interrupt controller to the system fn register_external_interrupt_controller( &self, - intc: &'static dyn ExternalInterruptController, + intc: &'static dyn ExternalInterruptController, ) -> Result<(), Error>; /// Adds a local interrupt controller to the system @@ -119,15 +118,17 @@ pub trait Architecture { // TODO only supports 1 extintc per system /// Returns the primary external interrupt controller - fn external_interrupt_controller(&self) -> &'static dyn ExternalInterruptController; + fn external_interrupt_controller( + &'static self, + ) -> &'static dyn ExternalInterruptController; /// Returns the local interrupt controller fn local_interrupt_controller( - &self, + &'static self, ) -> &'static dyn LocalInterruptController; /// Returns the monotonic timer - fn monotonic_timer(&self) -> &'static dyn MonotonicTimestampProviderDevice; + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice; /// Sends a message to the requested set of CPUs through an interprocessor interrupt. /// diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 4cc5c33a..ff5b50eb 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -1,19 +1,21 @@ //! x86-64 I/O APIC driver implementation use abi::error::Error; use acpi::platform::interrupt::{Apic as AcpiApic, Polarity, TriggerMode}; - -use crate::{ - arch::x86_64::apic::local::BSP_APIC_ID, - device::{ - interrupt::{ExternalInterruptController, InterruptSource}, - Device, InitializableDevice, +use device_api::{ + interrupt::{ + ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, + IrqLevel, IrqOptions, IrqTrigger, }, - mem::ConvertAddress, - sync::IrqSafeSpinlock, - util::OneTimeInit, + Device, }; -use super::{IrqNumber, Table, APIC_EXTERNAL_OFFSET}; +use crate::{ + arch::x86_64::{apic::local::BSP_APIC_ID, IrqNumber}, + mem::ConvertAddress, + sync::IrqSafeSpinlock, +}; + +use super::{APIC_EXTERNAL_OFFSET, MAX_EXTERNAL_VECTORS}; // IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET; @@ -54,10 +56,10 @@ struct Inner { /// I/O APIC interface. Provides a way to route and control how interrupts from external devices /// are handled. pub struct IoApic { - inner: OneTimeInit>, - isa_redirections: OneTimeInit<[Option; 16]>, + inner: IrqSafeSpinlock, + isa_redirections: [Option; 16], - table: IrqSafeSpinlock
, + table: IrqSafeSpinlock>, } impl Regs { @@ -108,21 +110,29 @@ impl Inner { Ok(()) } - fn configure_gsi(&mut self, gsi: u32, active_low: bool, level_triggered: bool) { + fn configure_gsi(&mut self, gsi: u32, options: IrqOptions) { assert!(gsi < self.max_gsi); let mut low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2); - if active_low { - low |= ENTRY_LOW_POLARITY_LOW; - } else { - low &= !ENTRY_LOW_POLARITY_LOW; + match options.level { + IrqLevel::Default => (), + IrqLevel::ActiveLow => { + low |= ENTRY_LOW_POLARITY_LOW; + } + IrqLevel::ActiveHigh => { + low &= !ENTRY_LOW_POLARITY_LOW; + } } - if level_triggered { - low |= ENTRY_LOW_TRIGGER_LEVEL; - } else { - low &= !ENTRY_LOW_TRIGGER_LEVEL; + match options.trigger { + IrqTrigger::Default => (), + IrqTrigger::Level => { + low |= ENTRY_LOW_TRIGGER_LEVEL; + } + IrqTrigger::Edge => { + low &= !ENTRY_LOW_TRIGGER_LEVEL; + } } self.regs.write(REG_REDIRECTION_BASE + gsi * 2, low); @@ -144,71 +154,69 @@ impl Inner { } impl Device for IoApic { - fn name(&self) -> &'static str { + fn display_name(&self) -> &'static str { "I/O APIC" } + + unsafe fn init(&'static self) -> Result<(), Error> { + todo!() + } } impl ExternalInterruptController for IoApic { type IrqNumber = IrqNumber; - fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { - let mut inner = self.inner.get().lock(); - let gsi = self.translate_irq(irq); - inner.set_gsi_enabled(gsi, true); - Ok(()) - } - - fn register_handler( + fn register_irq( &self, irq: Self::IrqNumber, - handler: &'static (dyn InterruptSource + Sync), + options: IrqOptions, + handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { - let mut inner = self.inner.get().lock(); - let mut table = self.table.lock(); + let mut inner = self.inner.lock(); + let table_vector = self.table.lock().insert_least_loaded(handler)?; - // Allocate a vector - let table_vector = table.least_loaded(); let gsi_target_vector = (table_vector as u32) + IO_APIC_VECTOR_OFFSET; let bsp_apic = *BSP_APIC_ID.get(); infoln!( "Binding {:?} ({}) to {}:{}", irq, - handler.name(), + handler.display_name(), bsp_apic, table_vector ); - table.add_handler(table_vector, handler)?; let gsi = self.translate_irq(irq); - inner.map_gsi(gsi, gsi_target_vector, *BSP_APIC_ID.get())?; + inner.configure_gsi(gsi, options); + inner.map_gsi(gsi, gsi_target_vector, bsp_apic)?; Ok(()) } - fn configure_irq( - &self, - irq: Self::IrqNumber, - active_low: bool, - level_triggered: bool, - ) -> Result<(), Error> { - let mut inner = self.inner.get().lock(); + fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { + let mut inner = self.inner.lock(); let gsi = self.translate_irq(irq); - - inner.configure_gsi(gsi, active_low, level_triggered); - + inner.set_gsi_enabled(gsi, true); Ok(()) } + + fn handle_specific_irq(&self, gsi: usize) { + let table = self.table.lock(); + + if let Some(handler) = table.handler(gsi) { + handler.handle_irq(); + } else { + warnln!("No handler set for GSI #{}", gsi); + } + } } -impl InitializableDevice for IoApic { - type Options = AcpiApic; +impl IoApic { + /// Creates an I/O APIC instance from its ACPI definition + pub fn from_acpi(info: &AcpiApic) -> Result { + let ioapic = info.io_apics.first().unwrap(); - unsafe fn init(&self, info: Self::Options) -> Result<(), Error> { - let io_apic = info.io_apics.first().unwrap(); - - infoln!("I/O APIC at {:#x}", io_apic.address); + infoln!("I/O APIC at {:#x}", ioapic.address); let mut isa_redirections = [None; 16]; @@ -236,8 +244,9 @@ impl InitializableDevice for IoApic { }); } + // TODO properly map this using DeviceMemory let regs = Regs { - base: unsafe { (io_apic.address as usize).virtualize() }, + base: unsafe { (ioapic.address as usize).virtualize() }, }; let max_gsi = (regs.read(REG_IOAPIC_VERSION) >> 16) & 0xFF; @@ -251,36 +260,15 @@ impl InitializableDevice for IoApic { inner.set_gsi_enabled(gsi, false); } - self.inner.init(IrqSafeSpinlock::new(inner)); - self.isa_redirections.init(isa_redirections); - - Ok(()) - } -} - -impl IoApic { - /// Constructs an uninitialized I/O APIC interface - pub const fn new() -> Self { - Self { - inner: OneTimeInit::new(), - isa_redirections: OneTimeInit::new(), - table: IrqSafeSpinlock::new(Table::new()), - } - } - - /// Handles an interrupt with given GSI number - pub fn handle_irq(&self, gsi: u32) { - let table = self.table.lock(); - - if let Some(handler) = table.vectors[gsi as usize] { - handler.handle_irq().expect("IRQ handler failed"); - } else { - warnln!("No handler set for GSI #{}", gsi); - } + Ok(Self { + isa_redirections, + inner: IrqSafeSpinlock::new(inner), + table: IrqSafeSpinlock::new(FixedInterruptTable::new()), + }) } fn translate_irq(&self, irq: IrqNumber) -> u32 { - let redir = self.isa_redirections.get(); + let redir = &self.isa_redirections; match irq { IrqNumber::Isa(isa) => redir[isa as usize] .map(|t| t.gsi_index) diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index df5d9b6f..20f0bebb 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -1,4 +1,6 @@ //! x86-64 Local APIC driver implementation +use device_api::{interrupt::LocalInterruptController, Device}; +use kernel_util::util::OneTimeInit; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -6,13 +8,11 @@ use tock_registers::{ }; use crate::{ - arch::x86_64::registers::MSR_IA32_APIC_BASE, device::interrupt::IpiDeliveryTarget, - mem::ConvertAddress, util::OneTimeInit, + arch::{x86_64::registers::MSR_IA32_APIC_BASE, CpuMessage}, + mem::ConvertAddress, }; -use super::{ - APIC_IPI_VECTOR, APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR, -}; +use super::{APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR}; const TIMER_INTERVAL: u32 = 150000; @@ -122,6 +122,31 @@ pub struct LocalApic { regs: &'static Regs, } +unsafe impl Send for LocalApic {} +unsafe impl Sync for LocalApic {} + +impl Device for LocalApic { + fn display_name(&self) -> &'static str { + "Local APIC" + } +} + +impl LocalInterruptController for LocalApic { + type IpiMessage = CpuMessage; + + fn send_ipi( + &self, + _target: device_api::interrupt::IpiDeliveryTarget, + _msg: Self::IpiMessage, + ) -> Result<(), abi::error::Error> { + todo!() + } + + unsafe fn init_ap(&self) -> Result<(), abi::error::Error> { + todo!() + } +} + impl LocalApic { /// Constructs a new instance of Local APIC. /// @@ -232,28 +257,28 @@ impl LocalApic { } } - /// Issues an interprocessor interrupt for the target. - /// - /// # Safety - /// - /// Unsafe: this function may break the control flow on the target processors. - pub unsafe fn send_ipi(&self, target: IpiDeliveryTarget) { - while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { - core::hint::spin_loop(); - } + // /// Issues an interprocessor interrupt for the target. + // /// + // /// # Safety + // /// + // /// Unsafe: this function may break the control flow on the target processors. + // pub unsafe fn send_ipi(&self, target: IpiDeliveryTarget) { + // while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + // core::hint::spin_loop(); + // } - match target { - IpiDeliveryTarget::AllExceptLocal => { - self.regs.ICR1.write(ICR1::PhysicalDestination.val(0)); - self.regs.ICR0.write( - ICR0::Vector.val(APIC_IPI_VECTOR + 32) - + ICR0::Destination::NMI - + ICR0::DestinationType::AllExceptThis, - ); - } - IpiDeliveryTarget::Specified(_) => todo!(), - } - } + // match target { + // IpiDeliveryTarget::AllExceptLocal => { + // self.regs.ICR1.write(ICR1::PhysicalDestination.val(0)); + // self.regs.ICR0.write( + // ICR0::Vector.val(APIC_IPI_VECTOR + 32) + // + ICR0::Destination::NMI + // + ICR0::DestinationType::AllExceptThis, + // ); + // } + // IpiDeliveryTarget::Specified(_) => todo!(), + // } + // } #[inline] fn base() -> usize { diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 5f5fdea5..a363b572 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -2,15 +2,15 @@ use core::arch::global_asm; -use abi::error::Error; - use crate::{ - arch::{x86_64::cpu::Cpu, PLATFORM}, - device::interrupt::IrqHandler, + arch::{x86_64::cpu::Cpu, Architecture}, task::process::Process, }; -use super::exception::{self, IrqFrame}; +use super::{ + exception::{self, IrqFrame}, + ARCHITECTURE, +}; pub mod ioapic; pub mod local; @@ -32,37 +32,6 @@ pub const APIC_EXTERNAL_OFFSET: u32 = 4; /// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC pub const MAX_EXTERNAL_VECTORS: u32 = 16; -/// x86-64 interrupt number wrapper -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum IrqNumber { - /// Legacy (ISA) interrupt. Can have value in range 0..16. - Isa(u8), - /// Global system interrupt. Means an external interrupt for I/O APIC. - Gsi(u8), -} - -struct Table { - vectors: [Option; MAX_EXTERNAL_VECTORS as usize], -} - -impl Table { - pub const fn new() -> Self { - Self { - vectors: [None; MAX_EXTERNAL_VECTORS as usize], - } - } - - pub fn add_handler(&mut self, vector: usize, handler: IrqHandler) -> Result<(), Error> { - let old = self.vectors[vector].replace(handler); - assert!(old.is_none()); - Ok(()) - } - - pub fn least_loaded(&self) -> usize { - self.vectors.iter().position(|p| p.is_none()).unwrap() - } -} - /// Fills the IDT with interrupt vectors for this APIC pub fn setup_vectors(idt: &mut [exception::Entry]) { extern "C" { @@ -83,7 +52,9 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { let cpu = Cpu::local(); let frame = &mut *frame; - PLATFORM.ioapic.handle_irq(vector as u32); + ARCHITECTURE + .external_interrupt_controller() + .handle_specific_irq(vector); cpu.local_apic().clear_interrupt(); if let Some(process) = Process::get_current() { diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index f28ce514..5dd6e45c 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -9,18 +9,21 @@ use yboot_proto::{ use crate::{ arch::{ - x86_64::{ - apic::local::LocalApic, cpu::Cpu, cpuid, exception, kernel_main, - registers::MSR_IA32_KERNEL_GS_BASE, syscall, CPU_INIT_FENCE, - }, - Architecture, ArchitectureImpl, + x86_64::{cpuid, exception, registers::MSR_IA32_KERNEL_GS_BASE, BootData}, + Architecture, ArchitectureImpl, ARCHITECTURE, + }, + fs::devfs, + kernel_main, kernel_secondary_main, + mem::{ + heap, + phys::{self, PageUsage}, + ConvertAddress, KERNEL_VIRT_OFFSET, }, - device::platform::Platform, - mem::KERNEL_VIRT_OFFSET, - task, }; -use super::ARCHITECTURE; +use super::smp::CPU_COUNT; + +// use super::ARCHITECTURE; const BOOT_STACK_SIZE: usize = 65536; @@ -76,45 +79,58 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { ARCHITECTURE.init_mmu(true); core::arch::asm!("wbinvd"); - kernel_main(&YBOOT_DATA) + ARCHITECTURE.set_boot_data(BootData::YBoot(&YBOOT_DATA)); + ARCHITECTURE + .init_physical_memory() + .expect("Failed to initialize the physical memory manager"); + + let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) + .expect("Couldn't allocate memory for heap"); + heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + + exception::init_exceptions(0); + + devfs::init(); + + // Initializes: local CPU, platform devices (timers/serials/etc), debug output + ARCHITECTURE.init_platform(0); + + cpuid::feature_gate(); + + kernel_main() } /// Application processor entry point pub extern "C" fn __x86_64_ap_entry() -> ! { - loop {} - // let cpu_id = CPU_COUNT.load(Ordering::Acquire); + let cpu_id = CPU_COUNT.load(Ordering::Acquire); - // MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); - // unsafe { - // core::arch::asm!("swapgs"); - // } - // MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); - // unsafe { - // core::arch::asm!("swapgs"); - // } + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + unsafe { + core::arch::asm!("swapgs"); + } + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + unsafe { + core::arch::asm!("swapgs"); + } - // // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base - // cpuid::feature_gate(); + // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base + cpuid::feature_gate(); - // infoln!("cpu{} initializing", cpu_id); - // unsafe { - // ARCHITECTURE.init_mmu(false); - // core::arch::asm!("wbinvd"); + infoln!("cpu{} initializing", cpu_id); + unsafe { + ARCHITECTURE.init_mmu(false); + core::arch::asm!("wbinvd"); - // Cpu::init_local(LocalApic::new(), cpu_id as u32); - // syscall::init_syscall(); - // exception::init_exceptions(cpu_id); + // Cpu::init_local(LocalApic::new(), cpu_id as u32); + // syscall::init_syscall(); + exception::init_exceptions(cpu_id); - // ARCHITECTURE.init(false).unwrap(); - // } + ARCHITECTURE.init_platform(cpu_id); + } - // CPU_COUNT.fetch_add(1, Ordering::Release); + CPU_COUNT.fetch_add(1, Ordering::Release); - // CPU_INIT_FENCE.wait_one(); - - // infoln!("cpu{} initialized", cpu_id); - - // unsafe { task::enter() } + kernel_secondary_main() } global_asm!( diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 9737049a..122fa42e 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -2,12 +2,12 @@ use core::ptr::null_mut; use alloc::boxed::Box; +use kernel_util::util::OneTimeInit; use tock_registers::interfaces::Writeable; use crate::{ - arch::x86_64::{gdt, registers::MSR_IA32_KERNEL_GS_BASE}, + arch::x86_64::{gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, task::sched::CpuQueue, - util::OneTimeInit, }; use super::apic::local::LocalApic; @@ -53,6 +53,8 @@ impl Cpu { MSR_IA32_KERNEL_GS_BASE.set(this as u64); core::arch::asm!("wbinvd; swapgs"); + + syscall::init_syscall(); } /// Returns this CPU's local data structure diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index ac4136f3..a867a5ae 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -2,50 +2,50 @@ use core::{ptr::NonNull, sync::atomic::Ordering}; use abi::error::Error; -use acpi::{ - platform::ProcessorInfo, AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping, -}; +use acpi::{AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; +use alloc::boxed::Box; use cpu::Cpu; +use device_api::{ + input::KeyboardProducer, + interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, + timer::MonotonicTimestampProviderDevice, + Device, +}; use git_version::git_version; -use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap, LoadProtocolV1}; +use kernel_util::util::OneTimeInit; +use yboot_proto::{AvailableRegion, IterableMemoryMap}; use crate::{ arch::x86_64::{ apic::local::LocalApic, + peripherals::serial::ComPort, table::{init_fixed_tables, KERNEL_TABLES}, }, debug::{self, LogLevel}, device::{ + self, display::{ - console::{add_console_autoflush, ConsoleBuffer, ConsoleRow, DisplayConsole}, + console::{self, ConsoleBuffer}, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer, }, - input::KeyboardDevice, - interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, - platform::Platform, - timer::TimestampSource, - tty::CombinedTerminal, - InitializableDevice, + tty::combined::CombinedTerminal, }, fs::{ devfs::{self, CharDeviceType}, Initrd, INITRD_DATA, }, mem::{ - heap, - phys::{self, reserved::reserve_region, PageUsage, PhysicalMemoryRegion}, + phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, ConvertAddress, }, - sync::SpinFence, - task, - util::OneTimeInit, + CPU_INIT_FENCE, }; use self::{ - apic::{ioapic::IoApic, IrqNumber}, + apic::ioapic::IoApic, intrinsics::IoPort, - peripherals::{hpet::Hpet, ps2::PS2Controller, serial::ComPort}, + peripherals::{hpet::Hpet, ps2::PS2Controller}, smp::CPU_COUNT, }; @@ -67,7 +67,14 @@ pub mod smp; pub mod syscall; pub mod table; -static CPU_INIT_FENCE: SpinFence = SpinFence::new(); +/// x86-64 interrupt number wrapper +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum IrqNumber { + /// Legacy (ISA) interrupt. Can have value in range 0..16. + Isa(u8), + /// Global system interrupt. Means an external interrupt for I/O APIC. + Gsi(u8), +} /// Helper trait to provide abstract access to available memory regions pub trait AbstractAvailableRegion { @@ -119,6 +126,27 @@ impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T { #[derive(Clone, Copy)] struct AcpiHandlerImpl; +struct SimpleLogger; + +impl log::Log for SimpleLogger { + fn enabled(&self, _metadata: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + match record.level() { + log::Level::Warn => warnln!("{}", record.args()), + log::Level::Info => infoln!("{}", record.args()), + log::Level::Trace | log::Level::Debug => infoln!("{}", record.args()), + log::Level::Error => errorln!("{}", record.args()), + } + } + + fn flush(&self) {} +} + +static LOGGER: SimpleLogger = SimpleLogger; + impl AcpiHandler for AcpiHandlerImpl { // No actual address space modification is performed unsafe fn map_physical_region( @@ -143,28 +171,44 @@ impl AcpiHandler for AcpiHandlerImpl { fn unmap_physical_region(_region: &PhysicalMapping) {} } +/// Describes which kind of bootloader data was provided to the kernel +pub enum BootData { + /// [yboot_proto::LoadProtocolV1] + YBoot(&'static yboot_proto::LoadProtocolV1), +} + /// x86-64 architecture + platform implementation pub struct X86_64 { - yboot_framebuffer: OneTimeInit, - rsdp_phys_base: OneTimeInit, + boot_data: OneTimeInit, + acpi: OneTimeInit>, - acpi_processor_info: OneTimeInit, - - // TODO make this fully dynamic? + framebuffer: OneTimeInit, + fb_console: OneTimeInit, combined_terminal: OneTimeInit, - // Static devices with dynamic addresses - ioapic: IoApic, - timer: Hpet, - - // Static devices - ps2: PS2Controller, - com1_3: ComPort, + ioapic: OneTimeInit, + timer: OneTimeInit, } +/// x86-64 architecture implementation +pub static ARCHITECTURE: X86_64 = X86_64 { + boot_data: OneTimeInit::new(), + acpi: OneTimeInit::new(), + + framebuffer: OneTimeInit::new(), + fb_console: OneTimeInit::new(), + combined_terminal: OneTimeInit::new(), + + // Devices + ioapic: OneTimeInit::new(), + timer: OneTimeInit::new(), +}; + impl Architecture for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; + type IrqNumber = IrqNumber; + unsafe fn init_mmu(&self, bsp: bool) { if bsp { init_fixed_tables(); @@ -174,16 +218,6 @@ impl Architecture for X86_64 { core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); } - fn map_device_pages(&self, phys: usize, count: usize) -> Result { - unsafe { KERNEL_TABLES.map_device_pages(phys, count) } - } - - fn wait_for_interrupt() { - unsafe { - core::arch::asm!("hlt"); - } - } - unsafe fn set_interrupt_mask(mask: bool) { if mask { core::arch::asm!("cli"); @@ -201,98 +235,18 @@ impl Architecture for X86_64 { flags & (1 << 9) == 0 } - fn cpu_count() -> usize { - CPU_COUNT.load(Ordering::Acquire) - } -} - -impl Platform for X86_64 { - type IrqNumber = apic::IrqNumber; - - const KERNEL_PHYS_BASE: usize = 0x400000; - - unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { - if is_bsp { - Self::disable_8259(); - - // Initialize I/O APIC from ACPI - let rsdp = *self.rsdp_phys_base.get(); - let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap(); - - let hpet = HpetInfo::new(&acpi_tables).unwrap(); - let platform_info = acpi_tables.platform_info().unwrap(); - - if let Some(processor_info) = platform_info.processor_info { - self.acpi_processor_info.init(processor_info); - } - - let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { - panic!("Processor does not have APIC"); - }; - - self.ioapic.init(apic_info)?; - self.timer.init(hpet).unwrap(); - - // Enable IRQs for the devices - self.ps2.init_irq()?; - self.timer.init_irq()?; - - // Add a terminal to the devfs - // TODO this is ugly - let combined_terminal = CombinedTerminal::new(&self.ps2, FB_CONSOLE.get()); - self.combined_terminal.init(combined_terminal); - self.ps2.attach(self.combined_terminal.get()); - - devfs::add_char_device(self.combined_terminal.get(), CharDeviceType::TtyRegular)?; + fn wait_for_interrupt() { + unsafe { + core::arch::asm!("hlt"); } - - Ok(()) } - unsafe fn init_debug(&'static self) { - // Serial output - self.com1_3.init(()).ok(); - debug::add_sink(self.com1_3.port_a(), LogLevel::Debug); - - // Graphical output - if let Some(fb) = self.yboot_framebuffer.try_get() { - LINEAR_FB.init( - LinearFramebuffer::from_physical_bits( - fb.res_address as _, - fb.res_size as _, - fb.res_stride as _, - fb.res_width, - fb.res_height, - ) - .unwrap(), - ); - FB_CONSOLE.init(FramebufferConsole::from_framebuffer( - LINEAR_FB.get(), - &bitmap_font::tamzen::FONT_6x12, - ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 32), - )); - - debug::add_sink(FB_CONSOLE.get(), LogLevel::Info); - } - - debug::reset(); + // CPU management + unsafe fn reset(&self) -> ! { + todo!() } - fn name(&self) -> &'static str { - "x86-64" - } - - fn interrupt_controller( - &self, - ) -> &dyn ExternalInterruptController { - &self.ioapic - } - - fn timestamp_source(&self) -> &dyn TimestampSource { - &self.timer - } - - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { if !CPU_INIT_FENCE.try_wait_all(1) { // Don't send an IPI: SMP not initialized yet return Ok(()); @@ -302,14 +256,132 @@ impl Platform for X86_64 { panic!("Local APIC has not been initialized yet"); }; - local_apic.send_ipi(target); + local_apic.send_ipi(target, msg) + } - Ok(()) + unsafe fn start_application_processors(&self) { + if let Some(acpi) = self.acpi.try_get() { + let Some(pinfo) = acpi.platform_info().ok().and_then(|p| p.processor_info) else { + return; + }; + + smp::start_ap_cores(&pinfo); + } + } + + fn cpu_count() -> usize { + CPU_COUNT.load(Ordering::Acquire) + } + + // Memory + fn map_device_pages(&self, phys: usize, count: usize) -> Result { + unsafe { KERNEL_TABLES.map_device_pages(phys, count) } + } + + // Devices + fn register_monotonic_timer( + &self, + _timer: &'static dyn device_api::timer::MonotonicTimestampProviderDevice, + ) -> Result<(), Error> { + todo!() + } + + fn register_local_interrupt_controller( + &self, + _intc: &'static dyn device_api::interrupt::LocalInterruptController< + IpiMessage = CpuMessage, + >, + ) -> Result<(), Error> { + todo!() + } + + fn register_external_interrupt_controller( + &self, + _intc: &'static dyn device_api::interrupt::ExternalInterruptController< + IrqNumber = Self::IrqNumber, + >, + ) -> Result<(), Error> { + todo!() + } + + fn register_reset_device( + &self, + _reset: &'static dyn device_api::ResetDevice, + ) -> Result<(), Error> { + todo!() + } + + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { + self.timer.get() + } + + fn local_interrupt_controller( + &'static self, + ) -> &'static dyn LocalInterruptController { + todo!() + } + + fn external_interrupt_controller( + &'static self, + ) -> &'static dyn ExternalInterruptController { + self.ioapic.get() } } impl X86_64 { - unsafe fn init_physical_memory<'a, M: AbstractMemoryMap<'a>>(memory_map: &M) { + fn set_boot_data(&self, boot_data: BootData) { + match &boot_data { + BootData::YBoot(data) => { + // Setup initrd + Self::init_initrd( + data.initrd_address as usize, + (data.initrd_address + data.initrd_size) as usize, + ); + } + } + + self.boot_data.init(boot_data); + } + + fn init_initrd(initrd_start: usize, initrd_end: usize) { + if initrd_start == 0 || initrd_end <= initrd_start { + infoln!("No initrd loaded"); + return; + } + + let start_aligned = initrd_start & !0xFFF; + let end_aligned = initrd_end & !0xFFF; + + let data = unsafe { + core::slice::from_raw_parts( + initrd_start.virtualize() as *const _, + initrd_end - initrd_start, + ) + }; + + let initrd = Initrd { + phys_page_start: start_aligned, + phys_page_len: end_aligned - start_aligned, + data, + }; + + INITRD_DATA.init(initrd); + } + + fn init_acpi_from_boot_data(&self) { + match *self.boot_data.get() { + BootData::YBoot(data) => { + self.init_acpi_from_rsdp(data.rsdp_address as usize); + } + } + } + + fn init_acpi_from_rsdp(&self, address: usize) { + let acpi_tables = unsafe { AcpiTables::from_rsdp(AcpiHandlerImpl, address).unwrap() }; + self.acpi.init(acpi_tables); + } + + unsafe fn init_physical_memory(&self) -> Result<(), Error> { // Reserve the lower 8MiB of memory reserve_region( "lower-memory", @@ -318,9 +390,8 @@ impl X86_64 { size: 8 << 21, }, ); - // Reserve memory map - reserve_region("memory-map", memory_map.reserved_range()); + // Reserve initrd if let Some(initrd) = INITRD_DATA.try_get() { reserve_region( "initrd", @@ -331,21 +402,125 @@ impl X86_64 { ); } - phys::init_from_iter(memory_map.iter().map(|r| PhysicalMemoryRegion { - base: r.start_address(), - size: r.page_count() * 0x1000, - })) - .expect("Failed to initialize the physical memory manager"); + match *self.boot_data.get() { + BootData::YBoot(data) => { + let memory_map = &data.memory_map; - // Reallocate the console buffer - FB_CONSOLE - .get() - .reallocate_buffer() - .expect("Failed to reallocate console buffer"); + reserve_region("memory-map", memory_map.reserved_range()); + + phys::init_from_iter(IterableMemoryMap::iter(memory_map).map(|r| { + PhysicalMemoryRegion { + base: AbstractAvailableRegion::start_address(r), + size: AbstractAvailableRegion::page_count(r) * 0x1000, + } + })) + .expect("Failed to initialize the physical memory manager"); + } + } + + Ok(()) } - fn init_rsdp(&self, phys: usize) { - self.rsdp_phys_base.init(phys); + unsafe fn init_platform_from_acpi(&self, acpi: &AcpiTables) { + let platform_info = acpi.platform_info().unwrap(); + + let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { + panic!("Processor does not have an APIC"); + }; + + self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap()); + + let hpet = HpetInfo::new(acpi).unwrap(); + self.timer.init(Hpet::from_acpi(&hpet).unwrap()); + } + + unsafe fn init_framebuffer(&'static self) { + match *self.boot_data.get() { + BootData::YBoot(data) => { + let fb = &data.opt_framebuffer; + self.framebuffer.init( + LinearFramebuffer::from_physical_bits( + fb.res_address as _, + fb.res_size as _, + fb.res_stride as _, + fb.res_width as _, + fb.res_height as _, + ) + .unwrap(), + ); + } + } + + self.fb_console.init(FramebufferConsole::from_framebuffer( + self.framebuffer.get(), + None, + ConsoleBuffer::new(32).unwrap(), + )); + + debug::add_sink(self.fb_console.get(), LogLevel::Info); + + // Add a terminal to the devfs + // TODO this is ugly + let combined_terminal = CombinedTerminal::new(self.fb_console.get()); + self.combined_terminal.init(combined_terminal); + + devfs::add_char_device(self.combined_terminal.get(), CharDeviceType::TtyRegular).unwrap(); + console::add_console_autoflush(self.fb_console.get()); + } + + unsafe fn init_platform(&'static self, cpu_id: usize) { + Cpu::init_local(LocalApic::new(), cpu_id as _); + + if cpu_id == 0 { + self.init_acpi_from_boot_data(); + Self::disable_8259(); + + // Initialize debug output as soon as possible + let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)))); + debug::add_sink(com1_3.port_a(), LogLevel::Debug); + // devfs::add_char_device(com1_3.port_a(), CharDeviceType::TtySerial).unwrap(); + + self.init_framebuffer(); + debug::reset(); + + log::set_logger(&LOGGER) + .map(|_| log::set_max_level(log::LevelFilter::Trace)) + .ok(); + + let ps2 = Box::leak(Box::new(PS2Controller::new( + IrqNumber::Isa(1), + IrqNumber::Isa(12), + 0x64, + 0x60, + ))); + ps2.init().unwrap(); + + // Print some stuff now that the output is initialized + infoln!( + "Yggdrasil v{} ({})", + env!("CARGO_PKG_VERSION"), + git_version!() + ); + infoln!("Initializing x86_64 platform"); + + if let Some(acpi) = self.acpi.try_get() { + self.init_platform_from_acpi(acpi); + } + + // Enable IRQs for the devices + self.timer.get().init_irq().unwrap(); + ps2.connect(self.combined_terminal.get()); + ps2.init_irq().unwrap(); + + device::register_device(self.ioapic.get()); + device::register_device(self.timer.get()); + device::register_device(ps2); + + infoln!("Device list:"); + for device in device::manager_lock().devices() { + infoln!("* {}", device.display_name()); + } + } } unsafe fn disable_8259() { @@ -370,121 +545,3 @@ impl X86_64 { pic_slave_cmd.write(0x20); } } - -/// x86-64 architecture + platform implementation -pub static ARCHITECTURE: X86_64 = X86_64 { - rsdp_phys_base: OneTimeInit::new(), - yboot_framebuffer: OneTimeInit::new(), - - acpi_processor_info: OneTimeInit::new(), - - combined_terminal: OneTimeInit::new(), - - ioapic: IoApic::new(), - timer: Hpet::uninit(), - - ps2: PS2Controller::new(IrqNumber::Isa(1), IrqNumber::Isa(12), 0x64, 0x60), - com1_3: ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)), -}; - -static LINEAR_FB: OneTimeInit = OneTimeInit::new(); -static FB_CONSOLE: OneTimeInit = OneTimeInit::new(); - -const EARLY_BUFFER_LINES: usize = 32; -static mut EARLY_CONSOLE_BUFFER: [ConsoleRow; EARLY_BUFFER_LINES] = - [ConsoleRow::zeroed(); EARLY_BUFFER_LINES]; - -fn setup_initrd(initrd_start: usize, initrd_end: usize) { - if initrd_start == 0 || initrd_end <= initrd_start { - infoln!("No initrd loaded"); - return; - } - - let start_aligned = initrd_start & !0xFFF; - let end_aligned = initrd_end & !0xFFF; - - let data = unsafe { - core::slice::from_raw_parts( - initrd_start.virtualize() as *const _, - initrd_end - initrd_start, - ) - }; - - let initrd = Initrd { - phys_page_start: start_aligned, - phys_page_len: end_aligned - start_aligned, - data, - }; - - INITRD_DATA.init(initrd); -} - -fn kernel_main(boot_data: &LoadProtocolV1) -> ! { - ARCHITECTURE - .yboot_framebuffer - .init(boot_data.opt_framebuffer); - - unsafe { - ARCHITECTURE.init_debug(); - } - - infoln!("Yggdrasil kernel git {} starting", git_version!()); - - cpuid::feature_gate(); - - if boot_data.memory_map.address > 0xFFFFFFFF { - errorln!("Unhandled case: memory map is above 4GiB"); - loop { - X86_64::wait_for_interrupt(); - } - } - - if boot_data.rsdp_address != 0 { - infoln!("ACPI RSDP at {:#x}", boot_data.rsdp_address); - ARCHITECTURE.init_rsdp(boot_data.rsdp_address as _); - } - - // Reserve initrd - let initrd_start = boot_data.initrd_address as usize; - let initrd_end = initrd_start + boot_data.initrd_size as usize; - - setup_initrd(initrd_start, initrd_end); - - // Setup physical memory allocation - unsafe { - X86_64::init_physical_memory(&boot_data.memory_map); - } - - // Allocate memory for the kernel heap - let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) - .expect("Could not allocate a block for heap"); - unsafe { - heap::init_heap(heap_base.virtualize(), 16 * 0x1000); - } - - // Add console to flush list - add_console_autoflush(FB_CONSOLE.get()); - - // Also initializes local APIC - unsafe { - Cpu::init_local(LocalApic::new(), 0); - - syscall::init_syscall(); - - // TODO per-CPU IDTs? - exception::init_exceptions(0); - - devfs::init(); - - ARCHITECTURE.init(true).unwrap(); - - if let Some(info) = ARCHITECTURE.acpi_processor_info.try_get() { - smp::start_ap_cores(info); - } - - CPU_INIT_FENCE.signal(); - - task::init().expect("Failed to initialize the scheduler"); - task::enter() - } -} diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 30766e6f..71fbc63a 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -3,6 +3,11 @@ use core::time::Duration; use abi::error::Error; use acpi::hpet::HpetInfo as AcpiHpet; +use device_api::{ + interrupt::{InterruptHandler, IrqLevel, IrqOptions, IrqTrigger}, + timer::MonotonicTimestampProviderDevice, + Device, +}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -10,16 +15,11 @@ use tock_registers::{ }; use crate::{ - arch::{x86_64::apic::IrqNumber, PLATFORM}, - device::{ - interrupt::InterruptSource, platform::Platform, timer::TimestampSource, Device, - InitializableDevice, - }, + arch::{x86_64::IrqNumber, Architecture, ARCHITECTURE}, mem::device::DeviceMemoryIo, proc::wait, sync::IrqSafeSpinlock, task::tasklet, - util::OneTimeInit, }; register_bitfields! { @@ -102,7 +102,7 @@ struct Inner { /// HPET timer group interface pub struct Hpet { - inner: OneTimeInit>, + inner: IrqSafeSpinlock, } impl Inner { @@ -135,17 +135,17 @@ impl Inner { } } -impl TimestampSource for Hpet { - fn timestamp(&self) -> Result { - let inner = self.inner.get().lock(); +impl MonotonicTimestampProviderDevice for Hpet { + fn monotonic_timestamp(&self) -> Result { + let inner = self.inner.lock(); Ok(Duration::from_millis(inner.tim0_counter)) } } -impl InterruptSource for Hpet { - fn handle_irq(&self) -> Result { +impl InterruptHandler for Hpet { + fn handle_irq(&self) -> bool { let now = { - let mut inner = self.inner.get().lock(); + let mut inner = self.inner.lock(); inner.regs.GeneralInterruptStatus.set(1); inner.tim0_counter = inner.tim0_counter.wrapping_add(1); @@ -155,7 +155,13 @@ impl InterruptSource for Hpet { wait::tick(now); tasklet::tick(now); - Ok(true) + true + } +} + +impl Device for Hpet { + fn display_name(&self) -> &'static str { + "HPET" } unsafe fn init_irq(&'static self) -> Result<(), Error> { @@ -163,8 +169,8 @@ impl InterruptSource for Hpet { const FS_IN_MS: u64 = 1000000000000; // Configure timer 0 - let intc = PLATFORM.interrupt_controller(); - let mut inner = self.inner.get().lock(); + let intc = ARCHITECTURE.external_interrupt_controller(); + let mut inner = self.inner.lock(); // Calculate the interrupt period let clk_period = inner @@ -206,9 +212,15 @@ impl InterruptSource for Hpet { let tim0_irq = tim0_irq.expect("Could not pick an IRQ for HPET TIM0"); // Bind and enable the IRQ - let irq = IrqNumber::Gsi(tim0_irq as _); - intc.register_handler(irq, self)?; - intc.configure_irq(irq, true, true)?; + let irq = IrqNumber::Gsi(tim0_irq); + intc.register_irq( + irq, + IrqOptions { + level: IrqLevel::ActiveLow, + trigger: IrqTrigger::Level, + }, + self, + )?; intc.enable_irq(irq)?; // Disable FSB interrupt route and 32 bit mode @@ -245,32 +257,16 @@ impl InterruptSource for Hpet { } } -impl InitializableDevice for Hpet { - type Options = AcpiHpet; - - unsafe fn init(&self, info: Self::Options) -> Result<(), Error> { +impl Hpet { + /// Creates a HPET instance from its ACPI definition + pub fn from_acpi(info: &AcpiHpet) -> Result { infoln!("Initializing HPET:"); infoln!("Address: {:#x}", info.base_address); let inner = unsafe { Inner::new(info.base_address) }?; - self.inner.init(IrqSafeSpinlock::new(inner)); - - Ok(()) - } -} - -impl Device for Hpet { - fn name(&self) -> &'static str { - "HPET" - } -} - -impl Hpet { - /// Constructs an uninitialized HPET instance - pub const fn uninit() -> Self { - Self { - inner: OneTimeInit::new(), - } + Ok(Self { + inner: IrqSafeSpinlock::new(inner), + }) } } diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index ec1b8820..0db12770 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -1,14 +1,15 @@ //! Intel 8042 PS/2 controller driver implemenation use abi::error::Error; +use device_api::{ + input::{KeyboardConsumer, KeyboardProducer}, + interrupt::InterruptHandler, + Device, +}; use crate::{ arch::{ - x86_64::{apic::IrqNumber, intrinsics::IoPort}, - PLATFORM, - }, - device::{ - input::KeyboardDevice, interrupt::InterruptSource, platform::Platform, tty::TtyDevice, - Device, + x86_64::{intrinsics::IoPort, IrqNumber}, + Architecture, ARCHITECTURE, }, sync::IrqSafeSpinlock, }; @@ -24,7 +25,7 @@ struct Inner { shift: bool, ctrl: bool, - tty: Option<&'static dyn TtyDevice<16>>, + consumer: Option<&'static dyn KeyboardConsumer>, // tty: Option<&'static dyn TtyDevice<16>>, } /// PS/2 controller driver @@ -80,37 +81,14 @@ impl Inner { } } -impl KeyboardDevice for PS2Controller { - fn attach(&self, terminal: &'static dyn TtyDevice<16>) { - self.inner.lock().tty.replace(terminal); +impl KeyboardProducer for PS2Controller { + fn connect(&self, to: &'static dyn KeyboardConsumer) { + self.inner.lock().consumer.replace(to); } } -impl InterruptSource for PS2Controller { - unsafe fn init_irq(&'static self) -> Result<(), Error> { - let mut inner = self.inner.lock(); - let intc = PLATFORM.interrupt_controller(); - - intc.register_handler(self.primary_irq, self)?; - - // Disable PS/2 devices from sending any further data - inner.send_command(0xAD); - inner.send_command(0xA7); - - // Flush the buffer - while inner.command.read() & Inner::STATUS_OUTPUT_FULL != 0 { - inner.data.read(); - } - - // Enable primary port - inner.send_command(0xAE); - - intc.enable_irq(self.primary_irq)?; - - Ok(()) - } - - fn handle_irq(&self) -> Result { +impl InterruptHandler for PS2Controller { + fn handle_irq(&self) -> bool { let mut count = 0; let mut inner = self.inner.lock(); loop { @@ -177,21 +155,49 @@ impl InterruptSource for PS2Controller { continue; } - if let Some(tty) = inner.tty { - tty.recv_byte(key); + if let Some(consumer) = inner.consumer { + consumer.handle_character(key).ok(); } count += 1; } - Ok(count != 0) + count != 0 } } impl Device for PS2Controller { - fn name(&self) -> &'static str { + fn display_name(&self) -> &'static str { "PS/2 Controller" } + + unsafe fn init(&'static self) -> Result<(), Error> { + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let mut inner = self.inner.lock(); + // let intc = PLATFORM.interrupt_controller(); + let intc = ARCHITECTURE.external_interrupt_controller(); + + intc.register_irq(self.primary_irq, Default::default(), self)?; + + // Disable PS/2 devices from sending any further data + inner.send_command(0xAD); + inner.send_command(0xA7); + + // Flush the buffer + while inner.command.read() & Inner::STATUS_OUTPUT_FULL != 0 { + inner.data.read(); + } + + // Enable primary port + inner.send_command(0xAE); + + intc.enable_irq(self.primary_irq)?; + + Ok(()) + } } impl PS2Controller { @@ -207,7 +213,7 @@ impl PS2Controller { data: IoPort::new(data_port), shift: false, ctrl: false, - tty: None, + consumer: None, }; Self { diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index 8aa50788..c76c4802 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -1,9 +1,10 @@ +//! Driver for x86 COM ports use abi::error::Error; +use device_api::{serial::SerialDevice, Device}; use crate::{ - arch::x86_64::{apic::IrqNumber, intrinsics::IoPort}, + arch::x86_64::{intrinsics::IoPort, IrqNumber}, debug::DebugSink, - device::{serial::SerialDevice, Device, InitializableDevice}, sync::IrqSafeSpinlock, }; @@ -13,11 +14,13 @@ struct Inner { lsr: IoPort, } +/// Single port of the COM port pair pub struct Port { inner: IrqSafeSpinlock, } -// Port pair +/// COM port pair +#[allow(unused)] pub struct ComPort { port_a: Port, port_b: Port, @@ -41,22 +44,22 @@ impl SerialDevice for Port { inner.dr.write(byte); Ok(()) } - - fn receive(&self, blocking: bool) -> Result { - todo!() - } } impl Device for Port { - fn name(&self) -> &'static str { - "com-port" + fn display_name(&self) -> &'static str { + "COM port" + } + + unsafe fn init(&'static self) -> Result<(), Error> { + Ok(()) } } impl Port { const LSR_THRE: u8 = 1 << 5; - pub const fn new(base: u16) -> Self { + const fn new(base: u16) -> Self { Self { inner: IrqSafeSpinlock::new(Inner { dr: IoPort::new(base), @@ -66,16 +69,8 @@ impl Port { } } -impl InitializableDevice for ComPort { - type Options = (); - - unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { - // TODO proper init - Ok(()) - } -} - impl ComPort { + /// Constructs a COM port pair pub const fn new(port_a: u16, port_b: u16, irq: IrqNumber) -> Self { Self { port_a: Port::new(port_a), @@ -84,6 +79,7 @@ impl ComPort { } } + /// Returns a reference to the A port of this COM pair pub fn port_a(&self) -> &Port { &self.port_a } diff --git a/src/debug.rs b/src/debug.rs index 4940ccc9..22ed097b 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,8 +2,9 @@ use core::fmt::{self, Arguments}; use abi::error::Error; +use kernel_util::util::StaticVector; -use crate::{sync::IrqSafeSpinlock, util::StaticVector}; +use crate::sync::IrqSafeSpinlock; const MAX_DEBUG_SINKS: usize = 4; diff --git a/src/device/bus/mod.rs b/src/device/bus/mod.rs index 00a9c9d6..42007f9c 100644 --- a/src/device/bus/mod.rs +++ b/src/device/bus/mod.rs @@ -1,3 +1,4 @@ //! Bus devices +#[cfg(feature = "device-tree")] pub mod simple_bus; diff --git a/src/device/bus/simple_bus.rs b/src/device/bus/simple_bus.rs index a5782d31..9fb1574c 100644 --- a/src/device/bus/simple_bus.rs +++ b/src/device/bus/simple_bus.rs @@ -1,6 +1,6 @@ //! Simple "passthrough" bus device -use crate::{arch::aarch64::devtree::DevTreeIndexNodeExt, device, device_tree_driver}; +use crate::{device::devtree, device_tree_driver}; device_tree_driver! { compatible: ["simple-bus"], @@ -11,8 +11,8 @@ device_tree_driver! { let nodes = dt.node.children(); // Iterate devices on the bus - device::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { - if let Some((device, _)) = device::probe_dt_node(&probe) { + devtree::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { + if let Some((device, _)) = devtree::probe_dt_node(&probe) { unsafe { device.init()?; } diff --git a/src/device/devtree.rs b/src/device/devtree.rs new file mode 100644 index 00000000..8882fb1e --- /dev/null +++ b/src/device/devtree.rs @@ -0,0 +1,454 @@ +//! ARM device tree utlities +use core::mem::size_of; + +use abi::error::Error; +use alloc::boxed::Box; +use device_api::{Device, DeviceId}; +use fdt_rs::{ + base::DevTree, + index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, + prelude::PropReader, +}; + +use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; + +use super::register_device; + +/// Helper macro to return the count of expressions supplied to it +#[macro_export] +macro_rules! count { + () => (0usize); + ($x:tt $($xs:tt)*) => (1usize + $crate::count!($($xs)*)); +} + +/// Registers a device driver for compatible device tree nodes +/// +/// # Usage example +/// +/// ``` +/// device_tree_driver! { +/// compatible: ["arm,pl011"], +/// probe(of) => { +/// let my_device = ...; // ... extract some info about the device ... +/// Some(Box::new(my_device)) +/// } +/// } +/// ``` +#[macro_export] +macro_rules! device_tree_driver { + ( + compatible: [$($compatible:literal),+], + probe ($node:ident) => $probe_body:block $(,)? + ) => { + const __COMPATIBLE_LEN: usize = $crate::count!($($compatible )+); + static __COMPATIBLE: [&str; __COMPATIBLE_LEN] = [$($compatible),+]; + + fn __probe($node: &$crate::device::devtree::DevTreeNodeInfo) -> + Option> $probe_body + + core::arch::global_asm!(r#" + .pushsection .dt_probes, "a" + .quad {compatible} + .quad {compatible_len} + .quad {probe_func} + .popsection + "#, + compatible = sym __COMPATIBLE, + compatible_len = const __COMPATIBLE_LEN, + probe_func = sym __probe + ); + }; +} + +const INDEX_BUFFER_SIZE: usize = 65536; + +#[repr(C, align(0x10))] +struct FdtIndexBuffer([u8; INDEX_BUFFER_SIZE]); + +static mut FDT_INDEX_BUFFER: FdtIndexBuffer = FdtIndexBuffer::zeroed(); + +impl FdtIndexBuffer { + const fn zeroed() -> Self { + Self([0; INDEX_BUFFER_SIZE]) + } +} + +/// Device tree node +pub type TNode<'a> = DevTreeIndexNode<'a, 'a, 'a>; +/// Device tree property +pub type TProp<'a> = DevTreeIndexProp<'a, 'a, 'a>; + +/// Helper trait to provide extra functionality for [DevTreeIndexProp] +pub trait DevTreeIndexPropExt { + /// Reads a cell value from single-type cell array at given cell index + fn cell1_array_item(&self, index: usize, cells: usize) -> Option; + /// Reads a cell pair from cell pair array at given pair index + fn cell2_array_item(&self, index: usize, cells0: usize, cells1: usize) -> Option<(u64, u64)>; + + /// Reads a cell value from the property at given offset + fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option; +} + +/// Helper trait to provide extra functionality for [DevTreeIndexNode] +pub trait DevTreeIndexNodeExt { + /// Returns the root node's `#address-cells` property, or the default value defined by the + /// specification if it's absent + fn address_cells(&self) -> usize { + self.get_address_cells().unwrap_or(2) + } + /// Returns the root node's `#size-cells` property, or the default value defined by the + /// specification if it's absent + fn size_cells(&self) -> usize { + self.get_size_cells().unwrap_or(1) + } + + /// Returns the #address-cells property of the node, if there is one + fn get_address_cells(&self) -> Option; + /// Returns the #size-cells property of the node, if there is one + fn get_size_cells(&self) -> Option; + + /// Prints the node contents to debug output + fn dump(&self, level: LogLevel, depth: usize); +} + +/// Extension trait for [DevTreeIndexNode] to obtain typed property values +pub trait DevTreeIndexNodePropGet { + /// Returns a property value of given type, if it exists + fn prop(&self, name: &str) -> Option; +} + +impl<'a, 'i, 'dt> DevTreeIndexNodePropGet for DevTreeIndexNode<'a, 'i, 'dt> { + fn prop(&self, name: &str) -> Option { + self.props().find_map(|prop| { + if prop.name().ok()? == name { + prop.u32(0).ok() + } else { + None + } + }) + } +} + +impl<'a, 'i, 'dt> DevTreeIndexNodePropGet<&'a str> for DevTreeIndexNode<'a, 'i, 'dt> { + fn prop(&self, name: &str) -> Option<&'a str> { + self.props().find_map(|prop| { + if prop.name().ok()? == name { + prop.str().ok() + } else { + None + } + }) + } +} + +/// Iterator for physical memory regions present in the device tree +#[derive(Clone)] +pub struct FdtMemoryRegionIter<'a> { + inner: DevTreeIndexNodeSiblingIter<'a, 'a, 'a>, + address_cells: usize, + size_cells: usize, +} + +/// Device tree wrapper struct +pub struct DeviceTree<'a> { + tree: DevTree<'a>, + index: DevTreeIndex<'a, 'a>, +} + +impl<'a> DeviceTree<'a> { + /// Constructs a device tree wrapper from the DTB virtual address. + /// + /// # Safety + /// + /// The caller must ensure the validity of the address. + pub unsafe fn from_addr(virt: usize) -> Self { + let tree = DevTree::from_raw_pointer(virt as _).unwrap(); + let index = DevTreeIndex::new(tree, &mut FDT_INDEX_BUFFER.0).unwrap(); + Self { tree, index } + } + + /// Looks up a node for a given path + pub fn node_by_path(&self, path: &str) -> Option { + find_node(self.index.root(), path.trim_start_matches('/')) + } + + /// Prints the device tree to log output + pub fn dump(&self, level: LogLevel) { + self.index.root().dump(level, 0) + } + + /// Returns the total size of the device tree in memory + pub fn size(&self) -> usize { + self.tree.totalsize() + } + + /// Returns the root node's `#address-cells` property, or the default value defined by the + /// specification if it's absent + pub fn address_cells(&self) -> usize { + self.index.root().address_cells() + } + + /// Returns the root node's `#size-cells` property, or the default value defined by the + /// specification if it's absent + pub fn size_cells(&self) -> usize { + self.index.root().size_cells() + } + + /// Returns the root node of the device tree + pub fn root(&self) -> DevTreeIndexNode { + self.index.root() + } +} + +impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { + fn get_address_cells(&self) -> Option { + self.props() + .find(|p| p.name().unwrap_or("") == "#address-cells") + .map(|p| p.u32(0).unwrap() as usize) + } + + fn get_size_cells(&self) -> Option { + self.props() + .find(|p| p.name().unwrap_or("") == "#size-cells") + .map(|p| p.u32(0).unwrap() as usize) + } + + fn dump(&self, level: LogLevel, depth: usize) { + fn indent(level: LogLevel, depth: usize) { + for _ in 0..depth { + log_print_raw!(level, " "); + } + } + + let node_name = self.name().unwrap(); + + // Don't dump these + if node_name.starts_with("virtio_mmio@") { + return; + } + + indent(level, depth); + log_print_raw!(level, "{:?} {{\n", node_name); + for prop in self.props() { + indent(level, depth + 1); + let name = prop.name().unwrap_or(""); + log_print_raw!(level, "{name:?} = "); + + match name { + "compatible" | "stdout-path" => { + log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) + } + _ => log_print_raw!(level, "{:x?}", prop.raw()), + } + + log_print_raw!(level, "\n"); + } + + for child in self.children() { + child.dump(level, depth + 1); + } + + indent(level, depth); + log_print_raw!(level, "}}\n"); + } +} + +impl<'a, 'i, 'dt> DevTreeIndexPropExt for DevTreeIndexProp<'a, 'i, 'dt> { + fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option { + match cell_size { + 1 => self.u32(u32_offset).map(|x| x as u64).ok(), + 2 => { + let high = self.u32(u32_offset).ok()? as u64; + let low = self.u32(u32_offset + 1).ok()? as u64; + + Some((high << 32) | low) + } + _ => unimplemented!(), + } + } + + fn cell1_array_item(&self, index: usize, cells: usize) -> Option { + self.read_cell(index * cells, cells) + } + + fn cell2_array_item(&self, index: usize, cells0: usize, cells1: usize) -> Option<(u64, u64)> { + let u32_index = index * (cells0 + cells1); + let cell0 = self.read_cell(u32_index, cells0)?; + let cell1 = self.read_cell(u32_index + cells0, cells1)?; + Some((cell0, cell1)) + } +} + +impl<'a> FdtMemoryRegionIter<'a> { + /// Constructs a memory region iterator for given device tree + pub fn new(dt: &'a DeviceTree) -> Self { + let inner = dt.index.root().children(); + let address_cells = dt.address_cells(); + let size_cells = dt.size_cells(); + Self { + inner, + address_cells, + size_cells, + } + } +} + +impl Iterator for FdtMemoryRegionIter<'_> { + type Item = PhysicalMemoryRegion; + + fn next(&mut self) -> Option { + loop { + let Some(item) = self.inner.next() else { + break None; + }; + + let name = item.name().unwrap_or(""); + + if name.starts_with("memory@") || name == "memory" { + let reg = item + .props() + .find(|p| p.name().unwrap_or("") == "reg") + .unwrap(); + + let (base, size) = reg + .cell2_array_item(0, self.address_cells, self.size_cells) + .unwrap(); + + break Some(PhysicalMemoryRegion { + base: base as usize, + size: size as usize, + }); + } + } + } +} + +/// Looks up a property with given name in the node +pub fn find_prop<'a>(node: &TNode<'a>, name: &str) -> Option> { + node.props().find(|p| p.name().unwrap_or("") == name) +} + +fn path_component_left(path: &str) -> (&str, &str) { + if let Some((left, right)) = path.split_once('/') { + (left, right.trim_start_matches('/')) + } else { + (path, "") + } +} + +fn find_node<'a>(at: TNode<'a>, path: &str) -> Option> { + let (item, path) = path_component_left(path); + if item.is_empty() { + assert_eq!(path, ""); + Some(at) + } else { + let child = at.children().find(|c| c.name().unwrap() == item)?; + if path.is_empty() { + Some(child) + } else { + find_node(child, path) + } + } +} + +struct DevTreeProbe<'a> { + compatible: &'static [&'static str], + probe_func: fn(&'a DevTreeNodeInfo<'a, 'a, 'a>) -> Option>, +} + +/// Provides information about a device tree node to device driver's "probe" function +pub struct DevTreeNodeInfo<'a, 'i, 'dt> { + /// #address-cells property of the parent bus/system + pub address_cells: usize, + /// #size-cells property of the parent bus/system + pub size_cells: usize, + /// Device tree node being probed + pub node: DevTreeIndexNode<'a, 'i, 'dt>, +} + +fn iter_dt_probes<'a>() -> impl Iterator> { + extern "C" { + static __dt_probes_start: u64; + static __dt_probes_end: u64; + } + + unsafe { + let base = &__dt_probes_start as *const u64; + let end = &__dt_probes_end as *const u64; + let len = (end as usize - base as usize) / (size_of::() * 3); + + (0..len).map(move |i| { + let compatible_ptr = *base.add(i * 3); + let compatible_len = *base.add(i * 3 + 1); + let probe_func_ptr = *base.add(i * 3 + 2); + + let compatible = + core::slice::from_raw_parts(compatible_ptr as *const &str, compatible_len as usize); + let probe_func = core::mem::transmute(probe_func_ptr); + + DevTreeProbe { + compatible, + probe_func, + } + }) + } +} + +fn dt_match_compatible(compatible: &str) -> Option { + iter_dt_probes().find(|probe| probe.compatible.contains(&compatible)) +} + +/// "Probes" a device tree node for any matching device, registering it if a compatible driver is +/// found +pub fn probe_dt_node(dt: &DevTreeNodeInfo) -> Option<(&'static dyn Device, DeviceId)> { + // TODO use list, not just the first item + let Some(compatible) = dt.node.prop("compatible") else { + return None; + }; + + let probe = dt_match_compatible(compatible)?; + let device = Box::leak((probe.probe_func)(dt)?); + let id = register_device(device); + Some((device, id)) +} + +/// Performs shallow walk of a device tree node and executes the visitor function on each node +pub fn enumerate_dt< + 'a, + I: Iterator>, + F: Fn(&str, DevTreeNodeInfo) -> Result<(), Error>, +>( + address_cells: usize, + size_cells: usize, + nodes: I, + f: F, +) -> Result<(), usize> { + let mut failed_count = 0; + + for node in nodes { + // Skip /cpus and /memory* + let probe = DevTreeNodeInfo { + address_cells, + size_cells, + node, + }; + + let Ok(name) = probe.node.name() else { + continue; + }; + let Some(compatible) = probe.node.prop("compatible") else { + continue; + }; + + if let Err(error) = f(compatible, probe) { + warnln!("{}: {:?}", name, error); + failed_count += 1; + } + } + + if failed_count == 0 { + Ok(()) + } else { + Err(failed_count) + } +} diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 1554c4ba..6ed38b46 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -4,6 +4,7 @@ use core::mem::size_of; use abi::{error::Error, primitive_enum}; use alloc::vec::Vec; use bitflags::bitflags; +use kernel_util::util::StaticVector; use crate::{ debug::DebugSink, @@ -13,7 +14,6 @@ use crate::{ }, sync::IrqSafeSpinlock, task::tasklet::TaskFlow, - util::StaticVector, }; const CONSOLE_ROW_LEN: usize = 128; @@ -23,36 +23,31 @@ const DEFAULT_FG_COLOR: ColorAttribute = ColorAttribute::White; const DEFAULT_BG_COLOR: ColorAttribute = ColorAttribute::Blue; primitive_enum! { + #[allow(missing_docs)] #[doc = "Color attribute of a console character"] pub enum ColorAttribute: u8 { - #[doc = "..."] Black = 0, - #[doc = "..."] Red = 1, - #[doc = "..."] Green = 2, - #[doc = "..."] Yellow = 3, - #[doc = "..."] Blue = 4, - #[doc = "..."] Magenta = 5, - #[doc = "..."] Cyan = 6, - #[doc = "..."] White = 7, } } bitflags! { + #[doc = "Extra attributes of a console character"] #[derive(Clone, Copy)] pub struct Attributes: u8 { + #[allow(missing_docs)] const BOLD = 1 << 0; } } impl ColorAttribute { - pub fn from_vt100(val: u8) -> Self { + fn from_vt100(val: u8) -> Self { match val { 0..=7 => Self::try_from(val).unwrap(), _ => ColorAttribute::Red, @@ -102,7 +97,7 @@ pub struct ConsoleBuffer { height: u32, } -pub enum EscapeState { +enum EscapeState { Normal, Escape, Csi, @@ -116,7 +111,9 @@ pub struct ConsoleState { pub cursor_col: u32, /// Current foreground color pub fg_color: ColorAttribute, + /// Current background color pub bg_color: ColorAttribute, + /// Current set of attributes pub attributes: Attributes, esc_args: StaticVector, @@ -177,6 +174,7 @@ impl ConsoleChar { } } + /// Returns the attributes of the character #[inline] pub fn attributes(self) -> (ColorAttribute, ColorAttribute, Attributes) { let fg = @@ -256,8 +254,20 @@ impl ConsoleRow { impl ConsoleBuffer { /// Constructs a fixed-size console buffer - pub const fn fixed(rows: &'static mut [ConsoleRow], height: u32) -> Self { - Self { rows, height } + pub fn new(height: u32) -> Result { + let size = size_of::() * (height as usize); + let page_count = (size + 0xFFF) / 0x1000; + let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?; + + let rows = unsafe { + core::slice::from_raw_parts_mut(pages.virtualize() as *mut ConsoleRow, height as usize) + }; + + for row in rows.iter_mut() { + row.clear(DEFAULT_BG_COLOR); + } + + Ok(Self { rows, height }) } /// Reallocates the internal buffer with a new size @@ -394,12 +404,12 @@ impl ConsoleState { // Move back one character b'D' => { if self.cursor_col > 0 { - self.cursor_col = self.cursor_col - 1; + self.cursor_col -= 1; } } // Manipulate display attributes b'm' => { - if let Some(arg) = self.esc_args.get(0) { + if let Some(arg) = self.esc_args.first() { match arg { // Reset 0 => { diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index a93e8f9d..1a675c52 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -112,11 +112,12 @@ impl FramebufferConsole { /// Constructs an instance of console from its framebuffer reference pub fn from_framebuffer( framebuffer: &'static LinearFramebuffer, - font: &'static BitmapFont<'static>, + font: Option<&'static BitmapFont<'static>>, buffer: ConsoleBuffer, ) -> Self { - let char_width = 6; - let char_height = 12; + let font = font.unwrap_or(&bitmap_font::tamzen::FONT_7x14); + let char_width = font.width(); + let char_height = font.height(); let dim = framebuffer.dimensions(); let inner = Inner { @@ -151,12 +152,6 @@ impl Inner { text.draw(self).ok(); } - fn fill_row(&mut self, y: u32, h: u32, val: u32) { - let mut fb = unsafe { self.framebuffer.lock() }; - - fb.fill_rows(y, h, val); - } - fn fill_rect(&mut self, x: u32, y: u32, w: u32, h: u32, val: u32) { let mut fb = unsafe { self.framebuffer.lock() }; diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 1df8eb75..5749643a 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -3,8 +3,9 @@ use core::ops::{Index, IndexMut}; use abi::error::Error; +use device_api::Device; -use crate::{device::Device, mem::device::DeviceMemory, sync::IrqSafeSpinlock}; +use crate::{mem::device::DeviceMemory, sync::IrqSafeSpinlock}; use super::{DisplayDevice, DisplayDimensions}; @@ -76,7 +77,7 @@ impl LinearFramebuffer { } impl Device for LinearFramebuffer { - fn name(&self) -> &'static str { + fn display_name(&self) -> &'static str { "Linear Framebuffer" } } diff --git a/src/device/display/mod.rs b/src/device/display/mod.rs index 9d6a6fea..e503e3ae 100644 --- a/src/device/display/mod.rs +++ b/src/device/display/mod.rs @@ -3,6 +3,8 @@ use super::Device; pub mod console; + +#[cfg(feature = "fb_console")] pub mod fb_console; pub mod linear_fb; diff --git a/src/device/mod.rs b/src/device/mod.rs index d7722ae7..2baf9495 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,178 +1,20 @@ //! Device management and interfaces -use core::mem::size_of; - -use abi::error::Error; -use alloc::boxed::Box; use device_api::{manager::DeviceManager, Device, DeviceId}; -use fdt_rs::{index::DevTreeIndexNode, prelude::PropReader}; -use crate::{ - arch::aarch64::devtree, - sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, -}; +use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; + +#[cfg(target_arch = "aarch64")] +pub mod devtree; pub mod bus; +pub mod display; pub mod power; pub mod serial; pub mod tty; -// pub mod display; - static DEVICE_MANAGER: IrqSafeSpinlock = IrqSafeSpinlock::new(DeviceManager::new()); -/// Helper macro to return the count of expressions supplied to it -#[macro_export] -macro_rules! count { - () => (0usize); - ($x:tt $($xs:tt)*) => (1usize + $crate::count!($($xs)*)); -} - -/// Registers a device driver for compatible device tree nodes -/// -/// # Usage example -/// -/// ``` -/// device_tree_driver! { -/// compatible: ["arm,pl011"], -/// probe(of) => { -/// let my_device = ...; // ... extract some info about the device ... -/// Some(Box::new(my_device)) -/// } -/// } -/// ``` -#[macro_export] -macro_rules! device_tree_driver { - ( - compatible: [$($compatible:literal),+], - probe ($node:ident) => $probe_body:block $(,)? - ) => { - const __COMPATIBLE_LEN: usize = $crate::count!($($compatible )+); - static __COMPATIBLE: [&str; __COMPATIBLE_LEN] = [$($compatible),+]; - - fn __probe($node: &$crate::device::DevTreeNodeInfo) -> - Option> $probe_body - - core::arch::global_asm!(r#" - .pushsection .dt_probes, "a" - .quad {compatible} - .quad {compatible_len} - .quad {probe_func} - .popsection - "#, - compatible = sym __COMPATIBLE, - compatible_len = const __COMPATIBLE_LEN, - probe_func = sym __probe - ); - }; -} - -struct DevTreeProbe<'a> { - compatible: &'static [&'static str], - probe_func: fn(&'a DevTreeNodeInfo<'a, 'a, 'a>) -> Option>, -} - -/// Provides information about a device tree node to device driver's "probe" function -pub struct DevTreeNodeInfo<'a, 'i, 'dt> { - /// #address-cells property of the parent bus/system - pub address_cells: usize, - /// #size-cells property of the parent bus/system - pub size_cells: usize, - /// Device tree node being probed - pub node: DevTreeIndexNode<'a, 'i, 'dt>, -} - -fn iter_dt_probes<'a>() -> impl Iterator> { - extern "C" { - static __dt_probes_start: u64; - static __dt_probes_end: u64; - } - - unsafe { - let base = &__dt_probes_start as *const u64; - let end = &__dt_probes_end as *const u64; - let len = (end as usize - base as usize) / (size_of::() * 3); - - (0..len).map(move |i| { - let compatible_ptr = *base.add(i * 3); - let compatible_len = *base.add(i * 3 + 1); - let probe_func_ptr = *base.add(i * 3 + 2); - - let compatible = - core::slice::from_raw_parts(compatible_ptr as *const &str, compatible_len as usize); - let probe_func = core::mem::transmute(probe_func_ptr); - - DevTreeProbe { - compatible, - probe_func, - } - }) - } -} - -fn dt_match_compatible(compatible: &str) -> Option { - iter_dt_probes().find(|probe| probe.compatible.contains(&compatible)) -} - -/// "Probes" a device tree node for any matching device, registering it if a compatible driver is -/// found -pub fn probe_dt_node(dt: &DevTreeNodeInfo) -> Option<(&'static dyn Device, DeviceId)> { - // TODO use list, not just the first item - let Some(compatible) = - devtree::find_prop(&dt.node, "compatible").and_then(|prop| prop.str().ok()) - else { - return None; - }; - - let probe = dt_match_compatible(compatible)?; - let device = Box::leak((probe.probe_func)(dt)?); - let id = register_device(device); - Some((device, id)) -} - -/// Performs shallow walk of a device tree node and executes the visitor function on each node -pub fn enumerate_dt< - 'a, - I: Iterator>, - F: Fn(&str, DevTreeNodeInfo) -> Result<(), Error>, ->( - address_cells: usize, - size_cells: usize, - nodes: I, - f: F, -) -> Result<(), usize> { - let mut failed_count = 0; - - for node in nodes { - // Skip /cpus and /memory* - let probe = DevTreeNodeInfo { - address_cells, - size_cells, - node, - }; - - let Ok(name) = probe.node.name() else { - continue; - }; - let Some(compatible) = - devtree::find_prop(&probe.node, "compatible").and_then(|prop| prop.str().ok()) - else { - continue; - }; - - if let Err(error) = f(compatible, probe) { - warnln!("{}: {:?}", name, error); - failed_count += 1; - } - } - - if failed_count == 0 { - Ok(()) - } else { - Err(failed_count) - } -} - /// Adds a device to the kernel's device table and returns the ID assigned to it pub fn register_device(device: &'static dyn Device) -> DeviceId { debugln!("Register {:?}", device.display_name()); diff --git a/src/device/power/arm_psci.rs b/src/device/power/arm_psci.rs index d7397796..4dce90a3 100644 --- a/src/device/power/arm_psci.rs +++ b/src/device/power/arm_psci.rs @@ -6,10 +6,8 @@ use device_api::{CpuBringupDevice, Device, ResetDevice}; use fdt_rs::prelude::PropReader; use crate::{ - arch::{ - aarch64::devtree::{self}, - Architecture, ArchitectureImpl, ARCHITECTURE, - }, + arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, + device::devtree, device_tree_driver, }; diff --git a/src/device/power/mod.rs b/src/device/power/mod.rs index 8b87a8c6..7cf0ce21 100644 --- a/src/device/power/mod.rs +++ b/src/device/power/mod.rs @@ -1,4 +1,10 @@ //! Power-management related device drivers -pub mod arm_psci; -pub mod sunxi_rwdog; +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub mod arm_psci; + pub mod sunxi_rwdog; + } +} diff --git a/src/device/power/sunxi_rwdog.rs b/src/device/power/sunxi_rwdog.rs index 7d47e3de..917adf60 100644 --- a/src/device/power/sunxi_rwdog.rs +++ b/src/device/power/sunxi_rwdog.rs @@ -3,19 +3,17 @@ use abi::error::Error; use alloc::boxed::Box; use device_api::{Device, ResetDevice}; +use kernel_util::util::OneTimeInit; use tock_registers::{ interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, }; use crate::{ - arch::{ - aarch64::devtree::{self, DevTreeIndexNodePropGet, DevTreeIndexPropExt}, - Architecture, ARCHITECTURE, - }, + arch::{Architecture, ARCHITECTURE}, + device::devtree::{self, DevTreeIndexNodePropGet, DevTreeIndexPropExt}, device_tree_driver, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, - util::OneTimeInit, }; register_bitfields! { diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index 757e4894..7f672fe6 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -1,4 +1,10 @@ //! Serial device interfaces -pub mod pl011; -pub mod sunxi_uart; +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub mod pl011; + pub mod sunxi_uart; + } +} diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 987eb1e2..c8d90f93 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -1,11 +1,8 @@ //! ARM PL011 driver use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; -use device_api::{ - interrupt::{InterruptHandler, IrqNumber}, - serial::SerialDevice, - Device, -}; +use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; +use kernel_util::util::OneTimeInit; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -14,17 +11,16 @@ use tock_registers::{ use vfs::CharDevice; use crate::{ - arch::{ - aarch64::devtree::{self, DevTreeIndexPropExt}, - Architecture, ARCHITECTURE, - }, + arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, debug::{self, DebugSink, LogLevel}, - device::tty::{CharRing, TtyDevice}, + device::{ + devtree::{self, DevTreeIndexPropExt}, + tty::{CharRing, TtyDevice}, + }, device_tree_driver, fs::devfs::{self, CharDeviceType}, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, - util::OneTimeInit, }; register_bitfields! { diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs index d1d24862..ca0a594f 100644 --- a/src/device/serial/sunxi_uart.rs +++ b/src/device/serial/sunxi_uart.rs @@ -2,11 +2,8 @@ use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; -use device_api::{ - interrupt::{InterruptHandler, IrqNumber}, - serial::SerialDevice, - Device, -}; +use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; +use kernel_util::util::OneTimeInit; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -15,17 +12,16 @@ use tock_registers::{ use vfs::CharDevice; use crate::{ - arch::{ - aarch64::devtree::{self, DevTreeIndexPropExt}, - Architecture, ARCHITECTURE, - }, + arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, debug::{self, DebugSink, LogLevel}, - device::tty::{CharRing, TtyDevice}, + device::{ + devtree::{self, DevTreeIndexPropExt}, + tty::{CharRing, TtyDevice}, + }, device_tree_driver, fs::devfs::{self, CharDeviceType}, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, - util::OneTimeInit, }; register_bitfields! { diff --git a/src/device/timer.rs b/src/device/timer.rs deleted file mode 100644 index bba34bd8..00000000 --- a/src/device/timer.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Time-providing device interfaces -use core::time::Duration; - -use abi::error::Error; - -use super::Device; - -/// Interface for devices capable of providing some notion of time -pub trait TimestampSource: Device { - /// Returns current time signalled by the device. The time may not be a "real" time and instead - /// is assumed to be monotonically increasing. - fn timestamp(&self) -> Result; -} diff --git a/src/device/tty.rs b/src/device/tty.rs index 44673f98..45b30bee 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -14,13 +14,13 @@ use crate::{ #[cfg(feature = "fb_console")] pub mod combined { + //! Combined console + keyboard terminal device use abi::{error::Error, io::DeviceRequest}; + use device_api::{input::KeyboardConsumer, serial::SerialDevice}; use vfs::CharDevice; use crate::device::{ display::{console::DisplayConsole, fb_console::FramebufferConsole}, - input::KeyboardDevice, - serial::SerialDevice, Device, }; @@ -29,21 +29,15 @@ pub mod combined { // TODO rewrite this /// Helper device to combine a display and a keyboard input into a single terminal pub struct CombinedTerminal { - #[allow(dead_code)] - input: &'static dyn KeyboardDevice, - output: &'static dyn DisplayConsole, + output: &'static (dyn DisplayConsole + Sync + 'static), input_ring: CharRing<16>, } impl CombinedTerminal { /// Create a combined terminal device from a keyboard and console output devices - pub fn new( - input: &'static dyn KeyboardDevice, - output: &'static FramebufferConsole, - ) -> Self { + pub fn new(output: &'static FramebufferConsole) -> Self { Self { - input, output, input_ring: CharRing::new(), } @@ -56,11 +50,14 @@ pub mod combined { } } - impl SerialDevice for CombinedTerminal { - fn receive(&self, _blocking: bool) -> Result { - todo!() + impl KeyboardConsumer for CombinedTerminal { + fn handle_character(&self, data: u8) -> Result<(), Error> { + self.recv_byte(data); + Ok(()) } + } + impl SerialDevice for CombinedTerminal { fn send(&self, byte: u8) -> Result<(), Error> { self.output.write_char(byte); Ok(()) @@ -68,7 +65,7 @@ pub mod combined { } impl Device for CombinedTerminal { - fn name(&self) -> &'static str { + fn display_name(&self) -> &'static str { "Combined terminal device" } } diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index a62c20c6..30441c39 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -6,12 +6,11 @@ use abi::{ io::{FileAttr, FileMode, FileType, OpenOptions}, }; use alloc::{boxed::Box, format, string::String}; +use kernel_util::util::OneTimeInit; use vfs::{ CharDevice, CharDeviceWrapper, Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE, }; -use crate::util::OneTimeInit; - /// Describes the kind of a character device #[derive(Debug)] pub enum CharDeviceType { @@ -24,16 +23,11 @@ pub enum CharDeviceType { struct DevfsDirectory; impl VnodeImpl for DevfsDirectory { - fn open( - &mut self, - _node: &VnodeRef, - _opts: OpenOptions, - _mode: FileMode, - ) -> Result { + fn open(&self, _node: &VnodeRef, _opts: OpenOptions, _mode: FileMode) -> Result { Ok(DIR_POSITION_FROM_CACHE) } - fn metadata(&mut self, _node: &VnodeRef) -> Result { + fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { size: 0, mode: FileMode::default_dir(), diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 513231a2..0f79554c 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -2,17 +2,15 @@ use core::ptr::NonNull; +use kernel_util::util::OneTimeInit; use memfs::block::{self, BlockAllocator}; use vfs::VnodeRef; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::{ - mem::{ - self, - phys::{self, PageUsage}, - ConvertAddress, - }, - util::OneTimeInit, +use crate::mem::{ + self, + phys::{self, PageUsage}, + ConvertAddress, }; pub mod devfs; diff --git a/src/init.rs b/src/init.rs index b61f7456..12b21c95 100644 --- a/src/init.rs +++ b/src/init.rs @@ -30,8 +30,8 @@ pub fn kinit() { { use core::time::Duration; - use device::display::console::task_update_consoles; - use task::tasklet; + use crate::device::display::console::task_update_consoles; + use crate::task::tasklet; tasklet::add_periodic( "update-console", diff --git a/src/main.rs b/src/main.rs index 6e7511fb..66d6a577 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,8 @@ arbitrary_self_types, const_mut_refs, let_chains, - linked_list_cursors + linked_list_cursors, + rustc_private )] #![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] #![warn(missing_docs)] @@ -17,7 +18,7 @@ #![no_main] use sync::SpinFence; -use task::{context::Cpu, spawn_kernel_closure}; +use task::spawn_kernel_closure; use crate::{ arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, @@ -27,6 +28,7 @@ use crate::{ extern crate yggdrasil_abi as abi; extern crate alloc; +extern crate compiler_builtins; #[macro_use] pub mod debug; @@ -42,7 +44,6 @@ pub mod proc; pub mod sync; pub mod syscall; pub mod task; -pub mod util; static CPU_INIT_FENCE: SpinFence = SpinFence::new(); @@ -75,7 +76,7 @@ pub fn kernel_main() -> ! { ARCHITECTURE.start_application_processors(); } - Cpu::init_ipi_queues(); + // Cpu::init_ipi_queues(); // Wait until all APs initialize CPU_INIT_FENCE.signal(); diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 83e941c9..9eda78af 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -286,70 +286,20 @@ pub fn validate_user_region( #[no_mangle] unsafe extern "C" fn memcpy(p0: *mut c_void, p1: *const c_void, len: usize) -> *mut c_void { - let mut offset = 0; - while offset < len { - let c = (p1 as *const u8).add(offset).read_volatile(); - (p0 as *mut u8).add(offset).write_volatile(c); - - offset += 1; - } - p0 + compiler_builtins::mem::memcpy(p0 as _, p1 as _, len) as _ } #[no_mangle] unsafe extern "C" fn memcmp(p0: *const c_void, p1: *const c_void, len: usize) -> i32 { - let mut offset = 0; - - if len == 0 { - return 0; - } - - while offset < len { - let c0 = (p0 as *const u8).add(offset).read_volatile(); - let c1 = (p1 as *const u8).add(offset).read_volatile(); - - #[allow(clippy::comparison_chain)] - if c0 > c1 { - return (c0 - c1) as i32; - } else if c0 < c1 { - return -((c1 - c0) as i32); - } - - offset += 1; - } - - 0 + compiler_builtins::mem::memcmp(p0 as _, p1 as _, len) } #[no_mangle] unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { - if dst as usize == src as usize { - return dst; - } - - if (src.add(len) as usize) <= (dst as usize) || (dst.add(len) as usize) <= (src as usize) { - return memcpy(dst, src, len); - } - - if (dst as usize) < (src as usize) { - let a = src as usize - dst as usize; - memcpy(dst, src, a); - memcpy(src as *mut _, src.add(a), len - a); - } else { - let a = dst as usize - src as usize; - memcpy(dst.add(a), dst, len - a); - memcpy(dst, src, a); - } - - dst + compiler_builtins::mem::memmove(dst as _, src as _, len) as _ } #[no_mangle] unsafe extern "C" fn memset(dst: *mut c_void, val: i32, len: usize) -> *mut c_void { - let mut offset = 0; - while offset < len { - (dst as *mut u8).add(offset).write_volatile(val as u8); - offset += 1; - } - dst + compiler_builtins::mem::memset(dst as _, val, len) as _ } diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 06b8b77d..54ce445c 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -2,6 +2,7 @@ use core::{iter::StepBy, mem::size_of, ops::Range}; use abi::error::Error; +use kernel_util::util::OneTimeInit; use crate::{ debug::LogLevel, @@ -10,7 +11,6 @@ use crate::{ ConvertAddress, /*, KERNEL_PHYS_BASE */ }, sync::IrqSafeSpinlock, - util::OneTimeInit, }; use self::manager::PhysicalMemoryManager; diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 8110ac90..697c77d5 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -1,6 +1,6 @@ //! Utilities for handling reserved memory regions -use crate::util::StaticVector; +use kernel_util::util::StaticVector; use super::PhysicalMemoryRegion; diff --git a/src/task/process.rs b/src/task/process.rs index 911cf8b8..b392e981 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -11,6 +11,7 @@ use abi::{ }; use alloc::{collections::VecDeque, rc::Rc, string::String}; use atomic_enum::atomic_enum; +use kernel_util::util::OneTimeInit; use vfs::VnodeRef; use crate::{ @@ -22,7 +23,6 @@ use crate::{ }, sync::{IrqGuard, IrqSafeSpinlock}, task::context::TaskContextImpl, - util::OneTimeInit, }; use super::{context::TaskFrame, sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; diff --git a/src/task/sched.rs b/src/task/sched.rs index f8040c31..b7df1b6d 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -3,12 +3,12 @@ // use aarch64_cpu::registers::CNTPCT_EL0; use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; use cfg_if::cfg_if; +use kernel_util::util::OneTimeInit; use crate::{ // arch::aarch64::{context::TaskContext, cpu::Cpu}, arch::{Architecture, ArchitectureImpl}, sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, - util::OneTimeInit, }; use super::{ From 818509570bbb691bd8b9c9332c9197cc8ffd50b3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 22 Aug 2023 09:43:33 +0300 Subject: [PATCH 056/211] dev: auto-allocation of console buffer in fbcon --- lib/device-api/src/bus.rs | 4 ++-- src/arch/x86_64/mod.rs | 7 ++----- src/device/display/fb_console.rs | 8 ++++---- src/device/input.rs | 10 ---------- 4 files changed, 8 insertions(+), 21 deletions(-) delete mode 100644 src/device/input.rs diff --git a/lib/device-api/src/bus.rs b/lib/device-api/src/bus.rs index b61b7e35..589540c2 100644 --- a/lib/device-api/src/bus.rs +++ b/lib/device-api/src/bus.rs @@ -1,7 +1,7 @@ use yggdrasil_abi::error::Error; -use crate::manager::DeviceManager; +use crate::{manager::DeviceManager, Device}; -pub trait Bus { +pub trait Bus: Device { fn enumerate(&self, manager: &mut DeviceManager) -> Result<(), Error>; } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index a867a5ae..76b89d0e 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -451,11 +451,8 @@ impl X86_64 { } } - self.fb_console.init(FramebufferConsole::from_framebuffer( - self.framebuffer.get(), - None, - ConsoleBuffer::new(32).unwrap(), - )); + self.fb_console + .init(FramebufferConsole::from_framebuffer(self.framebuffer.get(), None).unwrap()); debug::add_sink(self.fb_console.get(), LogLevel::Info); diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 1a675c52..952e547e 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -113,12 +113,12 @@ impl FramebufferConsole { pub fn from_framebuffer( framebuffer: &'static LinearFramebuffer, font: Option<&'static BitmapFont<'static>>, - buffer: ConsoleBuffer, - ) -> Self { + ) -> Result { let font = font.unwrap_or(&bitmap_font::tamzen::FONT_7x14); let char_width = font.width(); let char_height = font.height(); let dim = framebuffer.dimensions(); + let buffer = ConsoleBuffer::new(dim.height / char_height)?; let inner = Inner { framebuffer, @@ -132,10 +132,10 @@ impl FramebufferConsole { bg_color: 0, }; - Self { + Ok(Self { inner: IrqSafeSpinlock::new(inner), state: IrqSafeSpinlock::new(ConsoleState::new(buffer)), - } + }) } } diff --git a/src/device/input.rs b/src/device/input.rs deleted file mode 100644 index 7e462dc2..00000000 --- a/src/device/input.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! Input device interfaces - -use super::{tty::TtyDevice, Device}; - -/// Generic keyboard-like input device interface -pub trait KeyboardDevice: Device { - /// Indicates to the keyboard driver that it was attached to a terminal so it knows where to - /// send its input data to - fn attach(&self, terminal: &'static dyn TtyDevice<16>); -} From ee1a2cf2059ff58bc6128fa10e082f327a92c3e2 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 22 Aug 2023 10:00:56 +0300 Subject: [PATCH 057/211] ***: better logging facilities --- lib/vfs/Cargo.toml | 1 + lib/vfs/src/debug.rs | 23 -------------- lib/vfs/src/ioctx.rs | 7 +++-- lib/vfs/src/lib.rs | 10 ------ src/arch/x86_64/mod.rs | 33 ++------------------ src/arch/x86_64/peripherals/serial.rs | 4 +++ src/debug.rs | 45 +++++++++++++++++++++------ src/device/serial/pl011.rs | 4 +++ src/device/serial/sunxi_uart.rs | 4 +++ 9 files changed, 54 insertions(+), 77 deletions(-) delete mode 100644 lib/vfs/src/debug.rs diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index e0b452a7..10c8bf13 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -9,3 +9,4 @@ edition = "2021" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-util = { path = "../kernel-util" } bitflags = "2.3.3" +log = "0.4.20" diff --git a/lib/vfs/src/debug.rs b/lib/vfs/src/debug.rs deleted file mode 100644 index 677c9f9e..00000000 --- a/lib/vfs/src/debug.rs +++ /dev/null @@ -1,23 +0,0 @@ -use core::fmt; - -pub(crate) static mut DEBUG_HOOK: Option<&'static dyn Fn(fmt::Arguments)> = None; - -#[macro_export] -macro_rules! debug_raw { - ($($arg:tt)+) => { - $crate::debug::_debug_print(format_args!($($arg)+)) - }; -} - -#[macro_export] -macro_rules! debugln { - ($($arg:tt)+) => { - debug_raw!("[VFS] {}\n", format_args!($($arg)+)) - }; -} - -pub(crate) fn _debug_print(args: fmt::Arguments) { - if let Some(hook) = unsafe { DEBUG_HOOK.as_mut() } { - hook(args); - } -} diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index 838c11c4..db774a91 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -1,3 +1,4 @@ +use log::trace; use yggdrasil_abi::{ error::Error, io::{FileMode, OpenOptions}, @@ -51,7 +52,7 @@ impl IoContext { break; } - debugln!("resolve parent: {:?} -> {:?}", at, target); + trace!("resolve parent: {:?} -> {:?}", at, target); at = target; } } @@ -73,7 +74,7 @@ impl IoContext { break; } - debugln!("resolve node: {:?} -> {:?}", node, target); + trace!("resolve node: {:?} -> {:?}", node, target); node = target; } } @@ -92,7 +93,7 @@ impl IoContext { follow: bool, follow_mount: bool, ) -> Result { - debugln!("_find {:?} in {:?}", path, at); + trace!("find {:?} in {:?}", path, at); let at = if path.starts_with('/') { path = path.trim_start_matches('/'); diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 7c91c256..cb486bdc 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -10,9 +10,6 @@ extern crate alloc; #[cfg(test)] extern crate std; -#[macro_use] -pub(crate) mod debug; - pub(crate) mod block; pub(crate) mod char; pub(crate) mod file; @@ -70,10 +67,3 @@ fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Res Ok(()) } } - -/// # Safety -/// -/// Unsafe: only meant to be called early in kernel init sequence -pub unsafe fn init_debug_hook(hook: &'static dyn Fn(core::fmt::Arguments)) { - debug::DEBUG_HOOK.replace(hook); -} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 76b89d0e..0f7ffc80 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -24,11 +24,7 @@ use crate::{ debug::{self, LogLevel}, device::{ self, - display::{ - console::{self, ConsoleBuffer}, - fb_console::FramebufferConsole, - linear_fb::LinearFramebuffer, - }, + display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, tty::combined::CombinedTerminal, }, fs::{ @@ -126,27 +122,6 @@ impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T { #[derive(Clone, Copy)] struct AcpiHandlerImpl; -struct SimpleLogger; - -impl log::Log for SimpleLogger { - fn enabled(&self, _metadata: &log::Metadata) -> bool { - true - } - - fn log(&self, record: &log::Record) { - match record.level() { - log::Level::Warn => warnln!("{}", record.args()), - log::Level::Info => infoln!("{}", record.args()), - log::Level::Trace | log::Level::Debug => infoln!("{}", record.args()), - log::Level::Error => errorln!("{}", record.args()), - } - } - - fn flush(&self) {} -} - -static LOGGER: SimpleLogger = SimpleLogger; - impl AcpiHandler for AcpiHandlerImpl { // No actual address space modification is performed unsafe fn map_physical_region( @@ -478,11 +453,7 @@ impl X86_64 { // devfs::add_char_device(com1_3.port_a(), CharDeviceType::TtySerial).unwrap(); self.init_framebuffer(); - debug::reset(); - - log::set_logger(&LOGGER) - .map(|_| log::set_max_level(log::LevelFilter::Trace)) - .ok(); + debug::init(); let ps2 = Box::leak(Box::new(PS2Controller::new( IrqNumber::Isa(1), diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index c76c4802..61764dc1 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -31,6 +31,10 @@ impl DebugSink for Port { fn putc(&self, c: u8) -> Result<(), Error> { self.send(c) } + + fn supports_control_sequences(&self) -> bool { + true + } } impl SerialDevice for Port { diff --git a/src/debug.rs b/src/debug.rs index 22ed097b..e691d4d2 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -8,6 +8,8 @@ use crate::sync::IrqSafeSpinlock; const MAX_DEBUG_SINKS: usize = 4; +struct SimpleLogger; + /// Defines the severity of the message #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum LogLevel { @@ -85,8 +87,22 @@ struct DebugSinkWrapper { level: LogLevel, } -static DEBUG_SINKS: IrqSafeSpinlock> = - IrqSafeSpinlock::new(StaticVector::new()); +impl log::Log for SimpleLogger { + fn enabled(&self, _metadata: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + match record.level() { + log::Level::Warn => warnln!("{}", record.args()), + log::Level::Info => infoln!("{}", record.args()), + log::Level::Trace | log::Level::Debug => debugln!("{}", record.args()), + log::Level::Error => errorln!("{}", record.args()), + } + } + + fn flush(&self) {} +} impl LogLevel { fn log_prefix(self) -> &'static str { @@ -110,6 +126,18 @@ impl LogLevel { } } +impl fmt::Write for DebugSinkWrapper { + fn write_str(&mut self, s: &str) -> fmt::Result { + self.inner.puts(s).ok(); + Ok(()) + } +} + +static LOGGER: SimpleLogger = SimpleLogger; + +static DEBUG_SINKS: IrqSafeSpinlock> = + IrqSafeSpinlock::new(StaticVector::new()); + /// Prints a hex-dump of a slice, appending a virtual address offset to the output pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { for (i, b) in data.iter().enumerate() { @@ -131,13 +159,6 @@ pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { } } -impl fmt::Write for DebugSinkWrapper { - fn write_str(&mut self, s: &str) -> fmt::Result { - self.inner.puts(s).ok(); - Ok(()) - } -} - /// Adds a debugging output sink pub fn add_sink(sink: &'static dyn DebugSink, level: LogLevel) { DEBUG_SINKS @@ -146,7 +167,11 @@ pub fn add_sink(sink: &'static dyn DebugSink, level: LogLevel) { } /// Resets the debugging terminal by clearing it -pub fn reset() { +pub fn init() { + log::set_logger(&LOGGER) + .map(|_| log::set_max_level(log::LevelFilter::Trace)) + .ok(); + log_print_raw!(LogLevel::Info, "\x1b[0m"); } diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index c8d90f93..40ed12e6 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -95,6 +95,10 @@ impl DebugSink for Pl011 { fn putc(&self, byte: u8) -> Result<(), Error> { self.send(byte) } + + fn supports_control_sequences(&self) -> bool { + true + } } impl TtyDevice<16> for Pl011 { diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs index ca0a594f..8cf502be 100644 --- a/src/device/serial/sunxi_uart.rs +++ b/src/device/serial/sunxi_uart.rs @@ -67,6 +67,10 @@ impl DebugSink for SunxiUart { fn putc(&self, c: u8) -> Result<(), Error> { self.send(c) } + + fn supports_control_sequences(&self) -> bool { + true + } } impl CharDevice for SunxiUart { From 67fe71bfc40f22870332c8641c6bdadfbb229095 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 31 Aug 2023 09:13:23 +0300 Subject: [PATCH 058/211] x86-64: properly implement ISA IOAPIC redirections --- lib/device-api/src/interrupt.rs | 24 +++++++++--- src/arch/x86_64/apic/ioapic.rs | 66 ++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index 618f34dc..e8d9cf2a 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -27,12 +27,6 @@ pub enum IpiDeliveryTarget { OtherCpus, } -// #[derive(Clone, Copy, PartialEq, Eq, Debug)] -// pub enum IrqNumber { -// Private(u32), -// Shared(u32), -// } - #[derive(Default, Clone, Copy, Debug)] pub struct IrqOptions { pub level: IrqLevel, @@ -90,6 +84,24 @@ pub struct FixedInterruptTable { entries: [Option<&'static dyn InterruptHandler>; SIZE], } +impl IrqLevel { + pub fn override_default(self, value: IrqLevel) -> Self { + match self { + Self::Default => value, + _ => self, + } + } +} + +impl IrqTrigger { + pub fn override_default(self, value: IrqTrigger) -> Self { + match self { + Self::Default => value, + _ => self, + } + } +} + impl FixedInterruptTable { pub const fn new() -> Self { Self { diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index ff5b50eb..fa2c5a4b 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -30,18 +30,12 @@ const ENTRY_LOW_DESTINATION_LOGICAL: u32 = 1 << 11; const ENTRY_HIGH_APIC_ID_SHIFT: u32 = 24; -#[derive(Clone, Copy, PartialEq)] -enum ExternalTriggerMode { - Edge, - Level, -} - #[allow(dead_code)] #[derive(Clone, Copy)] struct IsaRedirection { gsi_index: u32, - polarity: bool, - trigger: ExternalTriggerMode, + level: IrqLevel, + trigger: IrqTrigger, } struct Regs { @@ -99,6 +93,8 @@ impl Inner { low |= vector; // Destination - physical low &= !ENTRY_LOW_DESTINATION_LOGICAL; + // Clear delivery mode + low &= !(0x7 << 8); // Destination APIC ID high &= !(0xFF << ENTRY_HIGH_APIC_ID_SHIFT); @@ -110,12 +106,12 @@ impl Inner { Ok(()) } - fn configure_gsi(&mut self, gsi: u32, options: IrqOptions) { + fn configure_gsi(&mut self, gsi: u32, level: IrqLevel, trigger: IrqTrigger) { assert!(gsi < self.max_gsi); let mut low = self.regs.read(REG_REDIRECTION_BASE + gsi * 2); - match options.level { + match level { IrqLevel::Default => (), IrqLevel::ActiveLow => { low |= ENTRY_LOW_POLARITY_LOW; @@ -125,7 +121,7 @@ impl Inner { } } - match options.trigger { + match trigger { IrqTrigger::Default => (), IrqTrigger::Level => { low |= ENTRY_LOW_TRIGGER_LEVEL; @@ -186,8 +182,24 @@ impl ExternalInterruptController for IoApic { table_vector ); - let gsi = self.translate_irq(irq); - inner.configure_gsi(gsi, options); + let (gsi, level, trigger) = match irq { + IrqNumber::Isa(irq) => { + if let Some(redir) = self.isa_redirections[irq as usize].as_ref() { + // Mapped to a (possibly different) GSI, but also with possibly different options + ( + redir.gsi_index, + options.level.override_default(redir.level), + options.trigger.override_default(redir.trigger), + ) + } else { + // Directly mapped to a GSI + (irq as u32, options.level, options.trigger) + } + } + IrqNumber::Gsi(irq) => (irq as u32, options.level, options.trigger), + }; + + inner.configure_gsi(gsi, level, trigger); inner.map_gsi(gsi, gsi_target_vector, bsp_apic)?; Ok(()) @@ -214,6 +226,9 @@ impl ExternalInterruptController for IoApic { impl IoApic { /// Creates an I/O APIC instance from its ACPI definition pub fn from_acpi(info: &AcpiApic) -> Result { + if info.io_apics.len() != 1 { + todo!(); + } let ioapic = info.io_apics.first().unwrap(); infoln!("I/O APIC at {:#x}", ioapic.address); @@ -222,14 +237,23 @@ impl IoApic { for redir in info.interrupt_source_overrides.iter() { let index = redir.isa_source as usize; - let polarity = match redir.polarity { - Polarity::ActiveLow => false, - // TODO this may not be correct - Polarity::ActiveHigh | Polarity::SameAsBus => true, - }; let trigger = match redir.trigger_mode { - TriggerMode::Edge | TriggerMode::SameAsBus => ExternalTriggerMode::Edge, - TriggerMode::Level => ExternalTriggerMode::Level, + // ISA IRQs are edge-triggered by default + TriggerMode::SameAsBus => IrqTrigger::Edge, + TriggerMode::Edge => IrqTrigger::Edge, + TriggerMode::Level => IrqTrigger::Level, + }; + let level = match redir.polarity { + // Level-triggered ISA IRQs are ActiveLow by default + Polarity::SameAsBus => { + if trigger == IrqTrigger::Level { + IrqLevel::ActiveLow + } else { + IrqLevel::ActiveHigh + } + } + Polarity::ActiveLow => IrqLevel::ActiveLow, + Polarity::ActiveHigh => IrqLevel::ActiveHigh, }; debugln!( @@ -239,7 +263,7 @@ impl IoApic { ); isa_redirections[index].replace(IsaRedirection { gsi_index: redir.global_system_interrupt, - polarity, + level, trigger, }); } From 9c419115e77add6bd451dc8757d7f4ab0d1f28a4 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 31 Aug 2023 09:21:24 +0300 Subject: [PATCH 059/211] x86-64: better IoPort interface --- src/arch/x86_64/intrinsics.rs | 86 ++++++++++++++++++++++---- src/arch/x86_64/mod.rs | 10 +-- src/arch/x86_64/peripherals/ps2/mod.rs | 5 +- src/arch/x86_64/peripherals/serial.rs | 5 +- 4 files changed, 88 insertions(+), 18 deletions(-) diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index c359d00e..fe8ef909 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -10,7 +10,15 @@ pub struct IoPort { _pd: PhantomData, } -impl IoPort { +/// Common interface for reading and writing I/O ports +pub trait IoPortAccess { + /// Reads a value from the port + fn read(&self) -> T; + /// Writes a value to the port + fn write(&self, value: T); +} + +impl IoPort { /// Constructs a new I/O port interface pub const fn new(address: u16) -> Self { Self { @@ -18,20 +26,76 @@ impl IoPort { _pd: PhantomData, } } +} - /// Writes a byte into the port - pub fn write(&self, value: u8) { +impl IoPortAccess for IoPort { + fn write(&self, value: u8) { unsafe { - core::arch::asm!("outb %al, %dx", in("al") value, in("dx") self.address, options(att_syntax)); + outb(self.address, value); } } - /// Reads a byte from the port - pub fn read(&self) -> u8 { - let value: u8; - unsafe { - core::arch::asm!("inb %dx, %al", in("dx") self.address, out("al") value, options(att_syntax)); - } - value + fn read(&self) -> u8 { + unsafe { inb(self.address) } } } + +impl IoPortAccess for IoPort { + fn write(&self, value: u16) { + unsafe { + outw(self.address, value); + } + } + + fn read(&self) -> u16 { + unsafe { inw(self.address) } + } +} + +impl IoPortAccess for IoPort { + fn write(&self, value: u32) { + unsafe { + outl(self.address, value); + } + } + + fn read(&self) -> u32 { + unsafe { inl(self.address) } + } +} + +#[inline] +pub unsafe fn inb(port: u16) -> u8 { + let value: u8; + core::arch::asm!("inb %dx, %al", in("dx") port, out("al") value, options(att_syntax)); + value +} + +#[inline] +pub unsafe fn inw(port: u16) -> u16 { + let value: u16; + core::arch::asm!("inb %dx, %ax", in("dx") port, out("ax") value, options(att_syntax)); + value +} + +#[inline] +pub unsafe fn inl(port: u16) -> u32 { + let value: u32; + core::arch::asm!("inb %dx, %eax", in("dx") port, out("eax") value, options(att_syntax)); + value +} + +#[inline] +pub unsafe fn outb(port: u16, value: u8) { + core::arch::asm!("outb %al, %dx", in("dx") port, in("al") value, options(att_syntax)); +} + +#[inline] +pub unsafe fn outw(port: u16, value: u16) { + core::arch::asm!("outw %ax, %dx", in("dx") port, in("ax") value, options(att_syntax)); +} + +#[inline] +pub unsafe fn outl(port: u16, value: u32) { + core::arch::asm!("outl %eax, %dx", in("dx") port, in("eax") value, options(att_syntax)); +} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 0f7ffc80..4cb15ecc 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -40,7 +40,7 @@ use crate::{ use self::{ apic::ioapic::IoApic, - intrinsics::IoPort, + intrinsics::{IoPort, IoPortAccess}, peripherals::{hpet::Hpet, ps2::PS2Controller}, smp::CPU_COUNT, }; @@ -494,10 +494,10 @@ impl X86_64 { unsafe fn disable_8259() { infoln!("Disabling i8259 PIC"); // TODO should I make a module for 8259 if I don't even use it? - let pic_master_cmd = IoPort::new(0x20); - let pic_master_data = IoPort::new(0x21); - let pic_slave_cmd = IoPort::new(0xA0); - let pic_slave_data = IoPort::new(0xA1); + let pic_master_cmd = IoPort::::new(0x20); + let pic_master_data = IoPort::::new(0x21); + let pic_slave_cmd = IoPort::::new(0xA0); + let pic_slave_data = IoPort::::new(0xA1); // Remap PIC IRQ vectors to 32.. pic_master_cmd.write(0x11); diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index 0db12770..8e50b1a5 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -8,7 +8,10 @@ use device_api::{ use crate::{ arch::{ - x86_64::{intrinsics::IoPort, IrqNumber}, + x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + IrqNumber, + }, Architecture, ARCHITECTURE, }, sync::IrqSafeSpinlock, diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index 61764dc1..f8b87635 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -3,7 +3,10 @@ use abi::error::Error; use device_api::{serial::SerialDevice, Device}; use crate::{ - arch::x86_64::{intrinsics::IoPort, IrqNumber}, + arch::x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + IrqNumber, + }, debug::DebugSink, sync::IrqSafeSpinlock, }; From 9b9a1f68b5ed753282c10b192b2dadb0c0422ce5 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 31 Aug 2023 09:25:46 +0300 Subject: [PATCH 060/211] x86-64: keep HPET enabled even without interrupts --- src/arch/x86_64/peripherals/hpet.rs | 59 +++++++++++++++++------------ 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 71fbc63a..7fa3d18c 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -107,6 +107,9 @@ pub struct Hpet { impl Inner { unsafe fn new(base: usize) -> Result { + // Femtoseconds in a millisecond + const FS_IN_MS: u64 = 1000000000000; + let regs = DeviceMemoryIo::::map("hpet", base)?; // Disable the counter first @@ -127,9 +130,29 @@ impl Inner { .modify(TimerConfigurationCapablities::TM_INT_ENB_CNF::InterruptDisabled); } + // Calculate the interrupt period + let clk_period = regs + .GeneralCapabilities + .read(GeneralCapabilities::COUNTER_CLK_PERIOD); + + // Will give an interrupt interval of 1ms + let tim0_period = FS_IN_MS / clk_period; + + // if tim0_period > 0x100000000 + // && !tim0 + // .ConfigurationCapability + // .matches_all(TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit) + // { + // panic!("Period is too large and timer doesn't support 64-bit"); + // } + + // Enable the main counter + regs.GeneralConfiguration + .modify(GeneralConfiguration::ENABLE_CNF::SET); + Ok(Self { regs, - tim0_period: 0, + tim0_period, tim0_counter: 0, }) } @@ -138,7 +161,9 @@ impl Inner { impl MonotonicTimestampProviderDevice for Hpet { fn monotonic_timestamp(&self) -> Result { let inner = self.inner.lock(); - Ok(Duration::from_millis(inner.tim0_counter)) + let counter = inner.regs.MainCounter.get(); + let period = inner.tim0_period; + Ok(Duration::from_millis(counter / period)) } } @@ -165,38 +190,22 @@ impl Device for Hpet { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - // Femtoseconds in a millisecond - const FS_IN_MS: u64 = 1000000000000; - // Configure timer 0 let intc = ARCHITECTURE.external_interrupt_controller(); let mut inner = self.inner.lock(); - // Calculate the interrupt period - let clk_period = inner - .regs - .GeneralCapabilities - .read(GeneralCapabilities::COUNTER_CLK_PERIOD); - - // Will give an interrupt interval of 1ms - let period = FS_IN_MS / clk_period; - - inner.tim0_period = period; - let tim0 = &inner.regs.Timers[0]; + // Temporarily disable the main counter + inner + .regs + .GeneralConfiguration + .modify(GeneralConfiguration::ENABLE_CNF::CLEAR); + assert!(tim0 .ConfigurationCapability .matches_all(TimerConfigurationCapablities::TM_PER_INT_CAP::SupportsPeriodic)); - if period > 0x100000000 - && !tim0 - .ConfigurationCapability - .matches_all(TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit) - { - panic!("Period is too large and timer doesn't support 64-bit"); - } - let mut tim0_irq = None; let tim0_irqs = tim0 .ConfigurationCapability @@ -247,7 +256,7 @@ impl Device for Hpet { tim0.ConfigurationCapability .modify(TimerConfigurationCapablities::TM_INT_ENB_CNF::InterruptEnabled); - // Enable the main counter + // Reenable the main counter inner .regs .GeneralConfiguration From c4e8697042108a1292628dbed3d3cb4d27cf463c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 31 Aug 2023 13:40:17 +0300 Subject: [PATCH 061/211] x86-64: dirty ACPI implementation --- Cargo.toml | 5 +- src/arch/mod.rs | 3 + src/arch/x86_64/acpi.rs | 393 ++++++++++++++++++++++++++++ src/arch/x86_64/apic/ioapic.rs | 6 +- src/arch/x86_64/apic/local.rs | 71 +++-- src/arch/x86_64/boot/mod.rs | 7 +- src/arch/x86_64/context.rs | 2 +- src/arch/x86_64/cpu.rs | 66 ++++- src/arch/x86_64/exception.rs | 14 +- src/arch/x86_64/intrinsics.rs | 4 +- src/arch/x86_64/mod.rs | 64 ++--- src/arch/x86_64/peripherals/hpet.rs | 2 +- src/arch/x86_64/smp.rs | 8 +- src/device/display/console.rs | 8 +- src/init.rs | 2 + src/main.rs | 6 +- src/mem/heap.rs | 4 +- src/panic.rs | 29 +- 18 files changed, 600 insertions(+), 94 deletions(-) create mode 100644 src/arch/x86_64/acpi.rs diff --git a/Cargo.toml b/Cargo.toml index 004e61c7..aa494d77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,8 +39,9 @@ fdt-rs = { version = "0.4.3", default-features = false } [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } -aml = "0.16.4" -acpi = "4.1.1" +aml = { git = "https://github.com/alnyan/acpi.git", version = "0.16.4" } +acpi_lib = { git = "https://github.com/alnyan/acpi.git", version = "4.1.1", package = "acpi" } +acpi-system = { git = "https://github.com/alnyan/acpi-system.git", version = "0.1.0" } [features] default = [] diff --git a/src/arch/mod.rs b/src/arch/mod.rs index ae4f3b11..8f1477ef 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -47,6 +47,9 @@ cfg_if! { pub enum CpuMessage { /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow Panic, + /// Indicates that the cores should either halt and wait for the caller to shut the system + /// down, or they should shut down by themselves, depending on the platform + Shutdown, } /// Interface for an architecture-specific facilities diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs new file mode 100644 index 00000000..bf92a485 --- /dev/null +++ b/src/arch/x86_64/acpi.rs @@ -0,0 +1,393 @@ +use core::{ + alloc::{AllocError, Allocator, GlobalAlloc, Layout}, + ptr::NonNull, + sync::atomic::Ordering, + time::Duration, +}; + +use acpi_lib::{AcpiHandler, AcpiTables, PhysicalMapping}; +use acpi_system::{ + AcpiInterruptMethod, AcpiSleepState, AcpiSystem, AcpiSystemError, EventAction, FixedEvent, +}; +use alloc::boxed::Box; +use device_api::{ + interrupt::{InterruptHandler, IpiDeliveryTarget}, + Device, +}; +use kernel_util::util::OneTimeInit; +use yggdrasil_abi::error::Error; + +use crate::{ + arch::{ + x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, + Architecture, CpuMessage, ARCHITECTURE, + }, + mem::{heap::GLOBAL_HEAP, ConvertAddress}, + sync::IrqSafeSpinlock, +}; + +use super::intrinsics; + +#[derive(Clone, Copy)] +pub struct AcpiAllocator; +#[derive(Clone, Copy)] +pub struct AcpiHandlerImpl; +struct SciHandler; + +static ACPI_SYSTEM: OneTimeInit>> = OneTimeInit::new(); + +impl Device for SciHandler { + fn display_name(&self) -> &'static str { + "ACPI interrupt handler" + } +} + +impl InterruptHandler for SciHandler { + fn handle_irq(&self) -> bool { + log::trace!("ACPI SCI received"); + ACPI_SYSTEM.get().lock().handle_sci(); + true + } +} + +unsafe impl Allocator for AcpiAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + let ptr = unsafe { GLOBAL_HEAP.alloc(layout) }; + log::trace!("ACPI alloc: {:?} -> {:p}", layout, ptr); + + if ptr.is_null() { + Err(AllocError) + } else { + unsafe { + Ok(NonNull::slice_from_raw_parts( + NonNull::new_unchecked(ptr), + layout.size(), + )) + } + } + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + log::trace!("ACPI dealloc: {:?}, {:?}", ptr, layout); + GLOBAL_HEAP.dealloc(ptr.as_ptr(), layout); + } +} + +impl acpi_system::Handler for AcpiHandlerImpl { + unsafe fn map_slice(address: u64, length: u64) -> &'static [u8] { + if address + length < 0x100000000 { + core::slice::from_raw_parts( + (address as usize).virtualize() as *const u8, + length as usize, + ) + } else { + panic!("Unhandled address: {:#x}", address) + } + } + + fn io_read_u8(port: u16) -> u8 { + let value = unsafe { intrinsics::inb(port) }; + log::trace!("io_read_u8 {:#x} <- {:#x}", port, value); + value + } + + fn io_read_u16(port: u16) -> u16 { + let value = unsafe { intrinsics::inw(port) }; + log::trace!("io_read_u16 {:#x} <- {:#x}", port, value); + value + } + + fn io_read_u32(port: u16) -> u32 { + let value = unsafe { intrinsics::inl(port) }; + log::trace!("io_read_u32 {:#x} <- {:#x}", port, value); + value + } + + fn io_write_u8(port: u16, value: u8) { + log::trace!("io_write_u8 {:#x}, {:#x}", port, value); + unsafe { intrinsics::outb(port, value) } + } + + fn io_write_u16(port: u16, value: u16) { + log::trace!("io_write_u16 {:#x}, {:#x}", port, value); + unsafe { intrinsics::outw(port, value) } + } + + fn io_write_u32(port: u16, value: u32) { + log::trace!("io_write_u32 {:#x}, {:#x}", port, value); + unsafe { intrinsics::outl(port, value) } + } + + fn mem_read_u8(address: u64) -> u8 { + let value = unsafe { (address as *const u8).virtualize().read_volatile() }; + log::trace!("mem_read_u8 {:#x} <- {:#x}", address, value); + value + } + + fn mem_read_u16(address: u64) -> u16 { + let value = if address & 0x1 == 0 { + unsafe { (address as *const u16).virtualize().read_volatile() } + } else { + unsafe { (address as *const u16).virtualize().read_unaligned() } + }; + log::trace!("mem_read_u16 {:#x} <- {:#x}", address, value); + value + } + + fn mem_read_u32(address: u64) -> u32 { + let value = if address & 0x3 == 0 { + unsafe { (address as *const u32).virtualize().read_volatile() } + } else { + unsafe { (address as *const u32).virtualize().read_unaligned() } + }; + log::trace!("mem_read_u32 {:#x} <- {:#x}", address, value); + value + } + + fn mem_read_u64(address: u64) -> u64 { + let value = if address & 0x7 == 0 { + unsafe { (address as *const u64).virtualize().read_volatile() } + } else { + unsafe { (address as *const u64).virtualize().read_unaligned() } + }; + log::trace!("mem_read_u64 {:#x} <- {:#x}", address, value); + value + } + + fn mem_write_u8(address: u64, value: u8) { + log::trace!("mem_write_u8 {:#x}, {:#x}", address, value); + unsafe { (address as *mut u8).virtualize().write_volatile(value) }; + } + + fn mem_write_u16(address: u64, value: u16) { + log::trace!("mem_write_u16 {:#x}, {:#x}", address, value); + if address & 0x1 == 0 { + unsafe { (address as *mut u16).virtualize().write_volatile(value) }; + } else { + unsafe { (address as *mut u16).virtualize().write_unaligned(value) }; + } + } + + fn mem_write_u32(address: u64, value: u32) { + log::trace!("mem_write_u32 {:#x}, {:#x}", address, value); + if address & 0x3 == 0 { + unsafe { (address as *mut u32).virtualize().write_volatile(value) }; + } else { + unsafe { (address as *mut u32).virtualize().write_unaligned(value) }; + } + } + + fn mem_write_u64(address: u64, value: u64) { + log::trace!("mem_write_u64 {:#x}, {:#x}", address, value); + if address & 0x7 == 0 { + unsafe { (address as *mut u64).virtualize().write_volatile(value) }; + } else { + unsafe { (address as *mut u64).virtualize().write_unaligned(value) }; + } + } + + fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> { + infoln!("Installing ACPI SCI handler at IRQ #{}", irq); + + let intc = ARCHITECTURE.external_interrupt_controller(); + let handler = Box::leak(Box::new(SciHandler)); + let irq = IrqNumber::Isa(irq as _); + + intc.register_irq(irq, Default::default(), handler).unwrap(); + intc.enable_irq(irq).unwrap(); + + Ok(()) + } +} + +impl aml::Handler for AcpiHandlerImpl { + fn read_io_u8(&self, port: u16) -> u8 { + ::io_read_u8(port) + } + + fn read_io_u16(&self, port: u16) -> u16 { + ::io_read_u16(port) + } + + fn read_io_u32(&self, port: u16) -> u32 { + ::io_read_u32(port) + } + + fn write_io_u8(&self, port: u16, value: u8) { + ::io_write_u8(port, value) + } + + fn write_io_u16(&self, port: u16, value: u16) { + ::io_write_u16(port, value) + } + + fn write_io_u32(&self, port: u16, value: u32) { + ::io_write_u32(port, value) + } + + fn read_u8(&self, address: usize) -> u8 { + ::mem_read_u8(address as u64) + } + + fn read_u16(&self, address: usize) -> u16 { + ::mem_read_u16(address as u64) + } + + fn read_u32(&self, address: usize) -> u32 { + ::mem_read_u32(address as u64) + } + + fn read_u64(&self, address: usize) -> u64 { + ::mem_read_u64(address as u64) + } + + fn write_u8(&self, address: usize, value: u8) { + ::mem_write_u8(address as u64, value) + } + + fn write_u16(&self, address: usize, value: u16) { + ::mem_write_u16(address as u64, value) + } + + fn write_u32(&self, address: usize, value: u32) { + ::mem_write_u32(address as u64, value) + } + + fn write_u64(&self, address: usize, value: u64) { + ::mem_write_u64(address as u64, value) + } + + fn read_pci_u8(&self, _segment: u16, _bus: u8, _device: u8, _function: u8, _offset: u16) -> u8 { + 0xFF + } + + fn read_pci_u16( + &self, + _segment: u16, + _bus: u8, + _device: u8, + _function: u8, + _offset: u16, + ) -> u16 { + 0xFFFF + } + + fn read_pci_u32( + &self, + _segment: u16, + _bus: u8, + _device: u8, + _function: u8, + _offset: u16, + ) -> u32 { + 0xFFFFFFFF + } + + fn write_pci_u8( + &self, + _segment: u16, + _bus: u8, + _device: u8, + _function: u8, + _offset: u16, + _value: u8, + ) { + } + + fn write_pci_u16( + &self, + _segment: u16, + _bus: u8, + _device: u8, + _function: u8, + _offset: u16, + _value: u16, + ) { + } + + fn write_pci_u32( + &self, + _segment: u16, + _bus: u8, + _device: u8, + _function: u8, + _offset: u16, + _value: u32, + ) { + } + + fn read_ec_u8(&self, _address: u64) -> u8 { + 0x00 + } + + fn write_ec_u8(&self, _address: u64, _value: u8) {} + + fn sleep(&self, duration: Duration) { + todo!() + } +} + +impl AcpiHandler for AcpiHandlerImpl { + // No actual address space modification is performed + unsafe fn map_physical_region( + &self, + physical_address: usize, + size: usize, + ) -> PhysicalMapping { + if physical_address <= 0xFFFFFFFF { + PhysicalMapping::new( + physical_address, + NonNull::new_unchecked(physical_address.virtualize() as *mut T), + size, + size, + *self, + ) + } else { + todo!() + } + } + + // Unmap nothing, these addresses are "virtualized" to high address space + fn unmap_physical_region(_region: &PhysicalMapping) {} +} + +pub fn init_acpi(tables: &'static AcpiTables) -> Result<(), Error> { + let mut system = AcpiSystem::new(tables, Box::new(AcpiHandlerImpl)).unwrap(); + + system.initialize(AcpiInterruptMethod::Apic).unwrap(); + + system + .enable_fixed_event( + &FixedEvent::POWER_BUTTON, + Box::new(|_| { + log::info!("Power button was pressed"); + + // TODO the correct way would be to + // 1. Nicely ask all the processes to quit + // 2. Wait for some time + // 3. Kill the remaining ones + // 4. Halt other cores + // 5. Sync filesystem + // 6. Do something with the devices + // 7. Actually enter the S5 state + + unsafe { + ARCHITECTURE + .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Shutdown) + .unwrap(); + } + + SHUTDOWN_FENCE.signal(); + SHUTDOWN_FENCE.wait_all(CPU_COUNT.load(Ordering::Acquire)); + + log::info!("CPUs are parked, can shutdown now"); + + EventAction::EnterSleepState(AcpiSleepState::S5) + }), + ) + .unwrap(); + + ACPI_SYSTEM.init(IrqSafeSpinlock::new(system)); + + Ok(()) +} diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index fa2c5a4b..9d15efa1 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -1,6 +1,6 @@ //! x86-64 I/O APIC driver implementation use abi::error::Error; -use acpi::platform::interrupt::{Apic as AcpiApic, Polarity, TriggerMode}; +use acpi_lib::platform::interrupt::{Apic as AcpiApic, Polarity, TriggerMode}; use device_api::{ interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, @@ -10,7 +10,7 @@ use device_api::{ }; use crate::{ - arch::x86_64::{apic::local::BSP_APIC_ID, IrqNumber}, + arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}, mem::ConvertAddress, sync::IrqSafeSpinlock, }; @@ -225,7 +225,7 @@ impl ExternalInterruptController for IoApic { impl IoApic { /// Creates an I/O APIC instance from its ACPI definition - pub fn from_acpi(info: &AcpiApic) -> Result { + pub fn from_acpi(info: &AcpiApic) -> Result { if info.io_apics.len() != 1 { todo!(); } diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 20f0bebb..092796fa 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -1,5 +1,10 @@ //! x86-64 Local APIC driver implementation -use device_api::{interrupt::LocalInterruptController, Device}; +use core::sync::atomic::Ordering; + +use device_api::{ + interrupt::{IpiDeliveryTarget, LocalInterruptController}, + Device, +}; use kernel_util::util::OneTimeInit; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -8,11 +13,17 @@ use tock_registers::{ }; use crate::{ - arch::{x86_64::registers::MSR_IA32_APIC_BASE, CpuMessage}, + arch::{ + x86_64::{registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT}, + CpuMessage, + }, mem::ConvertAddress, + task::Cpu, }; -use super::{APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR}; +use super::{ + APIC_IPI_VECTOR, APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR, +}; const TIMER_INTERVAL: u32 = 150000; @@ -136,10 +147,35 @@ impl LocalInterruptController for LocalApic { fn send_ipi( &self, - _target: device_api::interrupt::IpiDeliveryTarget, - _msg: Self::IpiMessage, + target: device_api::interrupt::IpiDeliveryTarget, + msg: Self::IpiMessage, ) -> Result<(), abi::error::Error> { - todo!() + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + // TODO use NMI or regular interrupt depending on severity of the message + match target { + IpiDeliveryTarget::OtherCpus => { + let local = Cpu::local_id(); + for i in 0..CPU_COUNT.load(Ordering::Acquire) { + if i != local as usize { + Cpu::push_ipi_queue(i as u32, msg); + } + } + + self.regs.ICR1.write(ICR1::PhysicalDestination.val(0)); + self.regs.ICR0.write( + ICR0::Vector.val(APIC_IPI_VECTOR + 32) + + ICR0::Destination::NMI + + ICR0::DestinationType::AllExceptThis, + ); + + Ok(()) + } + IpiDeliveryTarget::ThisCpu => todo!(), + IpiDeliveryTarget::Specific(_) => todo!(), + } } unsafe fn init_ap(&self) -> Result<(), abi::error::Error> { @@ -257,29 +293,6 @@ impl LocalApic { } } - // /// Issues an interprocessor interrupt for the target. - // /// - // /// # Safety - // /// - // /// Unsafe: this function may break the control flow on the target processors. - // pub unsafe fn send_ipi(&self, target: IpiDeliveryTarget) { - // while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { - // core::hint::spin_loop(); - // } - - // match target { - // IpiDeliveryTarget::AllExceptLocal => { - // self.regs.ICR1.write(ICR1::PhysicalDestination.val(0)); - // self.regs.ICR0.write( - // ICR0::Vector.val(APIC_IPI_VECTOR + 32) - // + ICR0::Destination::NMI - // + ICR0::DestinationType::AllExceptThis, - // ); - // } - // IpiDeliveryTarget::Specified(_) => todo!(), - // } - // } - #[inline] fn base() -> usize { MSR_IA32_APIC_BASE.read_base() as usize diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 5dd6e45c..4386db51 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -25,7 +25,8 @@ use super::smp::CPU_COUNT; // use super::ARCHITECTURE; -const BOOT_STACK_SIZE: usize = 65536; +const BOOT_STACK_SIZE: usize = 256 * 1024; +const HEAP_PAGES: usize = 256; #[repr(C, align(0x20))] struct BootStack { @@ -84,9 +85,9 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { .init_physical_memory() .expect("Failed to initialize the physical memory manager"); - let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) + let heap_base = phys::alloc_pages_contiguous(HEAP_PAGES, PageUsage::Used) .expect("Couldn't allocate memory for heap"); - heap::init_heap(heap_base.virtualize(), 16 * 0x1000); + heap::init_heap(heap_base.virtualize(), HEAP_PAGES * 0x1000); exception::init_exceptions(0); diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 3a64daa1..6e74efcd 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -88,7 +88,7 @@ impl TaskContextImpl for TaskContext { const USER_STACK_EXTRA_ALIGN: usize = 8; fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { - const KERNEL_TASK_PAGES: usize = 4; + const KERNEL_TASK_PAGES: usize = 32; let stack_base = unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() }; diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 122fa42e..57c1a7e7 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -1,16 +1,20 @@ //! Per-CPU information and data structures -use core::ptr::null_mut; +use core::{ptr::null_mut, sync::atomic::Ordering}; -use alloc::boxed::Box; +use alloc::{boxed::Box, vec::Vec}; use kernel_util::util::OneTimeInit; use tock_registers::interfaces::Writeable; use crate::{ - arch::x86_64::{gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, + arch::{ + x86_64::{gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, + CpuMessage, + }, + sync::IrqSafeSpinlock, task::sched::CpuQueue, }; -use super::apic::local::LocalApic; +use super::{apic::local::LocalApic, smp::CPU_COUNT}; /// Per-CPU data structure, only visible to the executing CPU #[repr(C, align(0x10))] @@ -28,6 +32,32 @@ pub struct Cpu { queue: OneTimeInit<&'static CpuQueue>, } +struct IpiQueue { + data: IrqSafeSpinlock>, +} + +static IPI_QUEUES: OneTimeInit> = OneTimeInit::new(); + +impl IpiQueue { + pub const fn new() -> Self { + Self { + data: IrqSafeSpinlock::new(None), + } + } + + pub fn push(&self, msg: CpuMessage) { + let mut lock = self.data.lock(); + + assert!(lock.is_none()); + lock.replace(msg); + } + + pub fn pop(&self) -> Option { + let mut lock = self.data.lock(); + lock.take() + } +} + impl Cpu { /// Initializes the per-CPU data structure. /// @@ -101,4 +131,32 @@ impl Cpu { pub fn init_queue(&self, queue: &'static CpuQueue) { self.queue.init(queue); } + + /// Inserts an IPI message to the back of the target CPU's message queue + pub fn push_ipi_queue(cpu_id: u32, msg: CpuMessage) { + let ipi_queue = &IPI_QUEUES.get()[cpu_id as usize]; + ipi_queue.push(msg); + } + + /// Pops the first IPI message received for this CPU. + /// + /// # Note + /// + /// Currently the queue consists of only one entry, so the CPU will only receive the last one. + pub fn get_ipi(&self) -> Option { + let ipi_queue = &IPI_QUEUES.get()[self.id as usize]; + ipi_queue.pop() + } + + /// Sets up global list of interprocessor message queues + pub fn init_ipi_queues() { + IPI_QUEUES.init(Vec::from_iter( + (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), + )); + } + + // /// Gets an IPI message from the processor's queue and takes corresponding actions. If there is + // /// none, this is treated as a spurious IPI and ignored. See [CpuMessage]. + // pub fn handle_ipi(&self) { + // } } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 55b1cbb1..ee5b068d 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -4,11 +4,13 @@ use core::{arch::global_asm, mem::size_of_val}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ - arch::x86_64::apic, + arch::{x86_64::apic, CpuMessage}, panic, - task::{context::TaskFrame, process::Process}, + task::{context::TaskFrame, process::Process, Cpu}, }; +use super::ARCHITECTURE; + primitive_enum! { enum ExceptionKind: u64 { DivisionError = 0, @@ -319,7 +321,13 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { } } else { if kind == ExceptionKind::NonMaskableInterrupt { - panic::panic_secondary(); + let cpu = Cpu::local(); + + if let Some(msg) = cpu.get_ipi() { + unsafe { + ARCHITECTURE.handle_ipi(msg); + } + } } kernel_exception_inner(kind, frame) diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index fe8ef909..0e51bdeb 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -74,14 +74,14 @@ pub unsafe fn inb(port: u16) -> u8 { #[inline] pub unsafe fn inw(port: u16) -> u16 { let value: u16; - core::arch::asm!("inb %dx, %ax", in("dx") port, out("ax") value, options(att_syntax)); + core::arch::asm!("inw %dx, %ax", in("dx") port, out("ax") value, options(att_syntax)); value } #[inline] pub unsafe fn inl(port: u16) -> u32 { let value: u32; - core::arch::asm!("inb %dx, %eax", in("dx") port, out("eax") value, options(att_syntax)); + core::arch::asm!("inl %dx, %eax", in("dx") port, out("eax") value, options(att_syntax)); value } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 4cb15ecc..0b2c7453 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -2,7 +2,7 @@ use core::{ptr::NonNull, sync::atomic::Ordering}; use abi::error::Error; -use acpi::{AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; +use acpi_lib::{AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; use alloc::boxed::Box; use cpu::Cpu; use device_api::{ @@ -35,10 +35,13 @@ use crate::{ phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, ConvertAddress, }, + panic, + sync::SpinFence, CPU_INIT_FENCE, }; use self::{ + acpi::{AcpiAllocator, AcpiHandlerImpl}, apic::ioapic::IoApic, intrinsics::{IoPort, IoPortAccess}, peripherals::{hpet::Hpet, ps2::PS2Controller}, @@ -50,6 +53,7 @@ use super::{Architecture, CpuMessage}; #[macro_use] pub mod intrinsics; +pub mod acpi; pub mod apic; pub mod boot; pub mod context; @@ -119,33 +123,6 @@ impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T { } } -#[derive(Clone, Copy)] -struct AcpiHandlerImpl; - -impl AcpiHandler for AcpiHandlerImpl { - // No actual address space modification is performed - unsafe fn map_physical_region( - &self, - physical_address: usize, - size: usize, - ) -> PhysicalMapping { - if physical_address <= 0xFFFFFFFF { - PhysicalMapping::new( - physical_address, - NonNull::new_unchecked(physical_address.virtualize() as *mut T), - size, - size, - *self, - ) - } else { - todo!() - } - } - - // Unmap nothing, these addresses are "virtualized" to high address space - fn unmap_physical_region(_region: &PhysicalMapping) {} -} - /// Describes which kind of bootloader data was provided to the kernel pub enum BootData { /// [yboot_proto::LoadProtocolV1] @@ -179,6 +156,8 @@ pub static ARCHITECTURE: X86_64 = X86_64 { timer: OneTimeInit::new(), }; +static SHUTDOWN_FENCE: SpinFence = SpinFence::new(); + impl Architecture for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; @@ -236,7 +215,11 @@ impl Architecture for X86_64 { unsafe fn start_application_processors(&self) { if let Some(acpi) = self.acpi.try_get() { - let Some(pinfo) = acpi.platform_info().ok().and_then(|p| p.processor_info) else { + let Some(pinfo) = acpi + .platform_info_in(AcpiAllocator) + .ok() + .and_then(|p| p.processor_info) + else { return; }; @@ -396,8 +379,8 @@ impl X86_64 { Ok(()) } - unsafe fn init_platform_from_acpi(&self, acpi: &AcpiTables) { - let platform_info = acpi.platform_info().unwrap(); + unsafe fn init_platform_from_acpi(&self, acpi: &'static AcpiTables) { + let platform_info = acpi.platform_info_in(AcpiAllocator).unwrap(); let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { panic!("Processor does not have an APIC"); @@ -407,6 +390,8 @@ impl X86_64 { let hpet = HpetInfo::new(acpi).unwrap(); self.timer.init(Hpet::from_acpi(&hpet).unwrap()); + + acpi::init_acpi(acpi).unwrap(); } unsafe fn init_framebuffer(&'static self) { @@ -512,4 +497,21 @@ impl X86_64 { pic_master_cmd.write(0x20); pic_slave_cmd.write(0x20); } + + unsafe fn handle_ipi(&self, msg: CpuMessage) { + match msg { + CpuMessage::Panic => panic::panic_secondary(), + CpuMessage::Shutdown => { + Self::set_interrupt_mask(true); + + let id = Cpu::local_id(); + infoln!("cpu{} shutdown", id); + SHUTDOWN_FENCE.signal(); + + loop { + Self::wait_for_interrupt(); + } + } + } + } } diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 7fa3d18c..78f480c8 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -2,7 +2,7 @@ use core::time::Duration; use abi::error::Error; -use acpi::hpet::HpetInfo as AcpiHpet; +use acpi_lib::hpet::HpetInfo as AcpiHpet; use device_api::{ interrupt::{InterruptHandler, IrqLevel, IrqOptions, IrqTrigger}, timer::MonotonicTimestampProviderDevice, diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index a75fe525..8735c8f6 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -4,7 +4,7 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; -use acpi::platform::{ProcessorInfo, ProcessorState}; +use acpi_lib::platform::{ProcessorInfo, ProcessorState}; use crate::{ arch::{x86_64::boot::__x86_64_ap_entry, Architecture, ArchitectureImpl}, @@ -15,7 +15,7 @@ use crate::{ task::Cpu, }; -use super::table::KERNEL_TABLES; +use super::{acpi::AcpiAllocator, table::KERNEL_TABLES}; /// The number of CPUs present in the system pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); @@ -107,7 +107,7 @@ unsafe fn start_ap_core(apic_id: u32) { /// # Safety /// /// Only meant to be called once by the BSP. -pub unsafe fn start_ap_cores(info: &ProcessorInfo) { +pub unsafe fn start_ap_cores(info: &ProcessorInfo) { let aps = &info.application_processors; if aps.is_empty() { @@ -116,7 +116,7 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo) { load_ap_bootstrap_code(); - for ap in aps { + for ap in aps.iter() { if ap.is_ap && ap.state == ProcessorState::WaitingForSipi { start_ap_core(ap.local_apic_id); } diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 6ed38b46..763015e7 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -513,11 +513,15 @@ pub fn add_console_autoflush(console: &'static dyn DisplayConsole) { CONSOLES.lock().push(console); } -/// Periodically flushes data from console buffers onto their displays -pub fn task_update_consoles() -> TaskFlow { +pub fn flush_consoles() { for console in CONSOLES.lock().iter() { let mut state = console.state().lock(); console.flush(&mut state); } +} + +/// Periodically flushes data from console buffers onto their displays +pub fn task_update_consoles() -> TaskFlow { + flush_consoles(); TaskFlow::Continue } diff --git a/src/init.rs b/src/init.rs index 12b21c95..f337cfef 100644 --- a/src/init.rs +++ b/src/init.rs @@ -24,6 +24,8 @@ fn setup_root() -> Result { /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kinit() { + loop {} + infoln!("In main"); #[cfg(feature = "fb_console")] diff --git a/src/main.rs b/src/main.rs index 66d6a577..7de3c63e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,8 @@ const_mut_refs, let_chains, linked_list_cursors, - rustc_private + rustc_private, + allocator_api )] #![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] #![warn(missing_docs)] @@ -23,6 +24,7 @@ use task::spawn_kernel_closure; use crate::{ arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, mem::heap, + task::Cpu, }; extern crate yggdrasil_abi as abi; @@ -76,7 +78,7 @@ pub fn kernel_main() -> ! { ARCHITECTURE.start_application_processors(); } - // Cpu::init_ipi_queues(); + Cpu::init_ipi_queues(); // Wait until all APs initialize CPU_INIT_FENCE.signal(); diff --git a/src/mem/heap.rs b/src/mem/heap.rs index 2c805182..56113575 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -8,7 +8,7 @@ use core::{ use linked_list_allocator::Heap; use spinning_top::Spinlock; -struct KernelAllocator { +pub struct KernelAllocator { inner: Spinlock, } @@ -47,7 +47,7 @@ unsafe impl GlobalAlloc for KernelAllocator { } #[global_allocator] -static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); +pub static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); /// Sets up kernel's global heap with given memory range. /// diff --git a/src/panic.rs b/src/panic.rs index 21a4d425..e9cb1555 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -1,22 +1,25 @@ //! Kernel panic handler code -use core::sync::atomic::{AtomicBool, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; use device_api::interrupt::IpiDeliveryTarget; use crate::{ arch::{Architecture, ArchitectureImpl, CpuMessage, ARCHITECTURE}, debug::{debug_internal, LogLevel}, + device::display::console::flush_consoles, sync::{hack_locks, SpinFence}, task::{sched::CpuQueue, Cpu}, }; -// Just a fence to ensure secondary panics don't trash the screen -static PANIC_FINISHED_FENCE: SpinFence = SpinFence::new(); - static PANIC_HANDLED_FENCE: SpinFence = SpinFence::new(); +// Just a simple sequencer to ensure secondary panics don't trash the screen +static PANIC_FINISHED_FENCE: SpinFence = SpinFence::new(); +static PANIC_SEQUENCE: AtomicU32 = AtomicU32::new(0); + /// Panic handler for CPUs other than the one that initiated it pub fn panic_secondary() -> ! { + let id = Cpu::local_id(); unsafe { ArchitectureImpl::set_interrupt_mask(true); } @@ -24,7 +27,14 @@ pub fn panic_secondary() -> ! { PANIC_HANDLED_FENCE.signal(); PANIC_FINISHED_FENCE.wait_one(); + while PANIC_SEQUENCE.load(Ordering::Acquire) != id { + core::hint::spin_loop(); + } + log_print_raw!(LogLevel::Fatal, "X"); + flush_consoles(); + + PANIC_SEQUENCE.fetch_add(1, Ordering::Release); loop { ArchitectureImpl::wait_for_interrupt(); @@ -43,6 +53,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() { + let id = Cpu::local_id(); // Let other CPUs know we're screwed unsafe { ARCHITECTURE @@ -95,8 +106,16 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); - log_print_raw!(LogLevel::Fatal, "X"); PANIC_FINISHED_FENCE.signal(); + while PANIC_SEQUENCE.load(Ordering::Acquire) != id { + core::hint::spin_loop(); + } + + log_print_raw!(LogLevel::Fatal, "X"); + + flush_consoles(); + + PANIC_SEQUENCE.fetch_add(1, Ordering::Release); unsafe { ARCHITECTURE.reset(); From 12e3e2fe28c251773909022b59c2b58210836386 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 31 Aug 2023 13:40:36 +0300 Subject: [PATCH 062/211] proc: remove the loop in kinit() --- src/init.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/init.rs b/src/init.rs index f337cfef..12b21c95 100644 --- a/src/init.rs +++ b/src/init.rs @@ -24,8 +24,6 @@ fn setup_root() -> Result { /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. pub fn kinit() { - loop {} - infoln!("In main"); #[cfg(feature = "fb_console")] From b00670ea7764d61c9e18cd56b142c620bc51ee4a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 31 Aug 2023 19:54:09 +0300 Subject: [PATCH 063/211] x86-64: implement ACPI stall function --- src/arch/x86_64/acpi.rs | 5 +++++ src/main.rs | 1 + src/util.rs | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 src/util.rs diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index bf92a485..d9a1871f 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -24,6 +24,7 @@ use crate::{ }, mem::{heap::GLOBAL_HEAP, ConvertAddress}, sync::IrqSafeSpinlock, + util, }; use super::intrinsics; @@ -198,6 +199,10 @@ impl acpi_system::Handler for AcpiHandlerImpl { Ok(()) } + + fn stall(duration: Duration) { + util::polling_sleep(duration).ok(); + } } impl aml::Handler for AcpiHandlerImpl { diff --git a/src/main.rs b/src/main.rs index 7de3c63e..63732170 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,6 +46,7 @@ pub mod proc; pub mod sync; pub mod syscall; pub mod task; +pub mod util; static CPU_INIT_FENCE: SpinFence = SpinFence::new(); diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 00000000..d03dd27f --- /dev/null +++ b/src/util.rs @@ -0,0 +1,16 @@ +use core::time::Duration; + +use yggdrasil_abi::error::Error; + +use crate::arch::{Architecture, ARCHITECTURE}; + +pub fn polling_sleep(duration: Duration) -> Result<(), Error> { + let timer = ARCHITECTURE.monotonic_timer(); + let deadline = timer.monotonic_timestamp()? + duration; + + while timer.monotonic_timestamp()? < deadline { + core::hint::spin_loop(); + } + + Ok(()) +} From 3b89f444ff575c0318f50e6188914db11a2fc361 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 2 Sep 2023 18:15:41 +0300 Subject: [PATCH 064/211] x86-64: fix ACPI init in Thinkpad E14g2 --- src/arch/x86_64/apic/ioapic.rs | 2 +- src/arch/x86_64/boot/mod.rs | 4 ++-- src/arch/x86_64/peripherals/hpet.rs | 16 ++++++++-------- src/device/display/console.rs | 2 +- src/panic.rs | 2 ++ 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 9d15efa1..736f98f5 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -227,7 +227,7 @@ impl IoApic { /// Creates an I/O APIC instance from its ACPI definition pub fn from_acpi(info: &AcpiApic) -> Result { if info.io_apics.len() != 1 { - todo!(); + warnln!("The system has multiple I/O APICs, but the kernel only knows how to use one"); } let ioapic = info.io_apics.first().unwrap(); diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 4386db51..0da96293 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -25,8 +25,8 @@ use super::smp::CPU_COUNT; // use super::ARCHITECTURE; -const BOOT_STACK_SIZE: usize = 256 * 1024; -const HEAP_PAGES: usize = 256; +const BOOT_STACK_SIZE: usize = 1024 * 1024; +const HEAP_PAGES: usize = 512; #[repr(C, align(0x20))] struct BootStack { diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 78f480c8..9e0aaa24 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -138,14 +138,6 @@ impl Inner { // Will give an interrupt interval of 1ms let tim0_period = FS_IN_MS / clk_period; - // if tim0_period > 0x100000000 - // && !tim0 - // .ConfigurationCapability - // .matches_all(TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit) - // { - // panic!("Period is too large and timer doesn't support 64-bit"); - // } - // Enable the main counter regs.GeneralConfiguration .modify(GeneralConfiguration::ENABLE_CNF::SET); @@ -196,6 +188,14 @@ impl Device for Hpet { let tim0 = &inner.regs.Timers[0]; + if inner.tim0_period > 0x100000000 + && !tim0 + .ConfigurationCapability + .matches_all(TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit) + { + panic!("Period is too large and timer doesn't support 64-bit"); + } + // Temporarily disable the main counter inner .regs diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 763015e7..5f8c05cc 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -16,7 +16,7 @@ use crate::{ task::tasklet::TaskFlow, }; -const CONSOLE_ROW_LEN: usize = 128; +const CONSOLE_ROW_LEN: usize = 80; const MAX_CSI_ARGS: usize = 8; const DEFAULT_FG_COLOR: ColorAttribute = ColorAttribute::White; diff --git a/src/panic.rs b/src/panic.rs index e9cb1555..7bfefc2a 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -49,6 +49,8 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { static PANIC_HAPPENED: AtomicBool = AtomicBool::new(false); + infoln!("{:?}", pi); + if PANIC_HAPPENED .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() From 309c5c3e9fa476d242abff263a07a6baaf7eb266 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 4 Sep 2023 11:27:16 +0300 Subject: [PATCH 065/211] x86-64: add PCIe enumeration and general PCI support --- Cargo.toml | 2 + src/arch/x86_64/mod.rs | 14 +- src/arch/x86_64/table/fixed.rs | 22 +++ src/device/bus/mod.rs | 2 + src/device/bus/pci/mod.rs | 238 +++++++++++++++++++++++++++++++ src/device/bus/pci/space/ecam.rs | 79 ++++++++++ src/device/bus/pci/space/mod.rs | 131 +++++++++++++++++ src/main.rs | 1 + src/mem/device.rs | 2 +- 9 files changed, 489 insertions(+), 2 deletions(-) create mode 100644 src/device/bus/pci/mod.rs create mode 100644 src/device/bus/pci/space/ecam.rs create mode 100644 src/device/bus/pci/space/mod.rs diff --git a/Cargo.toml b/Cargo.toml index aa494d77..b7d0abfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } aml = { git = "https://github.com/alnyan/acpi.git", version = "0.16.4" } acpi_lib = { git = "https://github.com/alnyan/acpi.git", version = "4.1.1", package = "acpi" } acpi-system = { git = "https://github.com/alnyan/acpi-system.git", version = "0.1.0" } +# TODO currently only supported here +xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } [features] default = [] diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 0b2c7453..bbf042a2 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -2,7 +2,7 @@ use core::{ptr::NonNull, sync::atomic::Ordering}; use abi::error::Error; -use acpi_lib::{AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; +use acpi_lib::{mcfg::Mcfg, AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; use alloc::boxed::Box; use cpu::Cpu; use device_api::{ @@ -24,6 +24,7 @@ use crate::{ debug::{self, LogLevel}, device::{ self, + bus::pci::PciBusManager, display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, tty::combined::CombinedTerminal, }, @@ -392,6 +393,14 @@ impl X86_64 { self.timer.init(Hpet::from_acpi(&hpet).unwrap()); acpi::init_acpi(acpi).unwrap(); + + // Enumerate PCIe root devices + // TODO can there be multiple MCFGs? + if let Ok(mcfg) = acpi.find_table::() { + for entry in mcfg.entries() { + PciBusManager::add_segment_from_mcfg(entry).unwrap(); + } + } } unsafe fn init_framebuffer(&'static self) { @@ -469,6 +478,9 @@ impl X86_64 { device::register_device(self.timer.get()); device::register_device(ps2); + // Initialize devices from PCI bus + PciBusManager::setup_bus_devices().unwrap(); + infoln!("Device list:"); for device in device::manager_lock().devices() { infoln!("* {}", device.display_name()); diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs index 0bb7f2a8..ed6cbc7a 100644 --- a/src/arch/x86_64/table/fixed.rs +++ b/src/arch/x86_64/table/fixed.rs @@ -72,6 +72,28 @@ impl FixedTables { Ok(virt) } else { // 4KiB mappings + // Check if a mapping to that address already exists + if self.device_l3i >= count { + for i in 0..self.device_l3i { + let mut matches = true; + + for j in 0..count { + let page = phys + j * 0x1000; + let existing = self.device_l3[i].as_page().unwrap(); + + if page != existing { + matches = false; + break; + } + } + + if matches { + let virt = DEVICE_VIRT_OFFSET + (i << 12); + return Ok(virt); + } + } + } + if self.device_l3i + count > 512 { return Err(Error::OutOfMemory); } diff --git a/src/device/bus/mod.rs b/src/device/bus/mod.rs index 42007f9c..e6810506 100644 --- a/src/device/bus/mod.rs +++ b/src/device/bus/mod.rs @@ -2,3 +2,5 @@ #[cfg(feature = "device-tree")] pub mod simple_bus; + +pub mod pci; diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs new file mode 100644 index 00000000..3754b413 --- /dev/null +++ b/src/device/bus/pci/mod.rs @@ -0,0 +1,238 @@ +use core::fmt; + +use acpi_lib::mcfg::McfgEntry; +use alloc::{rc::Rc, vec::Vec}; +use device_api::Device; +use yggdrasil_abi::error::Error; + +mod space; + +pub use space::{ + ecam::PciEcam, PciConfigSpace, PciConfigurationSpace, PciLegacyConfigurationSpace, +}; + +use crate::sync::IrqSafeSpinlock; + +pub const PCI_COMMAND_IO: u16 = 1 << 0; +pub const PCI_COMMAND_MEMORY: u16 = 1 << 1; +pub const PCI_COMMAND_BUS_MASTER: u16 = 1 << 2; +pub const PCI_COMMAND_INTERRUPT_DISABLE: u16 = 1 << 10; + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct PciAddress { + pub segment: u8, + pub bus: u8, + pub device: u8, + pub function: u8, +} + +// TODO other attributes +#[derive(Debug)] +pub enum PciBaseAddress { + Memory(usize), + Io(u16), +} + +#[derive(Debug)] +pub struct PciDeviceInfo { + pub address: PciAddress, + pub config_space: PciConfigSpace, +} + +pub trait FromPciBus: Sized { + fn from_pci_bus(info: &PciDeviceInfo) -> Result; +} + +pub struct PciBusDevice { + info: PciDeviceInfo, + driver: Option>, +} + +pub struct PciBusSegment { + segment_number: u8, + bus_number_start: u8, + bus_number_end: u8, + ecam_phys_base: Option, + + devices: Vec, +} + +pub struct PciBusManager { + segments: Vec, +} + +impl PciBusSegment { + fn probe_config_space(&self, address: PciAddress) -> Result, Error> { + match self.ecam_phys_base { + Some(ecam_phys_base) => Ok(unsafe { + PciEcam::probe_raw_parts(ecam_phys_base, self.bus_number_start, address)? + } + .map(PciConfigSpace::Ecam)), + None => todo!(), + } + } + + fn enumerate_function( + &mut self, + parent: &mut PciBusManager, + address: PciAddress, + ) -> Result<(), Error> { + let Some(config) = self.probe_config_space(address)? else { + return Ok(()); + }; + + let header_type = config.header_type(); + + // Enumerate multi-function devices + if address.function == 0 && header_type & 0x80 != 0 { + for function in 1..8 { + self.enumerate_function(parent, address.with_function(function))?; + } + } + + // PCI-to-PCI bridge + // if config.class_code() == 0x06 && config.subclass() == 0x04 { + // let secondary_bus = config.secondary_bus(); + // // TODO + // } + + let info = PciDeviceInfo { + address, + config_space: config, + }; + self.devices.push(PciBusDevice { info, driver: None }); + + Ok(()) + } + + fn enumerate_bus(&mut self, parent: &mut PciBusManager, bus: u8) -> Result<(), Error> { + let address = PciAddress::for_bus(self.segment_number, bus); + + for i in 0..32 { + let device_address = address.with_device(i); + + self.enumerate_function(parent, device_address)?; + } + + Ok(()) + } + + pub fn enumerate(&mut self, parent: &mut PciBusManager) -> Result<(), Error> { + for bus in self.bus_number_start..self.bus_number_end { + self.enumerate_bus(parent, bus)?; + } + Ok(()) + } +} + +impl PciBusManager { + pub const fn new() -> Self { + Self { + segments: Vec::new(), + } + } + + pub fn setup_bus_devices() -> Result<(), Error> { + Self::walk_bus_devices(|device| { + setup_bus_device(device)?; + Ok(true) + }) + } + + pub fn walk_bus_devices Result>( + mut f: F, + ) -> Result<(), Error> { + let mut this = PCI_MANAGER.lock(); + + for segment in this.segments.iter_mut() { + for device in segment.devices.iter_mut() { + if !f(device)? { + return Ok(()); + } + } + } + + Ok(()) + } + + pub fn add_segment_from_mcfg(entry: &McfgEntry) -> Result<(), Error> { + let mut bus_segment = PciBusSegment { + segment_number: entry.pci_segment_group as u8, + bus_number_start: entry.bus_number_start, + bus_number_end: entry.bus_number_end, + ecam_phys_base: Some(entry.base_address as usize), + + devices: Vec::new(), + }; + + let mut this = PCI_MANAGER.lock(); + bus_segment.enumerate(&mut *this)?; + this.segments.push(bus_segment); + + Ok(()) + } +} + +impl fmt::Display for PciAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}:{}", self.bus, self.device, self.function) + } +} + +impl PciAddress { + pub const fn for_bus(segment: u8, bus: u8) -> Self { + Self { + segment, + bus, + device: 0, + function: 0, + } + } + + pub const fn with_device(self, device: u8) -> Self { + Self { + device, + function: 0, + ..self + } + } + + pub const fn with_function(self, function: u8) -> Self { + Self { function, ..self } + } +} + +impl PciConfigurationSpace for PciConfigSpace { + fn read_u32(&self, offset: usize) -> u32 { + match self { + Self::Ecam(ecam) => ecam.read_u32(offset), + _ => todo!(), + } + } +} + +fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> { + if device.driver.is_some() { + return Ok(()); + } + + let config = &device.info.config_space; + + debugln!( + "{}: {:04x}:{:04x}", + device.info.address, + config.vendor_id(), + config.device_id() + ); + + // Match by class + match (config.class_code(), config.subclass(), config.prog_if()) { + _ => { + debugln!(" -> No driver"); + } + } + + Ok(()) +} + +static PCI_MANAGER: IrqSafeSpinlock = IrqSafeSpinlock::new(PciBusManager::new()); diff --git a/src/device/bus/pci/space/ecam.rs b/src/device/bus/pci/space/ecam.rs new file mode 100644 index 00000000..078e2704 --- /dev/null +++ b/src/device/bus/pci/space/ecam.rs @@ -0,0 +1,79 @@ +use yggdrasil_abi::error::Error; + +use crate::{ + device::bus::pci::{PciConfigSpace, PciDeviceInfo}, + mem::{device::DeviceMemory, ConvertAddress}, +}; + +use super::{PciAddress, PciConfigurationSpace}; + +#[derive(Debug)] +#[repr(transparent)] +pub struct PciEcam { + mapping: DeviceMemory, +} + +// Only used for probing +#[derive(Debug)] +#[repr(transparent)] +struct PciRawEcam { + virt_addr: usize, +} + +impl PciConfigurationSpace for PciRawEcam { + fn read_u32(&self, offset: usize) -> u32 { + assert_eq!(offset & 3, 0); + unsafe { ((self.virt_addr + offset) as *const u32).read_volatile() } + } +} + +impl PciConfigurationSpace for PciEcam { + fn read_u32(&self, offset: usize) -> u32 { + assert_eq!(offset & 3, 0); + unsafe { ((self.mapping.base() + offset) as *const u32).read_volatile() } + } +} + +impl PciEcam { + pub unsafe fn map(phys_addr: usize) -> Result { + // TODO check align + let mapping = DeviceMemory::map("pcie-ecam", phys_addr, 0x1000)?; + + Ok(PciEcam { mapping }) + } + + pub unsafe fn from_raw_parts( + segment_phys_addr: usize, + bus_offset: u8, + address: PciAddress, + ) -> Result { + todo!() + } + + pub unsafe fn probe_raw_parts( + segment_phys_addr: usize, + bus_offset: u8, + address: PciAddress, + ) -> Result, Error> { + let phys_addr = segment_phys_addr + + ((address.bus - bus_offset) as usize * 256 + + address.device as usize * 8 + + address.function as usize) + * 0x1000; + + if phys_addr + 0xFFF < 0x100000000 { + // Probe without allocating a mapping + let raw = PciRawEcam { + virt_addr: phys_addr.virtualize(), + }; + + if !raw.is_valid() { + return Ok(None); + } + + Self::map(phys_addr).map(Some) + } else { + todo!() + } + } +} diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs new file mode 100644 index 00000000..33d2ea27 --- /dev/null +++ b/src/device/bus/pci/space/mod.rs @@ -0,0 +1,131 @@ +use super::{PciAddress, PciBaseAddress, PciEcam}; + +pub(super) mod ecam; + +macro_rules! pci_config_field_getter { + (u32, $name:ident, $offset:expr) => { + fn $name(&self) -> u32 { + self.read_u32($offset) + } + }; + + (u16, $name:ident, $offset:expr) => { + fn $name(&self) -> u16 { + self.read_u16($offset) + } + }; + + (u8, $name:ident, $offset:expr) => { + fn $name(&self) -> u8 { + self.read_u8($offset) + } + }; +} + +macro_rules! pci_config_field { + ($offset:expr => $ty:ident, $getter:ident $(, $setter:ident)?) => { + pci_config_field_getter!($ty, $getter, $offset); + + $( + fn $setter(&self, value: $ty) { + todo!() + } + )? + }; +} + +#[derive(Debug)] +#[repr(transparent)] +pub struct PciLegacyConfigurationSpace { + address: PciAddress, +} + +#[derive(Debug)] +pub enum PciConfigSpace { + Legacy(PciAddress), + Ecam(PciEcam), +} + +pub trait PciConfigurationSpace { + fn read_u32(&self, offset: usize) -> u32; + + fn read_u16(&self, offset: usize) -> u16 { + assert_eq!(offset & 1, 0); + let value = self.read_u32(offset & !3); + (value >> ((offset & 3) * 8)) as u16 + } + + fn read_u8(&self, offset: usize) -> u8 { + let value = self.read_u32(offset & !3); + (value >> ((offset & 3) * 8)) as u8 + } + + fn is_valid(&self) -> bool { + self.vendor_id() != 0xFFFF && self.device_id() != 0xFFFF + } + + pci_config_field!(0x00 => u16, vendor_id); + pci_config_field!(0x02 => u16, device_id); + pci_config_field!(0x04 => u16, command, set_command); + pci_config_field!(0x06 => u16, status); + + pci_config_field!(0x08 => u8, rev_id); + pci_config_field!(0x09 => u8, prog_if); + pci_config_field!(0x0A => u8, subclass); + pci_config_field!(0x0B => u8, class_code); + + // ... + pci_config_field!(0x0E => u8, header_type); + // Header Type 1 only + pci_config_field!(0x19 => u8, secondary_bus); + + // Header Type 0 only + fn bar(&self, index: usize) -> Option { + assert!(index < 6); + + if index % 2 == 0 { + let w0 = self.read_u32(0x10 + index * 4); + + match w0 & 0 { + 0 => match (w0 >> 1) & 3 { + 0 => { + // 32-bit memory BAR + Some(PciBaseAddress::Memory((w0 as usize) & !0xF)) + } + 2 => { + // 64-bit memory BAR + let w1 = self.read_u32(0x10 + (index + 1) * 4); + Some(PciBaseAddress::Memory( + ((w1 as usize) << 32) | ((w0 as usize) & !0xF), + )) + } + _ => unimplemented!(), + }, + 1 => todo!(), + _ => unreachable!(), + } + } else { + let prev_w0 = self.read_u32(0x10 + (index - 1) * 4); + if prev_w0 & 0x7 == 0x4 { + // Previous BAR is 64-bit memory and this one is its continuation + return None; + } + + let w0 = self.read_u32(0x10 + index * 4); + + match w0 & 0 { + 0 => match (w0 >> 1) & 3 { + 0 => { + // 32-bit memory BAR + Some(PciBaseAddress::Memory((w0 as usize) & !0xF)) + } + // TODO can 64-bit BARs not be on a 64-bit boundary? + 2 => todo!(), + _ => unimplemented!(), + }, + 1 => todo!(), + _ => unreachable!(), + } + } + } +} diff --git a/src/main.rs b/src/main.rs index 63732170..c6a299f7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ //! osdev-x kernel crate #![feature( + decl_macro, naked_functions, asm_const, panic_info_message, diff --git a/src/mem/device.rs b/src/mem/device.rs index f9b9f36b..ca7a2491 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -6,7 +6,7 @@ use abi::error::Error; use crate::arch::{Architecture, ARCHITECTURE}; /// Generic MMIO access mapping -#[derive(Clone)] +#[derive(Clone, Debug)] #[allow(unused)] pub struct DeviceMemory { name: &'static str, From f88ca9b6d95d87b32c79c44ead36d2020f0a0c0b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 5 Sep 2023 16:13:15 +0300 Subject: [PATCH 066/211] refactor: fix kernel warnings --- src/arch/x86_64/acpi.rs | 6 +- src/arch/x86_64/exception.rs | 3 +- src/arch/x86_64/intrinsics.rs | 30 +++++++ src/arch/x86_64/mod.rs | 4 +- src/arch/x86_64/peripherals/hpet.rs | 2 +- src/device/bus/pci/mod.rs | 47 ++++++++-- src/device/bus/pci/space/ecam.rs | 28 +++--- src/device/bus/pci/space/mod.rs | 128 ++++++++++++++++++++++------ src/device/display/console.rs | 1 + src/mem/heap.rs | 2 + src/panic.rs | 2 +- src/util.rs | 2 + 12 files changed, 203 insertions(+), 52 deletions(-) diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index d9a1871f..3a24c0f2 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -1,3 +1,4 @@ +//! x86-64 implementation of ACPI management interfaces use core::{ alloc::{AllocError, Allocator, GlobalAlloc, Layout}, ptr::NonNull, @@ -30,8 +31,10 @@ use crate::{ use super::intrinsics; #[derive(Clone, Copy)] +#[doc(hidden)] pub struct AcpiAllocator; #[derive(Clone, Copy)] +#[doc(hidden)] pub struct AcpiHandlerImpl; struct SciHandler; @@ -328,7 +331,7 @@ impl aml::Handler for AcpiHandlerImpl { fn write_ec_u8(&self, _address: u64, _value: u8) {} fn sleep(&self, duration: Duration) { - todo!() + util::polling_sleep(duration).unwrap(); } } @@ -356,6 +359,7 @@ impl AcpiHandler for AcpiHandlerImpl { fn unmap_physical_region(_region: &PhysicalMapping) {} } +/// Initializes ACPI management pub fn init_acpi(tables: &'static AcpiTables) -> Result<(), Error> { let mut system = AcpiSystem::new(tables, Box::new(AcpiHandlerImpl)).unwrap(); diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index ee5b068d..0db3f6ae 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -4,8 +4,7 @@ use core::{arch::global_asm, mem::size_of_val}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ - arch::{x86_64::apic, CpuMessage}, - panic, + arch::x86_64::apic, task::{context::TaskFrame, process::Process, Cpu}, }; diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index 0e51bdeb..cfffb596 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -64,6 +64,11 @@ impl IoPortAccess for IoPort { } } +/// Reads a byte from the I/O port +/// +/// # Safety +/// +/// Provides direct access to port I/O, unsafe. #[inline] pub unsafe fn inb(port: u16) -> u8 { let value: u8; @@ -71,6 +76,11 @@ pub unsafe fn inb(port: u16) -> u8 { value } +/// Reads a 16-bit value from the I/O port +/// +/// # Safety +/// +/// Provides direct access to port I/O, unsafe. #[inline] pub unsafe fn inw(port: u16) -> u16 { let value: u16; @@ -78,6 +88,11 @@ pub unsafe fn inw(port: u16) -> u16 { value } +/// Reads a 32-bit value from the I/O port +/// +/// # Safety +/// +/// Provides direct access to port I/O, unsafe. #[inline] pub unsafe fn inl(port: u16) -> u32 { let value: u32; @@ -85,16 +100,31 @@ pub unsafe fn inl(port: u16) -> u32 { value } +/// Writes a byte to the I/O port. +/// +/// # Safety +/// +/// Provides direct access to port I/O, unsafe. #[inline] pub unsafe fn outb(port: u16, value: u8) { core::arch::asm!("outb %al, %dx", in("dx") port, in("al") value, options(att_syntax)); } +/// Writes a 16-bit value to the I/O port. +/// +/// # Safety +/// +/// Provides direct access to port I/O, unsafe. #[inline] pub unsafe fn outw(port: u16, value: u16) { core::arch::asm!("outw %ax, %dx", in("dx") port, in("ax") value, options(att_syntax)); } +/// Writes a 32-bit value to the I/O port. +/// +/// # Safety +/// +/// Provides direct access to port I/O, unsafe. #[inline] pub unsafe fn outl(port: u16, value: u32) { core::arch::asm!("outl %eax, %dx", in("dx") port, in("eax") value, options(att_syntax)); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index bbf042a2..312f52cf 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,8 +1,8 @@ //! x86-64 architecture and platform implementation -use core::{ptr::NonNull, sync::atomic::Ordering}; +use core::sync::atomic::Ordering; use abi::error::Error; -use acpi_lib::{mcfg::Mcfg, AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping}; +use acpi_lib::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel}; use alloc::boxed::Box; use cpu::Cpu; use device_api::{ diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 9e0aaa24..05e76021 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -184,7 +184,7 @@ impl Device for Hpet { unsafe fn init_irq(&'static self) -> Result<(), Error> { // Configure timer 0 let intc = ARCHITECTURE.external_interrupt_controller(); - let mut inner = self.inner.lock(); + let inner = self.inner.lock(); let tim0 = &inner.regs.Timers[0]; diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index 3754b413..5eb90692 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -1,7 +1,9 @@ +//! PCI/PCIe bus interfaces use core::fmt; use acpi_lib::mcfg::McfgEntry; use alloc::{rc::Rc, vec::Vec}; +use bitflags::bitflags; use device_api::Device; use yggdrasil_abi::error::Error; @@ -13,41 +15,64 @@ pub use space::{ use crate::sync::IrqSafeSpinlock; -pub const PCI_COMMAND_IO: u16 = 1 << 0; -pub const PCI_COMMAND_MEMORY: u16 = 1 << 1; -pub const PCI_COMMAND_BUS_MASTER: u16 = 1 << 2; -pub const PCI_COMMAND_INTERRUPT_DISABLE: u16 = 1 << 10; +bitflags! { + /// Command register of the PCI configuration space + pub struct PciCommandRegister: u16 { + /// If set, I/O access to the device is enabled + const ENABLE_IO = 1 << 0; + /// If set, memory-mapped access to the device is enabled + const ENABLE_MEMORY = 1 << 1; + /// If set, the device can generate PCI bus accesses on its own + const BUS_MASTER = 1 << 2; + /// If set, interrupts are masked from being raised + const DISABLE_INTERRUPTS = 1 << 10; + } +} +/// Represents the address of a single object on a bus (or the bus itself) #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct PciAddress { + /// PCIe segment group, ignored (?) with PCI pub segment: u8, + /// Bus number pub bus: u8, + /// Slot/device number pub device: u8, + /// Function number pub function: u8, } -// TODO other attributes +/// Address provided by PCI configuration space Base Address Register #[derive(Debug)] pub enum PciBaseAddress { + /// 32/64-bit memory address Memory(usize), + /// I/O space address Io(u16), } +/// Describes a PCI device #[derive(Debug)] pub struct PciDeviceInfo { + /// Address of the device pub address: PciAddress, + /// Configuration space access method pub config_space: PciConfigSpace, } +/// Interface for "taking" PCI devices from the bus pub trait FromPciBus: Sized { + /// Constructs an instance of a driver for the device using the information provided fn from_pci_bus(info: &PciDeviceInfo) -> Result; } +/// Used to store PCI bus devices which were enumerated by the kernel pub struct PciBusDevice { info: PciDeviceInfo, driver: Option>, } +/// Represents a single PCIe bus segment pub struct PciBusSegment { segment_number: u8, bus_number_start: u8, @@ -57,6 +82,7 @@ pub struct PciBusSegment { devices: Vec, } +/// Manager struct to store and control all PCI devices in the system pub struct PciBusManager { segments: Vec, } @@ -117,6 +143,7 @@ impl PciBusSegment { Ok(()) } + /// Enumerates the bus segment, placing found devices into the manager pub fn enumerate(&mut self, parent: &mut PciBusManager) -> Result<(), Error> { for bus in self.bus_number_start..self.bus_number_end { self.enumerate_bus(parent, bus)?; @@ -126,12 +153,14 @@ impl PciBusSegment { } impl PciBusManager { - pub const fn new() -> Self { + const fn new() -> Self { Self { segments: Vec::new(), } } + /// Walks the bus device list and calls init/init_irq functions on any devices with associated + /// drivers pub fn setup_bus_devices() -> Result<(), Error> { Self::walk_bus_devices(|device| { setup_bus_device(device)?; @@ -139,6 +168,8 @@ impl PciBusManager { }) } + /// Iterates over the bus devices, calling the function on each of them until either an error + /// or `Ok(false)` is returned pub fn walk_bus_devices Result>( mut f: F, ) -> Result<(), Error> { @@ -155,6 +186,7 @@ impl PciBusManager { Ok(()) } + /// Enumerates a bus segment provided by ACPI MCFG table entry pub fn add_segment_from_mcfg(entry: &McfgEntry) -> Result<(), Error> { let mut bus_segment = PciBusSegment { segment_number: entry.pci_segment_group as u8, @@ -180,6 +212,7 @@ impl fmt::Display for PciAddress { } impl PciAddress { + /// Constructs a [PciAddress] representing a bus pub const fn for_bus(segment: u8, bus: u8) -> Self { Self { segment, @@ -189,6 +222,7 @@ impl PciAddress { } } + /// Constructs a [PciAddress] representing a device on a given bus pub const fn with_device(self, device: u8) -> Self { Self { device, @@ -197,6 +231,7 @@ impl PciAddress { } } + /// Constructs a [PciAddress] representing a function of a given bus device pub const fn with_function(self, function: u8) -> Self { Self { function, ..self } } diff --git a/src/device/bus/pci/space/ecam.rs b/src/device/bus/pci/space/ecam.rs index 078e2704..6e8baccb 100644 --- a/src/device/bus/pci/space/ecam.rs +++ b/src/device/bus/pci/space/ecam.rs @@ -1,12 +1,11 @@ +//! PCI Express ECAM interface use yggdrasil_abi::error::Error; -use crate::{ - device::bus::pci::{PciConfigSpace, PciDeviceInfo}, - mem::{device::DeviceMemory, ConvertAddress}, -}; +use crate::mem::{device::DeviceMemory, ConvertAddress}; use super::{PciAddress, PciConfigurationSpace}; +/// PCI Express Enhanced Configuration Access Mechanism #[derive(Debug)] #[repr(transparent)] pub struct PciEcam { @@ -35,6 +34,13 @@ impl PciConfigurationSpace for PciEcam { } impl PciEcam { + /// Maps the physical address of a ECAM space for kernel access. + /// + /// # Safety + /// + /// The `phys_addr` must be a valid ECAM address. The address must not alias any other mapped + /// regions. The address must be aligned to a 4KiB boundary and be valid for accesses within a + /// 4KiB-sized range. pub unsafe fn map(phys_addr: usize) -> Result { // TODO check align let mapping = DeviceMemory::map("pcie-ecam", phys_addr, 0x1000)?; @@ -42,14 +48,12 @@ impl PciEcam { Ok(PciEcam { mapping }) } - pub unsafe fn from_raw_parts( - segment_phys_addr: usize, - bus_offset: u8, - address: PciAddress, - ) -> Result { - todo!() - } - + /// Checks if the ECAM contains a valid device configuration space, mapping and returning a + /// [PciEcam] if it does. + /// + /// # Safety + /// + /// See [PciEcam::map]. pub unsafe fn probe_raw_parts( segment_phys_addr: usize, bus_offset: u8, diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs index 33d2ea27..e0a6d43d 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/src/device/bus/pci/space/mod.rs @@ -3,83 +3,157 @@ use super::{PciAddress, PciBaseAddress, PciEcam}; pub(super) mod ecam; macro_rules! pci_config_field_getter { - (u32, $name:ident, $offset:expr) => { - fn $name(&self) -> u32 { - self.read_u32($offset) - } + ($self:ident, u32, $offset:expr) => { + $self.read_u32($offset) }; - (u16, $name:ident, $offset:expr) => { - fn $name(&self) -> u16 { - self.read_u16($offset) - } + ($self:ident, u16, $offset:expr) => { + $self.read_u16($offset) }; - (u8, $name:ident, $offset:expr) => { - fn $name(&self) -> u8 { - self.read_u8($offset) - } + ($self:ident, u8, $offset:expr) => { + $self.read_u8($offset) }; } macro_rules! pci_config_field { - ($offset:expr => $ty:ident, $getter:ident $(, $setter:ident)?) => { - pci_config_field_getter!($ty, $getter, $offset); + ( + $offset:expr => $ty:ident, + $(#[$getter_meta:meta])* $getter:ident + $(, $(#[$setter_meta:meta])* $setter:ident)? + ) => { + $(#[$getter_meta])* + fn $getter(&self) -> $ty { + pci_config_field_getter!(self, $ty, $offset) + } $( - fn $setter(&self, value: $ty) { + $(#[$setter_meta])* + fn $setter(&self, _value: $ty) { todo!() } )? }; } +/// Provides access to the legacy (port I/O-driven) PCI configuration space #[derive(Debug)] #[repr(transparent)] pub struct PciLegacyConfigurationSpace { + #[allow(unused)] address: PciAddress, } +/// Describes a configuration space access method for a PCI device #[derive(Debug)] pub enum PciConfigSpace { + /// Legacy configuration space. + /// + /// See [PciLegacyConfigurationSpace]. Legacy(PciAddress), + + /// Enhanced Configuration Access Mechanism (PCIe). + /// + /// See [PciEcam]. Ecam(PciEcam), } +/// Interface for accessing the configuration space of a device pub trait PciConfigurationSpace { + /// Reads a 32-bit value from the device configuration space. + /// + /// # Note + /// + /// The `offset` must be u32-aligned. fn read_u32(&self, offset: usize) -> u32; + /// Reads a 16-bit value from the device configuration space. + /// + /// # Note + /// + /// The `offset` must be u16-aligned. fn read_u16(&self, offset: usize) -> u16 { assert_eq!(offset & 1, 0); let value = self.read_u32(offset & !3); (value >> ((offset & 3) * 8)) as u16 } + /// Reads a byte from the device configuration space. fn read_u8(&self, offset: usize) -> u8 { let value = self.read_u32(offset & !3); (value >> ((offset & 3) * 8)) as u8 } + /// Returns `true` if the device is present on the bus (i.e. configuration space is not filled + /// with only 1's) fn is_valid(&self) -> bool { self.vendor_id() != 0xFFFF && self.device_id() != 0xFFFF } - pci_config_field!(0x00 => u16, vendor_id); - pci_config_field!(0x02 => u16, device_id); - pci_config_field!(0x04 => u16, command, set_command); - pci_config_field!(0x06 => u16, status); + pci_config_field!( + 0x00 => u16, + #[doc = "Returns the Vendor ID"] vendor_id + ); + pci_config_field!(0x02 => u16, + #[doc = "Returns the Device ID"] device_id + ); + pci_config_field!( + 0x04 => u16, + #[doc = "Returns the value of the command register"] command, + #[doc = "Writes to the command word register"] set_command + ); + pci_config_field!( + 0x06 => u16, + #[doc = "Returns the value of the status register"] status + ); - pci_config_field!(0x08 => u8, rev_id); - pci_config_field!(0x09 => u8, prog_if); - pci_config_field!(0x0A => u8, subclass); - pci_config_field!(0x0B => u8, class_code); + pci_config_field!( + 0x08 => u8, + #[doc = "Returns the device Revision ID"] + rev_id + ); + pci_config_field!( + 0x09 => u8, + #[doc = "Returns the device Prog IF field"] + prog_if + ); + pci_config_field!( + 0x0A => u8, + #[doc = "Returns the device Subclass field"] + subclass + ); + pci_config_field!( + 0x0B => u8, + #[doc = "Returns the device Class Code field"] + class_code + ); // ... - pci_config_field!(0x0E => u8, header_type); - // Header Type 1 only - pci_config_field!(0x19 => u8, secondary_bus); + pci_config_field!( + 0x0E => u8, + #[doc = "Returns the header type of the device"] + header_type + ); + pci_config_field!( + 0x19 => u8, + #[doc = r#" + Returns the secondary bus number associated with this device - // Header Type 0 only + # Note + + The function is only valid for devices with `header_type() == 1` + "#] + secondary_bus + ); + + /// Returns the value of the Base Address Register with given index. + /// + /// # Note + /// + /// The function is only valid for devices with `header_type() == 0` + /// + /// The `index` corresponds to the actual configuration space BAR index, i.e. if a 64-bit + /// address occupies [BAR0, BAR1] and BAR 1 is requested, the function will return [None]. fn bar(&self, index: usize) -> Option { assert!(index < 6); diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 5f8c05cc..7ad312d6 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -513,6 +513,7 @@ pub fn add_console_autoflush(console: &'static dyn DisplayConsole) { CONSOLES.lock().push(console); } +/// Flushes console buffers to their displays pub fn flush_consoles() { for console in CONSOLES.lock().iter() { let mut state = console.state().lock(); diff --git a/src/mem/heap.rs b/src/mem/heap.rs index 56113575..fca4f7ce 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -8,6 +8,7 @@ use core::{ use linked_list_allocator::Heap; use spinning_top::Spinlock; +/// Kernel heap manager pub struct KernelAllocator { inner: Spinlock, } @@ -46,6 +47,7 @@ unsafe impl GlobalAlloc for KernelAllocator { } } +/// Kernel's global allocator #[global_allocator] pub static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); diff --git a/src/panic.rs b/src/panic.rs index 7bfefc2a..f59a9110 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -1,5 +1,5 @@ //! Kernel panic handler code -use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use device_api::interrupt::IpiDeliveryTarget; diff --git a/src/util.rs b/src/util.rs index d03dd27f..ab2bf966 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,9 +1,11 @@ +//! Various kernel utility functions use core::time::Duration; use yggdrasil_abi::error::Error; use crate::arch::{Architecture, ARCHITECTURE}; +/// Performs a busy-loop sleep until the specified duration has passed pub fn polling_sleep(duration: Duration) -> Result<(), Error> { let timer = ARCHITECTURE.monotonic_timer(); let deadline = timer.monotonic_timestamp()? + duration; From cb65a1cff25e70ad02c975bdc1c9cef1ee7f0d75 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 5 Sep 2023 16:17:48 +0300 Subject: [PATCH 067/211] proc: (wip) async/await runtime --- Cargo.toml | 2 + lib/kernel-util/src/util.rs | 18 ++ src/arch/x86_64/peripherals/hpet.rs | 6 +- src/arch/x86_64/table/mod.rs | 3 + src/device/mod.rs | 1 + src/device/timer.rs | 10 + src/device/tty.rs | 281 ++++++++++++---------------- src/main.rs | 9 +- src/proc/exec.rs | 6 +- src/proc/mod.rs | 1 - src/proc/wait.rs | 183 ------------------ src/syscall/mod.rs | 31 +-- src/task/mod.rs | 19 +- src/task/process.rs | 143 +++++++------- src/task/runtime/executor.rs | 63 +++++++ src/task/runtime/macros.rs | 33 ++++ src/task/runtime/mod.rs | 78 ++++++++ src/task/runtime/task.rs | 23 +++ src/task/runtime/task_queue.rs | 55 ++++++ src/task/runtime/waker.rs | 63 +++++++ src/task/sched.rs | 24 ++- src/{util.rs => util/mod.rs} | 19 ++ src/util/queue.rs | 64 +++++++ src/util/ring.rs | 97 ++++++++++ 24 files changed, 762 insertions(+), 470 deletions(-) create mode 100644 src/device/timer.rs delete mode 100644 src/proc/wait.rs create mode 100644 src/task/runtime/executor.rs create mode 100644 src/task/runtime/macros.rs create mode 100644 src/task/runtime/mod.rs create mode 100644 src/task/runtime/task.rs create mode 100644 src/task/runtime/task_queue.rs create mode 100644 src/task/runtime/waker.rs rename src/{util.rs => util/mod.rs} (56%) create mode 100644 src/util/queue.rs create mode 100644 src/util/ring.rs diff --git a/Cargo.toml b/Cargo.toml index b7d0abfd..3221e470 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ bitmap-font = { version = "0.3.0", optional = true } embedded-graphics = { version = "0.8.0", optional = true } log = "0.4.20" +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } +crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } [dependencies.elf] version = "0.7.2" diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util.rs index 9db3ad13..672287ea 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util.rs @@ -70,6 +70,24 @@ impl OneTimeInit { unsafe { (*self.value.get()).assume_init_ref() } } + #[track_caller] + pub fn or_init_with T>(&self, init: F) -> &T { + if self + .state + .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed) + .is_err() + { + // Already initialized + unsafe { (*self.value.get()).assume_init_ref() } + } else { + // Initialize + unsafe { + (*self.value.get()).write((init)()); + (*self.value.get()).assume_init_ref() + } + } + } + /// Returns an immutable reference to the underlying value and [None] if the value hasn't yet /// been initialized pub fn try_get(&self) -> Option<&T> { diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs index 05e76021..3267918d 100644 --- a/src/arch/x86_64/peripherals/hpet.rs +++ b/src/arch/x86_64/peripherals/hpet.rs @@ -16,10 +16,9 @@ use tock_registers::{ use crate::{ arch::{x86_64::IrqNumber, Architecture, ARCHITECTURE}, + device::timer, mem::device::DeviceMemoryIo, - proc::wait, sync::IrqSafeSpinlock, - task::tasklet, }; register_bitfields! { @@ -169,8 +168,7 @@ impl InterruptHandler for Hpet { Duration::from_millis(inner.tim0_counter) }; - wait::tick(now); - tasklet::tick(now); + timer::tick(now); true } diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/table/mod.rs index 1c9327f4..868da138 100644 --- a/src/arch/x86_64/table/mod.rs +++ b/src/arch/x86_64/table/mod.rs @@ -380,3 +380,6 @@ impl AddressSpace { unsafe { (self.l0 as usize).physicalize() } } } + +unsafe impl Send for AddressSpace {} +unsafe impl Sync for AddressSpace {} diff --git a/src/device/mod.rs b/src/device/mod.rs index 2baf9495..51a99ea2 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -11,6 +11,7 @@ pub mod bus; pub mod display; pub mod power; pub mod serial; +pub mod timer; pub mod tty; static DEVICE_MANAGER: IrqSafeSpinlock = IrqSafeSpinlock::new(DeviceManager::new()); diff --git a/src/device/timer.rs b/src/device/timer.rs new file mode 100644 index 00000000..0dba8328 --- /dev/null +++ b/src/device/timer.rs @@ -0,0 +1,10 @@ +use core::time::Duration; + +use crate::task::{runtime, tasklet}; + +pub fn tick(now: Duration) { + runtime::tick(now); + + // TODO tasklets are no longer needed + tasklet::tick(now); +} diff --git a/src/device/tty.rs b/src/device/tty.rs index 45b30bee..3f22c212 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -7,14 +7,15 @@ use abi::{ use device_api::serial::SerialDevice; use crate::{ - proc::wait::Wait, sync::IrqSafeSpinlock, task::{process::Process, ProcessId}, + util::ring::AsyncRing, }; #[cfg(feature = "fb_console")] pub mod combined { //! Combined console + keyboard terminal device + use crate::block; use abi::{error::Error, io::DeviceRequest}; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; use vfs::CharDevice; @@ -24,14 +25,13 @@ pub mod combined { Device, }; - use super::{CharRing, TtyDevice}; + use super::{TtyContext, TtyDevice}; // TODO rewrite this /// Helper device to combine a display and a keyboard input into a single terminal pub struct CombinedTerminal { output: &'static (dyn DisplayConsole + Sync + 'static), - - input_ring: CharRing<16>, + context: TtyContext, } impl CombinedTerminal { @@ -39,14 +39,14 @@ pub mod combined { pub fn new(output: &'static FramebufferConsole) -> Self { Self { output, - input_ring: CharRing::new(), + context: TtyContext::new(), } } } - impl TtyDevice<16> for CombinedTerminal { - fn ring(&self) -> &CharRing<16> { - &self.input_ring + impl TtyDevice for CombinedTerminal { + fn context(&self) -> &TtyContext { + &self.context } } @@ -73,7 +73,10 @@ pub mod combined { impl CharDevice for CombinedTerminal { fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { assert!(blocking); - self.line_read(data) + block! { + self.line_read(data).await + } + // self.line_read(data) } fn write(&self, blocking: bool, data: &[u8]) -> Result { @@ -96,77 +99,74 @@ pub mod combined { #[cfg(feature = "fb_console")] pub use combined::CombinedTerminal; -struct CharRingInner { - rd: usize, - wr: usize, - data: [u8; N], - flags: u8, +struct TtyContextInner { + config: TerminalOptions, process_group: Option, } -/// Ring buffer for a character device. Handles reads, writes and channel notifications for a -/// terminal device. -pub struct CharRing { - wait_read: Wait, - wait_write: Wait, - inner: IrqSafeSpinlock>, - config: IrqSafeSpinlock, +pub struct TtyContext { + ring: AsyncRing, + // input_queue: AsyncQueue, + inner: IrqSafeSpinlock, } /// Terminal device interface -pub trait TtyDevice: SerialDevice { +pub trait TtyDevice: SerialDevice { /// Returns the ring buffer associated with the device - fn ring(&self) -> &CharRing; - - /// Returns `true` if data is ready to be read from or written to the terminal - fn is_ready(&self, write: bool) -> Result { - let ring = self.ring(); - if write { - todo!(); - } else { - Ok(ring.is_readable()) - } - } + fn context(&self) -> &TtyContext; /// Sets the process group to which signals from this terminal should be delivered fn set_signal_group(&self, id: ProcessId) { - self.ring().inner.lock().process_group = Some(id); + self.context().inner.lock().process_group.replace(id); } /// Sends a single byte to the terminal fn line_send(&self, byte: u8) -> Result<(), Error> { - let config = self.ring().config.lock(); + let cx = self.context(); + let inner = cx.inner.lock(); - if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + if byte == b'\n' + && inner + .config + .output + .contains(TerminalOutputOptions::NL_TO_CRNL) + { self.send(b'\r').ok(); } + drop(inner); + self.send(byte) } /// Receives a single byte from the terminal fn recv_byte(&self, mut byte: u8) { - let ring = self.ring(); - let config = ring.config.lock(); + let cx = self.context(); + let inner = cx.inner.lock(); - if byte == b'\r' && config.input.contains(TerminalInputOptions::CR_TO_NL) { + if byte == b'\r' && inner.config.input.contains(TerminalInputOptions::CR_TO_NL) { byte = b'\n'; } if byte == b'\n' { // TODO implement proper echo here - let _echo = config.line.contains(TerminalLineOptions::ECHO) - || config + let _echo = inner.config.line.contains(TerminalLineOptions::ECHO) + || inner + .config .line .contains(TerminalLineOptions::CANONICAL | TerminalLineOptions::ECHO_NL); - if config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + if inner + .config + .output + .contains(TerminalOutputOptions::NL_TO_CRNL) + { self.send(b'\r').ok(); } self.send(byte).ok(); - } else if config.line.contains(TerminalLineOptions::ECHO) { + } else if inner.config.line.contains(TerminalLineOptions::ECHO) { if byte.is_ascii_control() { - if byte != config.chars.erase && byte != config.chars.werase { + if byte != inner.config.chars.erase && byte != inner.config.chars.werase { self.send(b'^').ok(); self.send(byte + 0x40).ok(); } @@ -176,10 +176,13 @@ pub trait TtyDevice: SerialDevice { } // byte == config.chars.interrupt - if byte == config.chars.interrupt && config.line.contains(TerminalLineOptions::SIGNAL) { - drop(config); - let pgrp = ring.inner.lock().process_group; + if byte == inner.config.chars.interrupt + && inner.config.line.contains(TerminalLineOptions::SIGNAL) + { + let pgrp = inner.process_group; + if let Some(pgrp) = pgrp { + drop(inner); Process::signal_group(pgrp, Signal::Interrupted); return; } else { @@ -187,20 +190,22 @@ pub trait TtyDevice: SerialDevice { } } - ring.putc(byte, false).ok(); + drop(inner); + cx.putc(byte); } /// Reads and processes data from the terminal - fn line_read(&'static self, data: &mut [u8]) -> Result { - let ring = self.ring(); - let mut config = ring.config.lock(); + async fn line_read(&self, data: &mut [u8]) -> Result { + let cx = self.context(); + let mut inner = cx.inner.lock(); if data.is_empty() { return Ok(0); } - if !config.is_canonical() { - let byte = ring.getc()?; + if !inner.config.is_canonical() { + drop(inner); + let byte = cx.getc().await; data[0] = byte; Ok(1) } else { @@ -209,14 +214,14 @@ pub trait TtyDevice: SerialDevice { // Run until either end of buffer or return condition is reached while rem != 0 { - drop(config); - let byte = ring.getc()?; - config = ring.config.lock(); + drop(inner); + let byte = cx.getc().await; + inner = cx.inner.lock(); - if config.is_canonical() { - if byte == config.chars.eof { + if inner.config.is_canonical() { + if byte == inner.config.chars.eof { break; - } else if byte == config.chars.erase { + } else if byte == inner.config.chars.erase { // Erase if off != 0 { self.raw_write(b"\x1b[D \x1b[D")?; @@ -225,7 +230,7 @@ pub trait TtyDevice: SerialDevice { } continue; - } else if byte == config.chars.werase { + } else if byte == inner.config.chars.werase { todo!() } } @@ -260,113 +265,69 @@ pub trait TtyDevice: SerialDevice { } } -impl CharRingInner { - #[inline] - const fn is_readable(&self) -> bool { - if self.rd <= self.wr { - (self.wr - self.rd) > 0 - } else { - (self.wr + (N - self.rd)) > 0 - } - } - - #[inline] - unsafe fn read_unchecked(&mut self) -> u8 { - let res = self.data[self.rd]; - self.rd = (self.rd + 1) % N; - res - } - - #[inline] - unsafe fn write_unchecked(&mut self, ch: u8) { - self.data[self.wr] = ch; - self.wr = (self.wr + 1) % N; - } -} - -impl CharRing { - /// Constructs an empty ring buffer - pub const fn new() -> Self { +impl TtyContext { + pub fn new() -> Self { Self { - inner: IrqSafeSpinlock::new(CharRingInner { - rd: 0, - wr: 0, - data: [0; N], - flags: 0, + ring: AsyncRing::new(0), + inner: IrqSafeSpinlock::new(TtyContextInner { + config: TerminalOptions::const_default(), process_group: None, }), - wait_read: Wait::new("char_ring_read"), - wait_write: Wait::new("char_ring_write"), - config: IrqSafeSpinlock::new(TerminalOptions::const_default()), } } - /// Returns `true` if the buffer has data to read - pub fn is_readable(&self) -> bool { - let inner = self.inner.lock(); - let config = self.config.lock(); - - if config.is_canonical() { - let mut rd = inner.rd; - let mut count = 0usize; - - loop { - let readable = if rd <= inner.wr { - (inner.wr - rd) > 0 - } else { - (inner.wr + (N - rd)) > 0 - }; - - if !readable { - break; - } - - let byte = inner.data[rd]; - if byte == b'\n' { - count += 1; - } - - rd = (rd + 1) % N; - } - - count != 0 || inner.flags != 0 - } else { - inner.is_readable() || inner.flags != 0 - } + pub fn putc(&self, ch: u8) { + self.ring.try_write(ch).unwrap(); } - /// Reads a single character from the buffer, blocking until available - pub fn getc(&'static self) -> Result { - let mut lock = self.inner.lock(); - loop { - if !lock.is_readable() && lock.flags == 0 { - drop(lock); - self.wait_read.wait(None)?; - lock = self.inner.lock(); - } else { - break; - } - } - - let byte = unsafe { lock.read_unchecked() }; - drop(lock); - self.wait_write.wakeup_one(); - // TODO WAIT_SELECT - Ok(byte) - } - - /// Sends a single character to the buffer - pub fn putc(&self, ch: u8, blocking: bool) -> Result<(), Error> { - let mut lock = self.inner.lock(); - if blocking { - todo!(); - } - unsafe { - lock.write_unchecked(ch); - } - drop(lock); - self.wait_read.wakeup_one(); - // TODO WAIT_SELECT - Ok(()) + pub async fn getc(&self) -> u8 { + self.ring.read().await } } + +// impl CharRingInner { +// } +// +// impl CharRing { +// /// Constructs an empty ring buffer +// pub const fn new() -> Self { +// Self { +// inner: IrqSafeSpinlock::new(CharRingInner { +// rd: 0, +// wr: 0, +// data: [0; N], +// flags: 0, +// process_group: None, +// }), +// // wait_read: Wait::new("char_ring_read"), +// // wait_write: Wait::new("char_ring_write"), +// config: IrqSafeSpinlock::new(TerminalOptions::const_default()), +// } +// } +// +// /// Reads a single character from the buffer, blocking until available +// pub fn getc(&'static self) -> Result { +// todo!() +// // let mut lock = self.inner.lock(); +// // loop { +// // if !lock.is_readable() && lock.flags == 0 { +// // drop(lock); +// // self.wait_read.wait(None)?; +// // lock = self.inner.lock(); +// // } else { +// // break; +// // } +// // } +// +// // let byte = unsafe { lock.read_unchecked() }; +// // drop(lock); +// // self.wait_write.wakeup_one(); +// // // TODO WAIT_SELECT +// // Ok(byte) +// } +// +// /// Sends a single character to the buffer +// pub fn putc(&self, ch: u8, blocking: bool) -> Result<(), Error> { +// todo!() +// } +// } diff --git a/src/main.rs b/src/main.rs index c6a299f7..80a9191c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,20 +12,21 @@ let_chains, linked_list_cursors, rustc_private, - allocator_api + allocator_api, + async_fn_in_trait )] #![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] -#![warn(missing_docs)] +// #![warn(missing_docs)] +#![allow(missing_docs)] #![no_std] #![no_main] use sync::SpinFence; -use task::spawn_kernel_closure; use crate::{ arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, mem::heap, - task::Cpu, + task::{spawn_kernel_closure, Cpu}, }; extern crate yggdrasil_abi as abi; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 13856795..9212bc0e 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -2,7 +2,7 @@ use core::mem::size_of; use abi::error::Error; -use alloc::{rc::Rc, string::String}; +use alloc::{string::String, sync::Arc}; use vfs::FileRef; use crate::{ @@ -75,7 +75,7 @@ fn setup_binary>( space: AddressSpace, entry: usize, args: &[&str], -) -> Result, Error> { +) -> Result, Error> { const USER_STACK_PAGES: usize = 16; let virt_stack_base = 0x3000000; @@ -123,7 +123,7 @@ pub fn load_elf>( name: S, file: FileRef, args: &[&str], -) -> Result, Error> { +) -> Result, Error> { let space = AddressSpace::new_empty()?; let elf_entry = proc::elf::load_elf_from_file(&space, file)?; diff --git a/src/proc/mod.rs b/src/proc/mod.rs index 5c7e917e..aa280e3b 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -3,4 +3,3 @@ pub mod elf; pub mod exec; pub mod io; -pub mod wait; diff --git a/src/proc/wait.rs b/src/proc/wait.rs deleted file mode 100644 index 28772ce9..00000000 --- a/src/proc/wait.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! Wait channel implementation -use core::time::Duration; - -use abi::error::Error; -use alloc::{collections::LinkedList, rc::Rc}; - -use crate::{ - arch::{Architecture, ARCHITECTURE}, - sync::IrqSafeSpinlock, - task::process::{Process, ProcessState}, -}; - -/// Defines whether the wait channel is available for a specific task -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum WaitStatus { - /// Wait on the channel was interrupted - Interrupted, - /// Channel did not yet signal availability - Pending, - /// Channel has data available - Done, -} - -/// Wait notification channel -pub struct Wait { - queue: IrqSafeSpinlock>>, - // Used for tracing waits - #[allow(dead_code)] - name: &'static str, -} - -struct Timeout { - process: Rc, - #[allow(dead_code)] - deadline: Duration, -} - -/// Common notification channel for tasks waiting on process exits -pub static PROCESS_EXIT_WAIT: Wait = Wait::new("process-exit"); - -impl Wait { - /// Constructs a new wait notification channel - pub const fn new(name: &'static str) -> Self { - Self { - name, - queue: IrqSafeSpinlock::new(LinkedList::new()), - } - } - - /// Wakes up tasks waiting for availability on this channel, but no more than `limit` - pub fn wakeup_some(&self, mut limit: usize) -> usize { - let mut queue = self.queue.lock(); - let mut count = 0; - while limit != 0 && !queue.is_empty() { - let proc = queue.pop_front().unwrap(); - - { - let mut tick_lock = TICK_LIST.lock(); - let mut cursor = tick_lock.cursor_front_mut(); - - while let Some(item) = cursor.current() { - if proc.id() == item.process.id() { - cursor.remove_current(); - break; - } else { - cursor.move_next(); - } - } - - drop(tick_lock); - - unsafe { - proc.set_wait_status(WaitStatus::Done); - } - if proc.state() == ProcessState::Suspended { - proc.enqueue_somewhere(); - } - } - - limit -= 1; - count += 1; - } - - count - } - - /// Wakes up all tasks waiting on this channel - pub fn wakeup_all(&self) { - self.wakeup_some(usize::MAX); - } - - /// Wakes up a single task waiting on this channel - pub fn wakeup_one(&self) { - self.wakeup_some(1); - } - - /// Suspends the task until either the deadline is reached or this channel signals availability - pub fn wait(&'static self, deadline: Option) -> Result<(), Error> { - let process = Process::current(); - let mut queue_lock = self.queue.lock(); - queue_lock.push_back(process.clone()); - unsafe { - process.setup_wait(self); - } - - if let Some(deadline) = deadline { - TICK_LIST.lock().push_back(Timeout { - process: process.clone(), - deadline, - }); - } - - loop { - match process.wait_status() { - WaitStatus::Pending => (), - WaitStatus::Done => return Ok(()), - WaitStatus::Interrupted => return Err(Error::InvalidArgument), - } - - drop(queue_lock); - process.suspend(); - - queue_lock = self.queue.lock(); - - if let Some(deadline) = deadline { - let now = ARCHITECTURE.monotonic_timer().monotonic_timestamp()?; - - if now > deadline { - let mut cursor = queue_lock.cursor_front_mut(); - - while let Some(item) = cursor.current() { - if item.id() == process.id() { - cursor.remove_current(); - return Err(Error::TimedOut); - } else { - cursor.move_next(); - } - } - - // Most likely the process was killed by a signal - } - } - } - } -} - -static TICK_LIST: IrqSafeSpinlock> = IrqSafeSpinlock::new(LinkedList::new()); - -/// Suspends current task until given deadline -pub fn sleep(timeout: Duration, remaining: &mut Duration) -> Result<(), Error> { - static SLEEP_NOTIFY: Wait = Wait::new("sleep"); - let now = ARCHITECTURE.monotonic_timer().monotonic_timestamp()?; - let deadline = now + timeout; - - match SLEEP_NOTIFY.wait(Some(deadline)) { - // Just what we expected - Err(Error::TimedOut) => { - *remaining = Duration::ZERO; - Ok(()) - } - - Ok(_) => panic!("This should not happen"), - Err(e) => Err(e), - } -} - -/// Updates all pending timeouts and wakes up the tasks that have reached theirs -pub fn tick(now: Duration) { - let mut list = TICK_LIST.lock(); - let mut cursor = list.cursor_front_mut(); - - while let Some(item) = cursor.current() { - if now > item.deadline { - let t = cursor.remove_current().unwrap(); - - if t.process.state() == ProcessState::Suspended { - t.process.enqueue_somewhere(); - } - } else { - cursor.move_next(); - } - } -} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 01fc5348..5e72c89d 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -15,16 +15,11 @@ use yggdrasil_abi::{ }; use crate::{ - fs, + block, fs, mem::table::{MapAttributes, VirtualMemoryManager}, - proc::{ - self, - io::ProcessIo, - wait::{self, PROCESS_EXIT_WAIT}, - // wait::{self, PROCESS_EXIT_WAIT}, - }, + proc::{self, io::ProcessIo}, sync::IrqSafeSpinlockGuard, - task::{process::Process, ProcessId}, + task::{process::Process, runtime, ProcessId}, }; mod arg; @@ -66,9 +61,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let seconds = args[0]; let nanos = args[1] as u32; let duration = Duration::new(seconds, nanos); - let mut remaining = Duration::ZERO; - wait::sleep(duration, &mut remaining).unwrap(); + block! { + runtime::sleep(duration).await + }; Ok(0) } @@ -321,18 +317,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let target = Process::get(pid).ok_or(Error::DoesNotExist)?; - loop { - if let Some(exit_status) = target.get_exit_status() { - *status = exit_status; - break Ok(0); - } + *status = block! { + Process::wait_for_exit(target).await + }; - // Suspend and wait for signal - match PROCESS_EXIT_WAIT.wait(Some(Duration::from_secs(3))) { - Ok(()) | Err(Error::TimedOut) => (), - Err(_) => todo!(), - } - } + Ok(0) } SyscallFunction::SendSignal => { let pid = args[0] as u32; diff --git a/src/task/mod.rs b/src/task/mod.rs index 51c44f02..16fa231f 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -3,7 +3,7 @@ #![allow(dead_code)] use abi::error::Error; -use alloc::{rc::Rc, string::String, vec::Vec}; +use alloc::{string::String, sync::Arc, vec::Vec}; use crate::{ arch::{Architecture, ArchitectureImpl}, @@ -15,6 +15,7 @@ use self::{context::TaskContextImpl, process::Process}; pub mod context; pub mod process; +pub mod runtime; pub mod sched; pub mod tasklet; @@ -25,7 +26,7 @@ pub type ProcessId = usize; /// Wrapper structure to hold all the system's processes pub struct ProcessList { - data: Vec<(ProcessId, Rc)>, + data: Vec<(ProcessId, Arc)>, last_process_id: ProcessId, } @@ -44,7 +45,7 @@ impl ProcessList { /// /// Only meant to be called from inside the Process impl, as this function does not perform any /// accounting information updates. - pub unsafe fn push(&mut self, process: Rc) -> ProcessId { + pub unsafe fn push(&mut self, process: Arc) -> ProcessId { self.last_process_id += 1; debugln!("Insert process with ID {}", self.last_process_id); self.data.push((self.last_process_id, process)); @@ -52,7 +53,7 @@ impl ProcessList { } /// Looks up a process by its ID - pub fn get(&self, id: ProcessId) -> Option<&Rc> { + pub fn get(&self, id: ProcessId) -> Option<&Arc> { self.data .iter() .find_map(|(i, p)| if *i == id { Some(p) } else { None }) @@ -80,12 +81,10 @@ pub fn init() -> Result<(), Error> { // Create a queue for each CPU sched::init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); - // spawn_kernel_closure(move || loop { - // debugln!("B"); - // for _ in 0..100000 { - // core::hint::spin_loop(); - // } - // })?; + // Spawn async workers + (0..cpu_count).for_each(|index| { + runtime::spawn_async_worker(index).unwrap(); + }); Ok(()) } diff --git a/src/task/process.rs b/src/task/process.rs index b392e981..8079b354 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -2,30 +2,33 @@ use core::{ mem::size_of, ops::Deref, + pin::Pin, sync::atomic::{AtomicU32, Ordering}, + task::{Context, Poll}, }; use abi::{ error::Error, process::{ExitCode, Signal, SignalEntryData}, }; -use alloc::{collections::VecDeque, rc::Rc, string::String}; +use alloc::{collections::VecDeque, string::String, sync::Arc}; use atomic_enum::atomic_enum; +use futures_util::{task::ArcWake, Future}; use kernel_util::util::OneTimeInit; use vfs::VnodeRef; use crate::{ arch::{Architecture, ArchitectureImpl}, mem::{table::AddressSpace, ForeignPointer}, - proc::{ - io::ProcessIo, - wait::{Wait, WaitStatus, PROCESS_EXIT_WAIT}, - }, + proc::io::ProcessIo, sync::{IrqGuard, IrqSafeSpinlock}, task::context::TaskContextImpl, }; -use super::{context::TaskFrame, sched::CpuQueue, Cpu, ProcessId, TaskContext, PROCESSES}; +use super::{ + context::TaskFrame, runtime::QueueWaker, sched::CpuQueue, Cpu, ProcessId, TaskContext, + PROCESSES, +}; /// Represents the states a process can be at some point in time #[atomic_enum] @@ -49,9 +52,6 @@ pub struct SignalEntry { } struct ProcessInner { - // XXX - pending_wait: Option<&'static Wait>, - wait_status: WaitStatus, exit_status: i32, session_id: ProcessId, @@ -74,12 +74,15 @@ pub struct Process { cpu_id: AtomicU32, space: Option, inner: IrqSafeSpinlock, + + exit_waker: QueueWaker, + /// I/O state of the task pub io: IrqSafeSpinlock, } /// Guard type that provides [Process] operations only available for current processes -pub struct CurrentProcess(Rc); +pub struct CurrentProcess(Arc); impl Process { /// Creates a process from raw architecture-specific [TaskContext]. @@ -91,8 +94,8 @@ impl Process { name: S, space: Option, normal_context: TaskContext, - ) -> Rc { - let this = Rc::new(Self { + ) -> Arc { + let this = Arc::new(Self { normal_context, id: OneTimeInit::new(), @@ -101,10 +104,9 @@ impl Process { cpu_id: AtomicU32::new(0), space, + exit_waker: QueueWaker::new(), + inner: IrqSafeSpinlock::new(ProcessInner { - // XXX - pending_wait: None, - wait_status: WaitStatus::Done, exit_status: 0, session_id: 0, @@ -219,7 +221,7 @@ impl Process { /// # Panics /// /// Currently, the code will panic if the process is queued/executing on any queue. - pub fn enqueue_somewhere(self: Rc) -> usize { + pub fn enqueue_somewhere(self: Arc) -> usize { // Doesn't have to be precise, so even if something changes, we can still be rebalanced // to another CPU let (index, queue) = CpuQueue::least_loaded().unwrap(); @@ -234,13 +236,16 @@ impl Process { /// # Panics /// /// Currently, the code will panic if the process is queued/executing on any queue. - pub fn enqueue_to(self: Rc, queue: &'static CpuQueue) { + pub fn enqueue_to(self: Arc, queue: &'static CpuQueue) { let _irq = IrqGuard::acquire(); { let mut inner = self.inner.lock(); let old_queue = inner.queue.replace(queue); - assert!(old_queue.is_none()); + if old_queue.is_some() { + // Already in some queue + return; + } } let current_state = self.state.swap(ProcessState::Ready, Ordering::SeqCst); @@ -299,21 +304,6 @@ impl Process { self.dequeue(ProcessState::Suspended); } - /// Returns current wait status of the task - pub fn wait_status(&self) -> WaitStatus { - self.inner.lock().wait_status - } - - /// Updates the wait status for the task. - /// - /// # Safety - /// - /// This function is only meant to be called on waiting tasks, otherwise atomicity is not - /// guaranteed. - pub unsafe fn set_wait_status(&self, status: WaitStatus) { - self.inner.lock().wait_status = status; - } - /// Returns an exit code if the process exited, [None] if it didn't pub fn get_exit_status(&self) -> Option { if self.state() == ProcessState::Terminated { @@ -330,7 +320,7 @@ impl Process { } /// Returns a process by its ID - pub fn get(pid: ProcessId) -> Option> { + pub fn get(pid: ProcessId) -> Option> { PROCESSES.lock().get(pid).cloned() } @@ -361,24 +351,25 @@ impl Process { } // Notify any waiters we're done - PROCESS_EXIT_WAIT.wakeup_all(); + self.exit_waker.wake_all(); } /// Raises a signal for the currentprocess - pub fn raise_signal(self: &Rc, signal: Signal) { - { - let mut inner = self.inner.lock(); - inner.wait_status = WaitStatus::Interrupted; - inner.signal_stack.push_back(signal); - } + pub fn raise_signal(self: &Arc, _signal: Signal) { + // XXX handle signals + async + todo!(); + // { + // let mut inner = self.inner.lock(); + // inner.signal_stack.push_back(signal); + // } - if self.state() == ProcessState::Suspended { - self.clone().enqueue_somewhere(); - } + // if self.state() == ProcessState::Suspended { + // self.clone().enqueue_somewhere(); + // } } /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. - pub fn inherit(&self, parent: &Rc) -> Result<(), Error> { + pub fn inherit(&self, parent: &Arc) -> Result<(), Error> { let mut our_inner = self.inner.lock(); let their_inner = parent.inner.lock(); @@ -401,6 +392,37 @@ impl Process { } } } + + pub fn wait_for_exit(process: Arc) -> impl Future { + struct ProcessExitFuture { + process: Arc, + } + + impl Future for ProcessExitFuture { + type Output = ExitCode; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let process = &self.process; + + process.exit_waker.register(cx.waker()); + + if let Some(exit_status) = process.get_exit_status() { + process.exit_waker.remove(cx.waker()); + Poll::Ready(exit_status) + } else { + Poll::Pending + } + } + } + + ProcessExitFuture { process } + } +} + +impl ArcWake for Process { + fn wake_by_ref(arc_self: &Arc) { + arc_self.clone().enqueue_somewhere(); + } } impl Drop for Process { @@ -415,25 +437,13 @@ impl CurrentProcess { /// # Safety /// /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. - pub unsafe fn new(inner: Rc) -> Self { + pub unsafe fn new(inner: Arc) -> Self { // XXX // assert_eq!(DAIF.read(DAIF::I), 1); assert!(ArchitectureImpl::interrupt_mask()); Self(inner) } - /// Sets up a pending wait for the process. - /// - /// # Safety - /// - /// This function is only meant to be called in no-IRQ context and when caller can guarantee - /// the task won't get scheduled to a CPU in such state. - pub unsafe fn setup_wait(&self, wait: &'static Wait) { - let mut inner = self.inner.lock(); - inner.pending_wait.replace(wait); - inner.wait_status = WaitStatus::Pending; - } - /// Configures signal entry information for the process pub fn set_signal_entry(&self, entry: usize, stack: usize) { let mut inner = self.inner.lock(); @@ -447,21 +457,6 @@ impl CurrentProcess { self.handle_exit(); self.dequeue(ProcessState::Terminated); - - // let current_state = self.state.swap(ProcessState::Terminated, Ordering::SeqCst); - // assert_eq!(current_state, ProcessState::Running); - - // match current_state { - // ProcessState::Suspended => { - // todo!(); - // } - // ProcessState::Ready => todo!(), - // ProcessState::Running => { - // self.handle_exit(); - // unsafe { Cpu::local().queue().yield_cpu() } - // } - // ProcessState::Terminated => todo!(), - // } } /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. @@ -519,7 +514,7 @@ impl CurrentProcess { } impl Deref for CurrentProcess { - type Target = Rc; + type Target = Arc; fn deref(&self) -> &Self::Target { &self.0 diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs new file mode 100644 index 00000000..84681e1d --- /dev/null +++ b/src/task/runtime/executor.rs @@ -0,0 +1,63 @@ +use core::task::{Context, Poll}; + +use abi::error::Error; +use alloc::{boxed::Box, format, sync::Arc}; +use futures_util::{task::waker_ref, Future}; + +use crate::task::{process::Process, spawn_kernel_closure}; + +use super::{ + task::Task, + task_queue::{TaskQueue, TASK_QUEUE}, +}; + +fn init_async_queue() -> TaskQueue { + TaskQueue::new(128) +} + +pub fn enqueue(task: Arc) -> Result<(), Error> { + TASK_QUEUE.or_init_with(init_async_queue).enqueue(task) +} + +pub fn spawn_async_worker(index: usize) -> Result<(), Error> { + let name = format!("[async-worker-{}]", index); + + spawn_kernel_closure(name, move || { + let queue = TASK_QUEUE.or_init_with(init_async_queue); + + loop { + let task = queue.block_pop().unwrap(); + let mut future_slot = task.future.lock(); + + if let Some(mut future) = future_slot.take() { + let waker = waker_ref(&task); + let context = &mut Context::from_waker(&waker); + + if future.as_mut().poll(context).is_pending() { + *future_slot = Some(future); + } + } + } + }) +} + +pub fn spawn + Send + 'static>(future: F) -> Result<(), Error> { + enqueue(Task::new(future)) +} + +pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> T { + let process = Process::current(); + let mut future = Box::pin(future); + + loop { + let waker = waker_ref(&process); + let context = &mut Context::from_waker(&waker); + + match future.as_mut().poll(context) { + Poll::Ready(value) => break value, + Poll::Pending => { + process.suspend(); + } + } + } +} diff --git a/src/task/runtime/macros.rs b/src/task/runtime/macros.rs new file mode 100644 index 00000000..18aa331e --- /dev/null +++ b/src/task/runtime/macros.rs @@ -0,0 +1,33 @@ +#[macro_export] +macro_rules! block { + ($($stmt:tt)*) => { + $crate::task::runtime::run_to_completion(alloc::boxed::Box::pin(async move { + $($stmt)* + })) + }; +} + +#[macro_export] +macro_rules! any { + ($fut0:ident = $pat0:pat => $body0:expr, $fut1:ident = $pat1:pat => $body1:expr) => { + use futures_util::Future; + + let mut pin0 = alloc::boxed::Box::pin($fut0); + let mut pin1 = alloc::boxed::Box::pin($fut1); + + $crate::task::runtime::poll_fn(move |cx| { + match (pin0.as_mut().poll(cx), pin1.as_mut().poll(cx)) { + (core::task::Poll::Ready($pat0), _) => { + $body0; + core::task::Poll::Ready(()) + } + (_, core::task::Poll::Ready($pat1)) => { + $body1; + core::task::Poll::Ready(()) + } + (_, _) => core::task::Poll::Pending, + } + }) + .await; + }; +} diff --git a/src/task/runtime/mod.rs b/src/task/runtime/mod.rs new file mode 100644 index 00000000..e0494139 --- /dev/null +++ b/src/task/runtime/mod.rs @@ -0,0 +1,78 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use futures_util::Future; + +use crate::arch::{Architecture, ARCHITECTURE}; + +mod executor; +mod macros; +mod task; +mod task_queue; +mod waker; + +pub use executor::{run_to_completion, spawn, spawn_async_worker}; +pub use waker::QueueWaker; + +pub struct PollFn { + f: F, +} + +impl Future for PollFn +where + F: FnMut(&mut Context) -> Poll, +{ + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + (unsafe { &mut self.get_unchecked_mut().f })(cx) + } +} + +pub fn poll_fn(f: F) -> PollFn +where + F: FnMut(&mut Context) -> Poll, +{ + PollFn { f } +} + +pub static SLEEP_WAKER: QueueWaker = QueueWaker::new(); + +pub fn sleep(duration: Duration) -> impl Future { + struct SleepFuture { + deadline: Duration, + } + + impl Future for SleepFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + SLEEP_WAKER.register(cx.waker()); + let now = ARCHITECTURE + .monotonic_timer() + .monotonic_timestamp() + .unwrap(); + if now >= self.deadline { + SLEEP_WAKER.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + } + + let now = ARCHITECTURE + .monotonic_timer() + .monotonic_timestamp() + .unwrap(); + let deadline = now + duration; + + SleepFuture { deadline } +} + +pub fn tick(_now: Duration) { + SLEEP_WAKER.wake_all(); +} diff --git a/src/task/runtime/task.rs b/src/task/runtime/task.rs new file mode 100644 index 00000000..c4b5b253 --- /dev/null +++ b/src/task/runtime/task.rs @@ -0,0 +1,23 @@ +use alloc::sync::Arc; +use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt}; + +use crate::sync::IrqSafeSpinlock; + +use super::executor; + +pub struct Task { + pub(super) future: IrqSafeSpinlock>>, +} + +impl ArcWake for Task { + fn wake_by_ref(arc_self: &Arc) { + executor::enqueue(arc_self.clone()).unwrap(); + } +} + +impl Task { + pub fn new + Send + 'static>(future: F) -> Arc { + let future = IrqSafeSpinlock::new(Some(future.boxed())); + Arc::new(Self { future }) + } +} diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs new file mode 100644 index 00000000..5a237587 --- /dev/null +++ b/src/task/runtime/task_queue.rs @@ -0,0 +1,55 @@ +use abi::error::Error; +use alloc::sync::Arc; +use crossbeam_queue::ArrayQueue; +use kernel_util::util::OneTimeInit; + +use crate::{sync::IrqGuard, task::process::Process}; + +use super::task::Task; + +pub(super) static TASK_QUEUE: OneTimeInit = OneTimeInit::new(); + +pub(super) struct TaskQueue { + // Queue of workers waiting for an item + pending_workers: ArrayQueue>, + task_queue: ArrayQueue>, +} + +impl TaskQueue { + pub fn new(task_capacity: usize) -> Self { + Self { + pending_workers: ArrayQueue::new(16), + task_queue: ArrayQueue::new(task_capacity), + } + } + + fn wakeup_one(&self) { + if let Some(worker) = self.pending_workers.pop() { + worker.enqueue_somewhere(); + } + } + + pub fn enqueue(&self, task: Arc) -> Result<(), Error> { + let _irq = IrqGuard::acquire(); + if self.task_queue.push(task).is_err() { + todo!(); + } + self.wakeup_one(); + Ok(()) + } + + pub fn block_pop(&self) -> Result, Error> { + let _irq = IrqGuard::acquire(); + let process = Process::current(); + loop { + if let Some(task) = self.task_queue.pop() { + return Ok(task); + } + + if self.pending_workers.push(process.clone()).is_err() { + panic!("Pending worker queue overflow"); + } + process.suspend(); + } + } +} diff --git a/src/task/runtime/waker.rs b/src/task/runtime/waker.rs new file mode 100644 index 00000000..b6368b19 --- /dev/null +++ b/src/task/runtime/waker.rs @@ -0,0 +1,63 @@ +use core::task::Waker; + +use alloc::collections::VecDeque; + +use crate::sync::IrqSafeSpinlock; + +pub struct QueueWaker { + queue: IrqSafeSpinlock>, +} + +impl QueueWaker { + pub const fn new() -> Self { + Self { + queue: IrqSafeSpinlock::new(VecDeque::new()), + } + } + + pub fn register(&self, waker: &Waker) { + let mut queue = self.queue.lock(); + + if queue.iter().find(|other| other.will_wake(waker)).is_some() { + return; + } + + queue.push_back(waker.clone()); + } + + pub fn remove(&self, waker: &Waker) -> bool { + let mut queue = self.queue.lock(); + let mut index = 0; + let mut removed = false; + + while index < queue.len() { + if queue[index].will_wake(waker) { + removed = true; + queue.remove(index); + } + index += 1; + } + + removed + } + + pub fn wake_some(&self, limit: usize) -> usize { + let mut queue = self.queue.lock(); + let mut count = 0; + + while count < limit && let Some(item) = queue.pop_front() { + item.wake(); + count += 1; + } + + count + } + + pub fn wake_one(&self) -> bool { + self.wake_some(1) != 0 + } + + pub fn wake_all(&self) -> usize { + self.wake_some(usize::MAX) + } +} diff --git a/src/task/sched.rs b/src/task/sched.rs index b7df1b6d..aac2ea4e 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -1,7 +1,7 @@ //! Per-CPU queue implementation // use aarch64_cpu::registers::CNTPCT_EL0; -use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; +use alloc::{collections::VecDeque, sync::Arc, vec::Vec}; use cfg_if::cfg_if; use kernel_util::util::OneTimeInit; @@ -32,9 +32,9 @@ pub struct CpuQueueStats { /// Per-CPU queue's inner data, normally resides under a lock pub struct CpuQueueInner { /// Current process, None if idling - pub current: Option>, + pub current: Option>, /// LIFO queue for processes waiting for execution - pub queue: VecDeque>, + pub queue: VecDeque>, /// CPU time usage statistics pub stats: CpuQueueStats, @@ -80,7 +80,7 @@ impl CpuQueueInner { /// Picks a next task for execution, skipping (dropping) those that were suspended. May return /// None if the queue is empty or no valid task was found, in which case the scheduler should /// go idle. - pub fn next_ready_task(&mut self) -> Option> { + pub fn next_ready_task(&mut self) -> Option> { while !self.queue.is_empty() { let task = self.queue.pop_front().unwrap(); @@ -103,7 +103,7 @@ impl CpuQueueInner { /// Returns an iterator over all the processes in the queue plus the currently running process, /// if there is one. - pub fn iter(&self) -> impl Iterator> { + pub fn iter(&self) -> impl Iterator> { Iterator::chain(self.queue.iter(), self.current.iter()) } } @@ -134,7 +134,7 @@ impl CpuQueue { /// Only meant to be called from [crate::task::enter()] function. pub unsafe fn enter(&self) -> ! { assert!(ArchitectureImpl::interrupt_mask()); - // Start from idle thread to avoid having a Rc stuck here without getting dropped + // Start from idle thread to avoid having a Arc stuck here without getting dropped // let t = CNTPCT_EL0.get(); // self.lock().stats.measure_time = t; self.idle.enter() @@ -173,18 +173,18 @@ impl CpuQueue { inner.current = next.clone(); - // Can drop the lock, we hold current and next Rc's + // Can drop the lock, we hold current and next Arc's drop(inner); let (from, _from_rc) = if let Some(current) = current.as_ref() { - (current.current_context(), Rc::strong_count(current)) + (current.current_context(), Arc::strong_count(current)) } else { (&self.idle, 0) }; let (to, _to_rc) = if let Some(next) = next.as_ref() { next.set_running(Cpu::local_id()); - (next.current_context(), Rc::strong_count(next)) + (next.current_context(), Arc::strong_count(next)) } else { (&self.idle, 0) }; @@ -216,7 +216,7 @@ impl CpuQueue { /// /// Only meant to be called from Process impl. The function does not set any process accounting /// information, which may lead to invalid states. - pub unsafe fn enqueue(&self, p: Rc) { + pub unsafe fn enqueue(&self, p: Arc) { let mut inner = self.inner.lock(); assert!(ArchitectureImpl::interrupt_mask()); assert_eq!(p.state(), ProcessState::Ready); @@ -300,6 +300,10 @@ impl CpuQueue { queues.iter().enumerate().min_by_key(|(_, q)| q.len()) } + + pub fn index(&self) -> usize { + self.index + } } /// Initializes the global queue list diff --git a/src/util.rs b/src/util/mod.rs similarity index 56% rename from src/util.rs rename to src/util/mod.rs index ab2bf966..d69d94c1 100644 --- a/src/util.rs +++ b/src/util/mod.rs @@ -5,6 +5,25 @@ use yggdrasil_abi::error::Error; use crate::arch::{Architecture, ARCHITECTURE}; +pub mod queue; +pub mod ring; + +pub trait ResultIterator { + fn collect_error(self) -> Option; +} + +impl>> ResultIterator for I { + fn collect_error(self) -> Option { + for item in self { + match item { + Err(e) => return Some(e), + _ => (), + } + } + None + } +} + /// Performs a busy-loop sleep until the specified duration has passed pub fn polling_sleep(duration: Duration) -> Result<(), Error> { let timer = ARCHITECTURE.monotonic_timer(); diff --git a/src/util/queue.rs b/src/util/queue.rs new file mode 100644 index 00000000..09b0d55d --- /dev/null +++ b/src/util/queue.rs @@ -0,0 +1,64 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, +}; + +use alloc::sync::Arc; +use crossbeam_queue::ArrayQueue; +use futures_util::Future; + +use crate::task::runtime::QueueWaker; + +pub struct AsyncQueue { + waker: Arc, + queue: Arc>, +} + +impl AsyncQueue { + pub fn new(capacity: usize) -> Self { + Self { + waker: Arc::new(QueueWaker::new()), + queue: Arc::new(ArrayQueue::new(capacity)), + } + } + + pub fn send(&self, value: T) -> Result<(), T> { + let result = self.queue.push(value); + if result.is_ok() { + self.waker.wake_one(); + } + result + } + + pub fn recv(&self) -> impl Future { + struct AsyncQueueRecvFuture { + waker: Arc, + queue: Arc>, + } + + impl Future for AsyncQueueRecvFuture { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if let Some(value) = self.queue.pop() { + return Poll::Ready(value); + } + + self.waker.register(cx.waker()); + + match self.queue.pop() { + Some(value) => { + self.waker.remove(cx.waker()); + Poll::Ready(value) + } + None => Poll::Pending, + } + } + } + + AsyncQueueRecvFuture { + queue: self.queue.clone(), + waker: self.waker.clone(), + } + } +} diff --git a/src/util/ring.rs b/src/util/ring.rs new file mode 100644 index 00000000..9f903b04 --- /dev/null +++ b/src/util/ring.rs @@ -0,0 +1,97 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, +}; + +use abi::error::Error; +use alloc::sync::Arc; +use futures_util::Future; + +use crate::{sync::IrqSafeSpinlock, task::runtime::QueueWaker}; + +struct Inner { + rd: usize, + wr: usize, + data: [T; N], +} + +pub struct AsyncRing { + inner: Arc>>, + read_waker: Arc, +} + +impl Inner { + #[inline] + const fn is_readable(&self) -> bool { + if self.rd <= self.wr { + (self.wr - self.rd) > 0 + } else { + (self.wr + (N - self.rd)) > 0 + } + } + + #[inline] + unsafe fn read_unchecked(&mut self) -> T { + let res = self.data[self.rd]; + self.rd = (self.rd + 1) % N; + res + } + + #[inline] + unsafe fn write_unchecked(&mut self, ch: T) { + self.data[self.wr] = ch; + self.wr = (self.wr + 1) % N; + } +} + +impl AsyncRing { + pub fn new(value: T) -> Self { + Self { + inner: Arc::new(IrqSafeSpinlock::new(Inner { + rd: 0, + wr: 0, + data: [value; N], + })), + read_waker: Arc::new(QueueWaker::new()), + } + } + + pub fn try_write(&self, item: T) -> Result<(), Error> { + let mut lock = self.inner.lock(); + unsafe { + lock.write_unchecked(item); + } + drop(lock); + self.read_waker.wake_one(); + + Ok(()) + } + + pub fn read(&self) -> impl Future { + struct ReadFuture { + inner: Arc>>, + read_waker: Arc, + } + + impl Future for ReadFuture { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.read_waker.register(cx.waker()); + + let mut inner = self.inner.lock(); + if inner.is_readable() { + self.read_waker.remove(cx.waker()); + Poll::Ready(unsafe { inner.read_unchecked() }) + } else { + Poll::Pending + } + } + } + + ReadFuture { + inner: self.inner.clone(), + read_waker: self.read_waker.clone(), + } + } +} From 8ff58a48d2b4342ea02f640e4236f5bb636ba109 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 6 Sep 2023 10:57:02 +0300 Subject: [PATCH 068/211] rt: fix OneTimeInit init race --- lib/kernel-util/src/util.rs | 54 +++++++++++++++-------------- src/arch/x86_64/boot/mod.rs | 4 +++ src/arch/x86_64/mod.rs | 5 ++- src/device/display/console.rs | 13 ++++--- src/device/timer.rs | 5 +-- src/init.rs | 14 +++----- src/task/mod.rs | 1 - src/task/runtime/executor.rs | 12 +++---- src/task/runtime/mod.rs | 1 + src/task/runtime/task_queue.rs | 17 +++++++-- src/task/tasklet.rs | 63 ---------------------------------- 11 files changed, 71 insertions(+), 118 deletions(-) delete mode 100644 src/task/tasklet.rs diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util.rs index 672287ea..5e11e00a 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util.rs @@ -4,7 +4,7 @@ use core::{ mem::MaybeUninit, ops::{Deref, DerefMut}, panic, - sync::atomic::{AtomicBool, Ordering}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; /// Statically-allocated "dynamic" vector @@ -17,32 +17,43 @@ pub struct StaticVector { #[repr(C)] pub struct OneTimeInit { value: UnsafeCell>, - state: AtomicBool, + state: AtomicUsize, } unsafe impl Sync for OneTimeInit {} unsafe impl Send for OneTimeInit {} impl OneTimeInit { + const STATE_UNINITIALIZED: usize = 0; + const STATE_INITIALIZING: usize = 1; + const STATE_INITIALIZED: usize = 2; + /// Wraps the value in an [OneTimeInit] pub const fn new() -> Self { Self { value: UnsafeCell::new(MaybeUninit::uninit()), - state: AtomicBool::new(false), + state: AtomicUsize::new(Self::STATE_UNINITIALIZED), } } /// Returns `true` if the value has already been initialized + #[inline] pub fn is_initialized(&self) -> bool { - self.state.load(Ordering::Acquire) + self.state.load(Ordering::Acquire) == Self::STATE_INITIALIZED } /// Sets the underlying value of the [OneTimeInit]. If already initialized, panics. #[track_caller] pub fn init(&self, value: T) { + // Transition to "initializing" state if self .state - .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed) + .compare_exchange( + Self::STATE_UNINITIALIZED, + Self::STATE_INITIALIZING, + Ordering::Release, + Ordering::Relaxed, + ) .is_err() { panic!( @@ -54,13 +65,24 @@ impl OneTimeInit { unsafe { (*self.value.get()).write(value); } + + // Transition to "initialized" state. This must not fail + self.state + .compare_exchange( + Self::STATE_INITIALIZING, + Self::STATE_INITIALIZED, + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); } /// Returns an immutable reference to the underlying value and panics if it hasn't yet been /// initialized #[track_caller] pub fn get(&self) -> &T { - if !self.state.load(Ordering::Acquire) { + // TODO check for INITIALIZING state and wait until it becomes INITIALIZED? + if !self.is_initialized() { panic!( "{:?}: Attempt to dereference an uninitialized value", panic::Location::caller() @@ -70,28 +92,10 @@ impl OneTimeInit { unsafe { (*self.value.get()).assume_init_ref() } } - #[track_caller] - pub fn or_init_with T>(&self, init: F) -> &T { - if self - .state - .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed) - .is_err() - { - // Already initialized - unsafe { (*self.value.get()).assume_init_ref() } - } else { - // Initialize - unsafe { - (*self.value.get()).write((init)()); - (*self.value.get()).assume_init_ref() - } - } - } - /// Returns an immutable reference to the underlying value and [None] if the value hasn't yet /// been initialized pub fn try_get(&self) -> Option<&T> { - if self.state.load(Ordering::Acquire) { + if self.is_initialized() { Some(self.get()) } else { None diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 0da96293..9be041ff 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -19,6 +19,7 @@ use crate::{ phys::{self, PageUsage}, ConvertAddress, KERNEL_VIRT_OFFSET, }, + task::runtime, }; use super::smp::CPU_COUNT; @@ -91,6 +92,9 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { exception::init_exceptions(0); + // Initialize async executor queue + runtime::init_task_queue(); + devfs::init(); // Initializes: local CPU, platform devices (timers/serials/etc), debug output diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 312f52cf..4cb59e63 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -198,7 +198,10 @@ impl Architecture for X86_64 { // CPU management unsafe fn reset(&self) -> ! { - todo!() + Self::set_interrupt_mask(true); + loop { + Self::wait_for_interrupt(); + } } unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 7ad312d6..01da5ad4 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -1,5 +1,5 @@ //! Console device interfaces -use core::mem::size_of; +use core::{mem::size_of, time::Duration}; use abi::{error::Error, primitive_enum}; use alloc::vec::Vec; @@ -13,7 +13,7 @@ use crate::{ ConvertAddress, }, sync::IrqSafeSpinlock, - task::tasklet::TaskFlow, + task::runtime, }; const CONSOLE_ROW_LEN: usize = 80; @@ -522,7 +522,10 @@ pub fn flush_consoles() { } /// Periodically flushes data from console buffers onto their displays -pub fn task_update_consoles() -> TaskFlow { - flush_consoles(); - TaskFlow::Continue +pub async fn update_consoles_task() { + loop { + flush_consoles(); + + runtime::sleep(Duration::from_millis(20)).await; + } } diff --git a/src/device/timer.rs b/src/device/timer.rs index 0dba8328..bbf89fab 100644 --- a/src/device/timer.rs +++ b/src/device/timer.rs @@ -1,10 +1,7 @@ use core::time::Duration; -use crate::task::{runtime, tasklet}; +use crate::task::runtime; pub fn tick(now: Duration) { runtime::tick(now); - - // TODO tasklets are no longer needed - tasklet::tick(now); } diff --git a/src/init.rs b/src/init.rs index 12b21c95..5cb9e73b 100644 --- a/src/init.rs +++ b/src/init.rs @@ -28,16 +28,12 @@ pub fn kinit() { #[cfg(feature = "fb_console")] { - use core::time::Duration; + use crate::{device::display::console::update_consoles_task, task::runtime}; - use crate::device::display::console::task_update_consoles; - use crate::task::tasklet; - - tasklet::add_periodic( - "update-console", - Duration::from_millis(15), - task_update_consoles, - ); + runtime::spawn(async move { + update_consoles_task().await; + }) + .expect("Could not start periodic console auto-flush task"); } let root = match setup_root() { diff --git a/src/task/mod.rs b/src/task/mod.rs index 16fa231f..7f1ac299 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -17,7 +17,6 @@ pub mod context; pub mod process; pub mod runtime; pub mod sched; -pub mod tasklet; pub use context::{Cpu, TaskContext}; diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs index 84681e1d..d55ddb1d 100644 --- a/src/task/runtime/executor.rs +++ b/src/task/runtime/executor.rs @@ -8,25 +8,21 @@ use crate::task::{process::Process, spawn_kernel_closure}; use super::{ task::Task, - task_queue::{TaskQueue, TASK_QUEUE}, + task_queue::{self}, }; -fn init_async_queue() -> TaskQueue { - TaskQueue::new(128) -} - pub fn enqueue(task: Arc) -> Result<(), Error> { - TASK_QUEUE.or_init_with(init_async_queue).enqueue(task) + task_queue::push_task(task) } pub fn spawn_async_worker(index: usize) -> Result<(), Error> { let name = format!("[async-worker-{}]", index); spawn_kernel_closure(name, move || { - let queue = TASK_QUEUE.or_init_with(init_async_queue); + // let queue = TASK_QUEUE.or_init_with(task_queue::init_async_queue); loop { - let task = queue.block_pop().unwrap(); + let task = task_queue::pop_task().unwrap(); // queue.block_pop().unwrap(); let mut future_slot = task.future.lock(); if let Some(mut future) = future_slot.take() { diff --git a/src/task/runtime/mod.rs b/src/task/runtime/mod.rs index e0494139..93fccd9e 100644 --- a/src/task/runtime/mod.rs +++ b/src/task/runtime/mod.rs @@ -15,6 +15,7 @@ mod task_queue; mod waker; pub use executor::{run_to_completion, spawn, spawn_async_worker}; +pub use task_queue::init_task_queue; pub use waker::QueueWaker; pub struct PollFn { diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs index 5a237587..0e9d67fb 100644 --- a/src/task/runtime/task_queue.rs +++ b/src/task/runtime/task_queue.rs @@ -17,6 +17,7 @@ pub(super) struct TaskQueue { impl TaskQueue { pub fn new(task_capacity: usize) -> Self { + assert!(task_capacity > 0); Self { pending_workers: ArrayQueue::new(16), task_queue: ArrayQueue::new(task_capacity), @@ -38,8 +39,7 @@ impl TaskQueue { Ok(()) } - pub fn block_pop(&self) -> Result, Error> { - let _irq = IrqGuard::acquire(); + pub fn dequeue(&self) -> Result, Error> { let process = Process::current(); loop { if let Some(task) = self.task_queue.pop() { @@ -49,7 +49,20 @@ impl TaskQueue { if self.pending_workers.push(process.clone()).is_err() { panic!("Pending worker queue overflow"); } + process.suspend(); } } } + +pub fn init_task_queue() { + TASK_QUEUE.init(TaskQueue::new(128)); +} + +pub(super) fn push_task(task: Arc) -> Result<(), Error> { + TASK_QUEUE.get().enqueue(task) +} + +pub(super) fn pop_task() -> Result, Error> { + TASK_QUEUE.get().dequeue() +} diff --git a/src/task/tasklet.rs b/src/task/tasklet.rs deleted file mode 100644 index bcecea04..00000000 --- a/src/task/tasklet.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Small deferred tasks which run in interrupt context -use alloc::{boxed::Box, vec::Vec}; -use core::time::Duration; - -use crate::sync::IrqSafeSpinlock; - -/// Specifies whether a periodic task should be cancelled after its current iteration -#[derive(Debug, PartialEq)] -pub enum TaskFlow { - /// The task will be executed again later - Continue, - /// The task is finished and will be removed from the list - Cancel, -} - -struct PeriodicTaskState { - last_run: Duration, - next_run: Duration, - interval: Duration, - f: Box TaskFlow>, -} - -struct OneTimeTask { - run_time: Duration, - f: Box, -} - -static PERIODIC_TASKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); -static ONE_TIME_TASKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); - -/// Setup a periodic task to run at a specified interval -pub fn add_periodic TaskFlow + 'static>(name: &str, interval: Duration, f: F) { - debugln!("Schedule {:?} every {:?}", name, interval); - let f = Box::new(f); - PERIODIC_TASKS.lock().push(PeriodicTaskState { - interval, - last_run: Duration::ZERO, - next_run: Duration::ZERO, - f, - }); -} - -/// Updates the state of the tasklets, running the ones which meet their deadline conditions -pub fn tick(now: Duration) { - PERIODIC_TASKS.lock().retain_mut(|task| { - if now >= task.next_run { - task.last_run = now; - task.next_run = now + task.interval; - (task.f)() == TaskFlow::Continue - } else { - true - } - }); - - ONE_TIME_TASKS.lock().retain(|task| { - if now >= task.run_time { - (task.f)(); - false - } else { - true - } - }) -} From 1719b84534e50ac00ded0384d5fa1fc5b31a254c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 6 Sep 2023 11:40:59 +0300 Subject: [PATCH 069/211] proc: re-enable signal delivery --- src/device/tty.rs | 5 +++- src/syscall/mod.rs | 10 ++++--- src/task/process.rs | 55 +++++++++++++++++++++++----------- src/task/runtime/executor.rs | 10 +++---- src/task/runtime/task_queue.rs | 3 +- 5 files changed, 55 insertions(+), 28 deletions(-) diff --git a/src/device/tty.rs b/src/device/tty.rs index 3f22c212..158fc04d 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -73,8 +73,11 @@ pub mod combined { impl CharDevice for CombinedTerminal { fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { assert!(blocking); - block! { + match block! { self.line_read(data).await + } { + Ok(res) => res, + Err(err) => Err(err), } // self.line_read(data) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 5e72c89d..075f08dc 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -64,9 +64,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result block! { runtime::sleep(duration).await - }; - - Ok(0) + } + .map(|_| 0) } SyscallFunction::Exit => { let code = ExitCode::from(args[0] as i32); @@ -317,8 +316,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let target = Process::get(pid).ok_or(Error::DoesNotExist)?; - *status = block! { + *status = match block! { Process::wait_for_exit(target).await + } { + Ok(status) => status, + Err(err) => return Err(err), }; Ok(0) diff --git a/src/task/process.rs b/src/task/process.rs index 8079b354..e58bfddc 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -247,14 +247,26 @@ impl Process { return; } } - let current_state = self.state.swap(ProcessState::Ready, Ordering::SeqCst); - - if current_state == ProcessState::Terminated { - todo!("Handle attempt to enqueue an already queued/running/terminated process"); - } else if current_state == ProcessState::Suspended { - unsafe { - queue.enqueue(self); + match self.state.compare_exchange( + ProcessState::Suspended, + ProcessState::Ready, + Ordering::SeqCst, + Ordering::Relaxed, + ) { + Err(ProcessState::Terminated) => { + // Process might've been killed while `await`ing in a `block!` + debugln!( + "Process {} {:?} already terminated, dropping", + self.id(), + self.name() + ); } + Err(state) => { + todo!("Unexpected process state when enqueueing: {:?}", state) + } + Ok(_) => unsafe { + queue.enqueue(self); + }, } } @@ -355,17 +367,15 @@ impl Process { } /// Raises a signal for the currentprocess - pub fn raise_signal(self: &Arc, _signal: Signal) { - // XXX handle signals + async - todo!(); - // { - // let mut inner = self.inner.lock(); - // inner.signal_stack.push_back(signal); - // } + pub fn raise_signal(self: &Arc, signal: Signal) { + { + let mut inner = self.inner.lock(); + inner.signal_stack.push_back(signal); + } - // if self.state() == ProcessState::Suspended { - // self.clone().enqueue_somewhere(); - // } + if self.state() == ProcessState::Suspended { + self.clone().enqueue_somewhere(); + } } /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. @@ -450,6 +460,17 @@ impl CurrentProcess { inner.signal_entry.replace(SignalEntry { entry, stack }); } + pub fn suspend(&self) -> Result<(), Error> { + self.dequeue(ProcessState::Suspended); + + let inner = self.inner.lock(); + if !inner.signal_stack.is_empty() { + return Err(Error::Interrupted); + } + + Ok(()) + } + /// Terminate the current process pub fn exit(&self, status: ExitCode) { self.inner.lock().exit_status = status.into(); diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs index d55ddb1d..0d69ac8f 100644 --- a/src/task/runtime/executor.rs +++ b/src/task/runtime/executor.rs @@ -19,8 +19,6 @@ pub fn spawn_async_worker(index: usize) -> Result<(), Error> { let name = format!("[async-worker-{}]", index); spawn_kernel_closure(name, move || { - // let queue = TASK_QUEUE.or_init_with(task_queue::init_async_queue); - loop { let task = task_queue::pop_task().unwrap(); // queue.block_pop().unwrap(); let mut future_slot = task.future.lock(); @@ -41,7 +39,7 @@ pub fn spawn + Send + 'static>(future: F) -> Result<(), E enqueue(Task::new(future)) } -pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> T { +pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { let process = Process::current(); let mut future = Box::pin(future); @@ -50,9 +48,11 @@ pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> let context = &mut Context::from_waker(&waker); match future.as_mut().poll(context) { - Poll::Ready(value) => break value, + Poll::Ready(value) => break Ok(value), Poll::Pending => { - process.suspend(); + if let Err(error) = process.suspend() { + break Err(error); + } } } } diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs index 0e9d67fb..90fde118 100644 --- a/src/task/runtime/task_queue.rs +++ b/src/task/runtime/task_queue.rs @@ -50,7 +50,8 @@ impl TaskQueue { panic!("Pending worker queue overflow"); } - process.suspend(); + // This must not fail. Signals must not be raised. + process.suspend().unwrap(); } } } From c54357668577d1e16a378758a0851a3393cdafa3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 6 Sep 2023 18:48:00 +0300 Subject: [PATCH 070/211] dev: optimize display output a bit --- Cargo.toml | 7 +- lib/kernel-util/src/lib.rs | 6 ++ lib/kernel-util/src/util.rs | 2 +- src/device/display/console.rs | 11 +-- src/device/display/fb_console.rs | 136 ++++++++++++------------------- src/device/display/font.psfu | Bin 0 -> 4969 bytes src/device/display/font.rs | 65 +++++++++++++++ src/device/display/mod.rs | 1 + 8 files changed, 135 insertions(+), 93 deletions(-) create mode 100644 src/device/display/font.psfu create mode 100644 src/device/display/font.rs diff --git a/Cargo.toml b/Cargo.toml index 3221e470..94605465 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,12 +22,13 @@ tock-registers = "0.8.1" cfg-if = "1.0.0" git-version = "0.3.5" -bitmap-font = { version = "0.3.0", optional = true } -embedded-graphics = { version = "0.8.0", optional = true } +# bitmap-font = { version = "0.3.0", optional = true } +# embedded-graphics = { version = "0.8.0", optional = true } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } +bytemuck = { version = "1.14.0", features = ["derive"] } [dependencies.elf] version = "0.7.2" @@ -49,7 +50,7 @@ xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } [features] default = [] -fb_console = ["bitmap-font", "embedded-graphics"] +fb_console = [] aarch64_qemu = [] aarch64_orange_pi3 = [] aarch64_rpi3b = [] diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 06f7b372..9597c0aa 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -3,3 +3,9 @@ pub mod sync; pub mod util; + +#[repr(C)] +pub struct AlignedTo { + pub align: [Align; 0], + pub bytes: Bytes, +} diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util.rs index 5e11e00a..8f505750 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util.rs @@ -4,7 +4,7 @@ use core::{ mem::MaybeUninit, ops::{Deref, DerefMut}, panic, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + sync::atomic::{AtomicUsize, Ordering}, }; /// Statically-allocated "dynamic" vector diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 01da5ad4..1b8927ad 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -133,8 +133,6 @@ pub struct RowIter<'a> { pub trait DisplayConsole { /// Returns the state lock fn state(&self) -> &IrqSafeSpinlock; - /// Allocates a new buffer for the console according to the display's size - fn reallocate_buffer(&self) -> Result<(), Error>; /// Flushes the data from console buffer to the display fn flush(&self, state: &mut ConsoleState); @@ -157,7 +155,7 @@ impl ConsoleChar { }; /// Constructs a console character from a char and its attributes - #[inline] + #[inline(always)] pub fn from_parts( char: u8, fg: ColorAttribute, @@ -175,7 +173,7 @@ impl ConsoleChar { } /// Returns the attributes of the character - #[inline] + #[inline(always)] pub fn attributes(self) -> (ColorAttribute, ColorAttribute, Attributes) { let fg = ColorAttribute::try_from((self.attributes & 0xF) as u8).unwrap_or(DEFAULT_FG_COLOR); @@ -188,7 +186,7 @@ impl ConsoleChar { } /// Returns the character data of this [ConsoleChar] - #[inline] + #[inline(always)] pub const fn character(self) -> u8 { self.char } @@ -369,6 +367,9 @@ impl ConsoleState { self.esc_state = EscapeState::Escape; return false; } + b'\r' => { + self.cursor_col = 0; + } b'\n' => { self.cursor_row += 1; self.cursor_col = 0; diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 952e547e..6890b168 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -1,33 +1,23 @@ //! Framebuffer console driver use abi::error::Error; -use bitmap_font::{BitmapFont, TextStyle}; -use embedded_graphics::{ - pixelcolor::BinaryColor, - prelude::{DrawTarget, OriginDimensions, Point, Size}, - text::Text, - Drawable, Pixel, -}; use crate::{debug::DebugSink, sync::IrqSafeSpinlock}; use super::{ console::{Attributes, ConsoleBuffer, ConsoleState, DisplayConsole}, + font::PcScreenFont, linear_fb::LinearFramebuffer, - DisplayDevice, DisplayDimensions, + DisplayDevice, }; struct Inner { framebuffer: &'static LinearFramebuffer, - font: &'static BitmapFont<'static>, + font: PcScreenFont<'static>, char_width: u32, char_height: u32, width: u32, height: u32, - - fg_color: u32, - #[allow(dead_code)] - bg_color: u32, } /// Framebuffer console device wrapper @@ -52,20 +42,14 @@ impl DisplayConsole for FramebufferConsole { &self.state } - fn reallocate_buffer(&self) -> Result<(), Error> { - let mut state = self.state.lock(); - let inner = self.inner.lock(); - - let new_height = inner.height; - state.buffer.reallocate(new_height) - } - fn flush(&self, state: &mut ConsoleState) { let mut inner = self.inner.lock(); let font = inner.font; let cw = inner.char_width; let ch = inner.char_height; + let bytes_per_line = (font.width() as usize + 7) / 8; + let mut iter = state.buffer.flush_rows(); while let Some((row_idx, row)) = iter.next_dirty() { @@ -77,20 +61,18 @@ impl DisplayConsole for FramebufferConsole { let glyph = chr.character(); let (fg, bg, attr) = chr.attributes(); - if glyph == 0 { - continue; - } + let fg = fg.as_rgba(attr.contains(Attributes::BOLD)); + let bg = bg.as_rgba(false); - inner.fg_color = fg.as_rgba(attr.contains(Attributes::BOLD)); - - inner.fill_rect( + inner.draw_glyph( + font, (col_idx as u32) * cw, row_idx * ch, - cw, - ch, - bg.as_rgba(false), + glyph, + fg, + bg, + bytes_per_line, ); - inner.draw_glyph(font, (col_idx as u32) * cw, row_idx * ch, glyph); } } @@ -112,9 +94,9 @@ impl FramebufferConsole { /// Constructs an instance of console from its framebuffer reference pub fn from_framebuffer( framebuffer: &'static LinearFramebuffer, - font: Option<&'static BitmapFont<'static>>, + font: Option>, ) -> Result { - let font = font.unwrap_or(&bitmap_font::tamzen::FONT_7x14); + let font = font.unwrap_or(PcScreenFont::default()); let char_width = font.width(); let char_height = font.height(); let dim = framebuffer.dimensions(); @@ -127,9 +109,6 @@ impl FramebufferConsole { height: dim.height / char_height, char_width, char_height, - - fg_color: 0, - bg_color: 0, }; Ok(Self { @@ -140,62 +119,51 @@ impl FramebufferConsole { } impl Inner { - fn draw_glyph(&mut self, font: &BitmapFont, x: u32, y: u32, c: u8) { - let text_data = [c]; - let text_str = unsafe { core::str::from_utf8_unchecked(&text_data) }; - let text = Text::new( - text_str, - Point::new(x as _, y as _), - TextStyle::new(font, BinaryColor::On), - ); + #[optimize(speed)] + fn draw_glyph( + &mut self, + font: PcScreenFont<'static>, + sx: u32, + sy: u32, + c: u8, + fg: u32, + bg: u32, + bytes_per_line: usize, + ) { + let mut fb = unsafe { self.framebuffer.lock() }; - text.draw(self).ok(); + let mut c = c as u32; + if c >= font.len() { + c = b'?' as u32; + } + + let mut glyph = font.raw_glyph_data(c); + + let mut y = 0; + + while y < font.height() { + let mut mask = 1 << (font.width() - 1); + let mut x = 0; + + while x < font.width() { + let v = if glyph[0] & mask != 0 { fg } else { bg }; + fb[sy + y][(sx + x) as usize] = v; + mask >>= 1; + x += 1; + } + + glyph = &glyph[bytes_per_line..]; + y += 1; + } } + #[optimize(speed)] fn fill_rect(&mut self, x: u32, y: u32, w: u32, h: u32, val: u32) { let mut fb = unsafe { self.framebuffer.lock() }; for i in 0..h { let row = &mut fb[i + y]; - for j in 0..w { - row[(j + x) as usize] = val; - } + row[x as usize..(x + w) as usize].fill(val); } } } - -impl OriginDimensions for Inner { - fn size(&self) -> Size { - self.framebuffer.dimensions().into() - } -} - -impl DrawTarget for Inner { - type Color = BinaryColor; - type Error = (); - - fn draw_iter(&mut self, pixels: I) -> Result<(), Self::Error> - where - I: IntoIterator>, - { - let mut fb = unsafe { self.framebuffer.lock() }; - for Pixel(coord, color) in pixels { - let row = &mut fb[coord.y as u32]; - let color = if color.is_on() { - self.fg_color - } else { - 0xFF000000 - }; - - row[coord.x as usize] = color; - } - - Ok(()) - } -} - -impl From for Size { - fn from(value: DisplayDimensions) -> Self { - Self::new(value.width as _, value.height as _) - } -} diff --git a/src/device/display/font.psfu b/src/device/display/font.psfu new file mode 100644 index 0000000000000000000000000000000000000000..e789b77189b9c3e585075eeed5a5eba693f3c435 GIT binary patch literal 4969 zcmZu!TWlO>6&`}L;$h{6w-K?N0^xq?(t@+JgAJkF3KS@m`(;VWHo(xD0*;}>W}P~& zLoL*ccN5dnI7yQ<7bp!)5NV<;Gt!6$Yx9OyT4P1?V80+_sZ!#HXg|L5UuI`qj{VQf z`Oo*C|J?pFvyYs=XWAj%%JBwy;AJy!GTtQo>!>z)d~)*g{N!X!i)!cs(JEkg$MJil zpe!mBZ7v5{*7dyA)m7K^jM(*RT-@h5O^Bd}QQ9 z18=!~Vv#hVHF#B$=W+whkKQp_B8{zG)BN6Ddv}o@TwQ7D3D@m#b{R>%H~0 zYpuExZz>E7jr2x_1`3-D+@or@W({^^I}-0&zPH=$KDtFn@j+ZE`*G}-i*cX<>M9}A zEW@GonRsfRQ^tOIBK0?9xQe}^kt!XfO0QDND0f<|meHa`7h)B;MLkn;i;CR7hgA=riMGNpI(9Fsg@$AnIBBh>a_bK#HWkD^lW!-8 z<1T4UbKP$rrprp*JM{8{=iFA~Nw<7)Sgq(z{fl?6MhxS@%U)t@`U)-mC81J+Zs8vufgH_)29h zvwk^_t9NQ%c2e7!{iOI5d0IZCs-b`0SGd*01UFsL^0kVhbU*1N-AKsjU%B2}Zuk9B zu@=_&tNarc#y%te*K$xC5DiK$KAQa~uat|0bUnKBIeSI5BBNiK5=CrL+8)JsyRiL3 ziCnL0?WLSplJOtWv?T8zk?xI-r;kgiWaJJz*jRZm?N^>YKE3MaoRVor~7C{OLVitgn-ynDL2{-3kKtaqyVC7~1fr`qJi7(~j0$&V@s)bX( zKQ8)Bd|8@h6;R_}q2LVgIFQ)pxtMre00ytY{df_w0`}x?mrt6OM7@nSr*w+@{%CoS zmVmX-9_gj=YQH|zdRBn-Z*!@1Q&KWKXm)cPVx(*}>xlDJe{lPx{i6d$yDkEwmeZi%EK!<%=Fopq#~vb5DQ29e^;d>+5mP4ULQ?baW4 z-(DI`can$EQeJ+;N%bonoEsa~r+6bz@d|gNk5~4gug~a7@pk=2|8n~c52yAj9Q{-4 z>?Px`nHt@&9+ubksN-t4l>5efrSVfgf-)DQKb=|IK(}ll1GJXv`}k6)(^=Br=Z^fg z>MXJav9SyB+l+@%NH`(+{1?(30qkl0_Y-GpRM%16MM2cdmmRic7K|@xR}EWZ3s=A3 z3~$;q+k_%vW7+P%njhll?1-q;{YfTkeA(Bul-D;F$iG&z+YOc~(%vhjL4&aUHMT2w zSoFV{Fh4JBO_ziR^`9N7mVR!_9&D2Q*QENPdG6or%RHTN z?b9vl^X@;s?`eNJ5*nKdg2pPX;Q<%dU9|I$#jDYFCn~u@FZo#OuxwS6vs0h zPjftpWae-(OO4ZWgq-8Jgj>Ufq+U9|WFt($NkK#_;1rHy?2uAU7 zd;&Z0N!*Q3;nVmGK8w%c9(*2Oz!z~ZzJxF1E4U9|#n*5@zK(C;oA?&Kjql*Q_#VEG zAE1ICVkdUt0sIJ6Jc!-+F@A!d!pG0>5FW-DevU`*3;Ytl!Z`L|0>8#@@F)WO7BwWZ z`w%^j=n|q+h>jw90nq}Y(2!XgILk2pjqD7{%D_#bt?PtX7W literal 0 HcmV?d00001 diff --git a/src/device/display/font.rs b/src/device/display/font.rs new file mode 100644 index 00000000..cbebdcfc --- /dev/null +++ b/src/device/display/font.rs @@ -0,0 +1,65 @@ +use core::mem::size_of; + +use abi::error::Error; +use bytemuck::{Pod, Zeroable}; +use kernel_util::AlignedTo; + +// static CONSOLE_FONT: &[u8] = include_bytes!("font.psfu"); +static CONSOLE_FONT: &'static AlignedTo = &AlignedTo { + align: [], + bytes: *include_bytes!("font.psfu"), +}; + +#[repr(C)] +#[derive(Pod, Zeroable, Clone, Copy)] +struct PsfHeader { + magic: u32, + version: u32, + header_size: u32, + flags: u32, + num_glyph: u32, + bytes_per_glyph: u32, + height: u32, + width: u32, +} + +#[derive(Clone, Copy)] +pub struct PcScreenFont<'a> { + header: &'a PsfHeader, + glyph_data: &'a [u8], +} + +impl Default for PcScreenFont<'static> { + fn default() -> Self { + Self::from_bytes(&CONSOLE_FONT.bytes).unwrap() + } +} + +impl<'a> PcScreenFont<'a> { + pub fn from_bytes(bytes: &'a [u8]) -> Result { + let header: &PsfHeader = bytemuck::from_bytes(&bytes[..size_of::()]); + let glyph_data = &bytes[header.header_size as usize..]; + + Ok(Self { header, glyph_data }) + } + + #[inline] + pub const fn width(&self) -> u32 { + self.header.width + } + + #[inline] + pub const fn height(&self) -> u32 { + self.header.height + } + + #[inline] + pub const fn len(&self) -> u32 { + self.header.num_glyph + } + + #[inline] + pub fn raw_glyph_data(&self, index: u32) -> &[u8] { + &self.glyph_data[(index * self.header.bytes_per_glyph) as usize..] + } +} diff --git a/src/device/display/mod.rs b/src/device/display/mod.rs index e503e3ae..02df0cca 100644 --- a/src/device/display/mod.rs +++ b/src/device/display/mod.rs @@ -3,6 +3,7 @@ use super::Device; pub mod console; +pub mod font; #[cfg(feature = "fb_console")] pub mod fb_console; From 3330beedfd864cfa8a3a8027d2060b0d868a3a41 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 7 Sep 2023 11:59:20 +0300 Subject: [PATCH 071/211] x86-64: couldn't get HPET to work on real hw --- Cargo.toml | 5 +- src/arch/x86_64/mod.rs | 14 +- src/arch/x86_64/peripherals/hpet.rs | 279 --------------------------- src/arch/x86_64/peripherals/i8253.rs | 97 ++++++++++ src/arch/x86_64/peripherals/mod.rs | 2 +- 5 files changed, 107 insertions(+), 290 deletions(-) delete mode 100644 src/arch/x86_64/peripherals/hpet.rs create mode 100644 src/arch/x86_64/peripherals/i8253.rs diff --git a/Cargo.toml b/Cargo.toml index 94605465..5d6c01f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,8 @@ edition = "2021" build = "build.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[profile.dev.package.tock-registers] +opt-level = 3 [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } @@ -22,9 +24,6 @@ tock-registers = "0.8.1" cfg-if = "1.0.0" git-version = "0.3.5" -# bitmap-font = { version = "0.3.0", optional = true } -# embedded-graphics = { version = "0.8.0", optional = true } - log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 4cb59e63..b6621f82 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -2,7 +2,7 @@ use core::sync::atomic::Ordering; use abi::error::Error; -use acpi_lib::{mcfg::Mcfg, AcpiTables, HpetInfo, InterruptModel}; +use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use cpu::Cpu; use device_api::{ @@ -45,7 +45,7 @@ use self::{ acpi::{AcpiAllocator, AcpiHandlerImpl}, apic::ioapic::IoApic, intrinsics::{IoPort, IoPortAccess}, - peripherals::{hpet::Hpet, ps2::PS2Controller}, + peripherals::{i8253::I8253, ps2::PS2Controller}, smp::CPU_COUNT, }; @@ -140,7 +140,7 @@ pub struct X86_64 { combined_terminal: OneTimeInit, ioapic: OneTimeInit, - timer: OneTimeInit, + timer: OneTimeInit, } /// x86-64 architecture implementation @@ -392,9 +392,6 @@ impl X86_64 { self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap()); - let hpet = HpetInfo::new(acpi).unwrap(); - self.timer.init(Hpet::from_acpi(&hpet).unwrap()); - acpi::init_acpi(acpi).unwrap(); // Enumerate PCIe root devices @@ -442,8 +439,11 @@ impl X86_64 { if cpu_id == 0 { self.init_acpi_from_boot_data(); + Self::disable_8259(); + self.timer.init(I8253::new()); + // Initialize debug output as soon as possible let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)))); debug::add_sink(com1_3.port_a(), LogLevel::Debug); @@ -478,7 +478,7 @@ impl X86_64 { ps2.init_irq().unwrap(); device::register_device(self.ioapic.get()); - device::register_device(self.timer.get()); + // device::register_device(self.timer.get()); device::register_device(ps2); // Initialize devices from PCI bus diff --git a/src/arch/x86_64/peripherals/hpet.rs b/src/arch/x86_64/peripherals/hpet.rs deleted file mode 100644 index 3267918d..00000000 --- a/src/arch/x86_64/peripherals/hpet.rs +++ /dev/null @@ -1,279 +0,0 @@ -//! x86-64 implementation of a HPET -use core::time::Duration; - -use abi::error::Error; -use acpi_lib::hpet::HpetInfo as AcpiHpet; -use device_api::{ - interrupt::{InterruptHandler, IrqLevel, IrqOptions, IrqTrigger}, - timer::MonotonicTimestampProviderDevice, - Device, -}; -use tock_registers::{ - interfaces::{ReadWriteable, Readable, Writeable}, - register_bitfields, register_structs, - registers::{ReadOnly, ReadWrite}, -}; - -use crate::{ - arch::{x86_64::IrqNumber, Architecture, ARCHITECTURE}, - device::timer, - mem::device::DeviceMemoryIo, - sync::IrqSafeSpinlock, -}; - -register_bitfields! { - u64, - GeneralCapabilities [ - COUNTER_CLK_PERIOD OFFSET(32) NUMBITS(32) [], - COUNT_SIZE_CAP OFFSET(13) NUMBITS(1) [ - Supports64Bit = 1 - ], - NUM_TIM_CAP OFFSET(8) NUMBITS(5) [], - ], - GeneralConfiguration [ - LEG_RT_CNF OFFSET(1) NUMBITS(1) [], - ENABLE_CNF OFFSET(0) NUMBITS(1) [], - ], - - TimerConfigurationCapablities [ - TM_INT_TYPE_CNF OFFSET(1) NUMBITS(1) [ - EdgeTriggered = 0, - LevelTriggered = 1, - ], - TM_INT_ENB_CNF OFFSET(2) NUMBITS(1) [ - InterruptEnabled = 1, - InterruptDisabled = 0 - ], - TM_TYPE_CNF OFFSET(3) NUMBITS(1) [ - Periodic = 1, - OneShot = 0 - ], - TM_PER_INT_CAP OFFSET(4) NUMBITS(1) [ - SupportsPeriodic = 1, - ], - TM_SIZE_CAP OFFSET(5) NUMBITS(1) [ - Timer64Bit = 1, - Timer32Bit = 0, - ], - TM_VAL_SET_CNF OFFSET(6) NUMBITS(1) [], - TM_32MODE_CNF OFFSET(8) NUMBITS(1) [], - TM_INT_ROUTE_CNF OFFSET(9) NUMBITS(5) [], - TM_FSB_EN_CNF OFFSET(14) NUMBITS(1) [], - TM_FSB_INT_DEL_CAP OFFSET(15) NUMBITS(1) [], - TM_INT_ROUTE_CAP OFFSET(32) NUMBITS(32) [], - ] -} - -register_structs! { - #[allow(non_snake_case)] - TimerRegs { - (0x000 => ConfigurationCapability: ReadWrite), - (0x008 => Comparator: ReadWrite), - (0x010 => FsbInterruptRoute: ReadWrite), - (0x018 => _0), - (0x020 => @END), - } -} - -register_structs! { - #[allow(non_snake_case)] - Regs { - (0x000 => GeneralCapabilities: ReadOnly), - (0x008 => _0), - (0x010 => GeneralConfiguration: ReadWrite), - (0x018 => _1), - (0x020 => GeneralInterruptStatus: ReadWrite), - (0x028 => _2), - - (0x0F0 => MainCounter: ReadWrite), - (0x0F8 => _3), - (0x100 => Timers: [TimerRegs; 3]), - (0x160 => _4), - (0x400 => @END), - } -} - -struct Inner { - regs: DeviceMemoryIo, - tim0_period: u64, - tim0_counter: u64, -} - -/// HPET timer group interface -pub struct Hpet { - inner: IrqSafeSpinlock, -} - -impl Inner { - unsafe fn new(base: usize) -> Result { - // Femtoseconds in a millisecond - const FS_IN_MS: u64 = 1000000000000; - - let regs = DeviceMemoryIo::::map("hpet", base)?; - - // Disable the counter first - regs.GeneralConfiguration - .modify(GeneralConfiguration::ENABLE_CNF::CLEAR); - - // Disable legacy routing - regs.GeneralConfiguration - .modify(GeneralConfiguration::LEG_RT_CNF::CLEAR); - - // Reset the counter - regs.MainCounter.set(0); - - // Disable all comparator interrupts - for i in 0..3 { - regs.Timers[i] - .ConfigurationCapability - .modify(TimerConfigurationCapablities::TM_INT_ENB_CNF::InterruptDisabled); - } - - // Calculate the interrupt period - let clk_period = regs - .GeneralCapabilities - .read(GeneralCapabilities::COUNTER_CLK_PERIOD); - - // Will give an interrupt interval of 1ms - let tim0_period = FS_IN_MS / clk_period; - - // Enable the main counter - regs.GeneralConfiguration - .modify(GeneralConfiguration::ENABLE_CNF::SET); - - Ok(Self { - regs, - tim0_period, - tim0_counter: 0, - }) - } -} - -impl MonotonicTimestampProviderDevice for Hpet { - fn monotonic_timestamp(&self) -> Result { - let inner = self.inner.lock(); - let counter = inner.regs.MainCounter.get(); - let period = inner.tim0_period; - Ok(Duration::from_millis(counter / period)) - } -} - -impl InterruptHandler for Hpet { - fn handle_irq(&self) -> bool { - let now = { - let mut inner = self.inner.lock(); - inner.regs.GeneralInterruptStatus.set(1); - - inner.tim0_counter = inner.tim0_counter.wrapping_add(1); - Duration::from_millis(inner.tim0_counter) - }; - - timer::tick(now); - - true - } -} - -impl Device for Hpet { - fn display_name(&self) -> &'static str { - "HPET" - } - - unsafe fn init_irq(&'static self) -> Result<(), Error> { - // Configure timer 0 - let intc = ARCHITECTURE.external_interrupt_controller(); - let inner = self.inner.lock(); - - let tim0 = &inner.regs.Timers[0]; - - if inner.tim0_period > 0x100000000 - && !tim0 - .ConfigurationCapability - .matches_all(TimerConfigurationCapablities::TM_SIZE_CAP::Timer64Bit) - { - panic!("Period is too large and timer doesn't support 64-bit"); - } - - // Temporarily disable the main counter - inner - .regs - .GeneralConfiguration - .modify(GeneralConfiguration::ENABLE_CNF::CLEAR); - - assert!(tim0 - .ConfigurationCapability - .matches_all(TimerConfigurationCapablities::TM_PER_INT_CAP::SupportsPeriodic)); - - let mut tim0_irq = None; - let tim0_irqs = tim0 - .ConfigurationCapability - .read(TimerConfigurationCapablities::TM_INT_ROUTE_CAP); - - // Skip most likely legacy routes: IRQ2, IRQ0 - for i in 3..32 { - if tim0_irqs & (1 << i) != 0 { - tim0_irq = Some(i); - break; - } - } - let tim0_irq = tim0_irq.expect("Could not pick an IRQ for HPET TIM0"); - - // Bind and enable the IRQ - let irq = IrqNumber::Gsi(tim0_irq); - intc.register_irq( - irq, - IrqOptions { - level: IrqLevel::ActiveLow, - trigger: IrqTrigger::Level, - }, - self, - )?; - intc.enable_irq(irq)?; - - // Disable FSB interrupt route and 32 bit mode - tim0.ConfigurationCapability.modify( - TimerConfigurationCapablities::TM_FSB_EN_CNF::CLEAR - + TimerConfigurationCapablities::TM_32MODE_CNF::CLEAR - + TimerConfigurationCapablities::TM_VAL_SET_CNF::SET, - ); - - // Setup interrupt route + edge-triggered - tim0.ConfigurationCapability.modify( - TimerConfigurationCapablities::TM_INT_ROUTE_CNF.val(tim0_irq as u64) - + TimerConfigurationCapablities::TM_INT_TYPE_CNF::LevelTriggered, - ); - - // Setup periodic mode - tim0.ConfigurationCapability - .modify(TimerConfigurationCapablities::TM_TYPE_CNF::Periodic); - - // Set the comparator - tim0.Comparator.set(inner.tim0_period); - - // Enable the timer - tim0.ConfigurationCapability - .modify(TimerConfigurationCapablities::TM_INT_ENB_CNF::InterruptEnabled); - - // Reenable the main counter - inner - .regs - .GeneralConfiguration - .modify(GeneralConfiguration::ENABLE_CNF::SET); - - Ok(()) - } -} - -impl Hpet { - /// Creates a HPET instance from its ACPI definition - pub fn from_acpi(info: &AcpiHpet) -> Result { - infoln!("Initializing HPET:"); - infoln!("Address: {:#x}", info.base_address); - - let inner = unsafe { Inner::new(info.base_address) }?; - - Ok(Self { - inner: IrqSafeSpinlock::new(inner), - }) - } -} diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs new file mode 100644 index 00000000..ab0bb491 --- /dev/null +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -0,0 +1,97 @@ +use core::time::Duration; + +use abi::error::Error; +use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; + +use crate::{ + arch::{ + x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + IrqNumber, + }, + Architecture, ARCHITECTURE, + }, + sync::IrqSafeSpinlock, + task::runtime, +}; + +const FREQUENCY: u32 = 1193180; + +const CMD_CH0: u8 = 0 << 6; +const CMD_ACC_WORD: u8 = 3 << 4; +const CMD_MODE_RATE: u8 = 2 << 1; + +struct Inner { + ch0_data: IoPort, + #[allow(unused)] + ch1_data: IoPort, + #[allow(unused)] + ch2_data: IoPort, + cmd: IoPort, + + tick: u64, +} + +pub struct I8253 { + inner: IrqSafeSpinlock, +} + +impl MonotonicTimestampProviderDevice for I8253 { + fn monotonic_timestamp(&self) -> Result { + let tick = self.inner.lock().tick; + + Ok(Duration::from_millis(tick)) + } +} + +impl InterruptHandler for I8253 { + fn handle_irq(&self) -> bool { + let mut inner = self.inner.lock(); + inner.tick += 1; + + let now = Duration::from_millis(inner.tick); + drop(inner); + + runtime::tick(now); + + true + } +} + +impl Device for I8253 { + fn display_name(&self) -> &'static str { + "i8253 PIT" + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + let intc = ARCHITECTURE.external_interrupt_controller(); + let inner = self.inner.lock(); + + let div: u16 = (FREQUENCY / 1000).try_into().unwrap(); + + inner.cmd.write(CMD_CH0 | CMD_ACC_WORD | CMD_MODE_RATE); + inner.ch0_data.write(div as u8); + inner.ch0_data.write((div >> 8) as u8); + + let irq = IrqNumber::Isa(0); + intc.register_irq(irq, Default::default(), self)?; + intc.enable_irq(irq)?; + + Ok(()) + } +} + +impl I8253 { + pub const fn new() -> Self { + Self { + inner: IrqSafeSpinlock::new(Inner { + ch0_data: IoPort::new(0x40), + ch1_data: IoPort::new(0x41), + ch2_data: IoPort::new(0x42), + cmd: IoPort::new(0x43), + + tick: 0, + }), + } + } +} diff --git a/src/arch/x86_64/peripherals/mod.rs b/src/arch/x86_64/peripherals/mod.rs index d2183adc..64a4ff6d 100644 --- a/src/arch/x86_64/peripherals/mod.rs +++ b/src/arch/x86_64/peripherals/mod.rs @@ -1,5 +1,5 @@ //! x86-64 platform peripheral drivers -pub mod hpet; +pub mod i8253; pub mod ps2; pub mod serial; From 6949f8c44a30e8405b175982da1ea9c49652e9ad Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 13 Sep 2023 18:21:45 +0300 Subject: [PATCH 072/211] mem: rework phys/virt mem management, get process working --- Cargo.toml | 5 +- lib/memtables/Cargo.toml | 9 + lib/memtables/src/lib.rs | 39 + src/arch/mod.rs | 67 +- src/arch/x86_64/acpi.rs | 91 +-- src/arch/x86_64/apic/ioapic.rs | 47 +- src/arch/x86_64/apic/local.rs | 11 +- src/arch/x86_64/boot/mod.rs | 144 ++-- src/arch/x86_64/context.rs | 29 +- src/arch/x86_64/cpuid.rs | 67 +- src/arch/x86_64/mem/mod.rs | 318 ++++++++ .../x86_64/{table/mod.rs => mem/table.rs} | 159 ++-- src/arch/x86_64/mod.rs | 687 ++++++++---------- src/arch/x86_64/smp.rs | 33 +- src/arch/x86_64/table/fixed.rs | 161 ---- src/debug.rs | 2 +- src/device/display/console.rs | 62 +- src/device/display/linear_fb.rs | 12 +- src/device/mod.rs | 4 +- src/fs/mod.rs | 16 +- src/main.rs | 9 +- src/mem/address.rs | 185 +++++ src/mem/device.rs | 119 ++- src/mem/heap.rs | 2 +- src/mem/mod.rs | 150 ++-- src/mem/phys/manager.rs | 190 +++-- src/mem/phys/mod.rs | 347 +++++---- src/mem/phys/reserved.rs | 16 +- src/mem/pointer.rs | 132 ++++ src/mem/table.rs | 23 +- src/proc/elf.rs | 31 +- src/proc/exec.rs | 12 +- src/task/context.rs | 9 +- src/task/process.rs | 9 +- src/task/runtime/task_queue.rs | 7 +- src/task/sched.rs | 5 +- tools/gentables/Cargo.toml | 15 + tools/gentables/src/main.rs | 186 +++++ tools/gentables/src/x86_64.rs | 189 +++++ 39 files changed, 2264 insertions(+), 1335 deletions(-) create mode 100644 lib/memtables/Cargo.toml create mode 100644 lib/memtables/src/lib.rs create mode 100644 src/arch/x86_64/mem/mod.rs rename src/arch/x86_64/{table/mod.rs => mem/table.rs} (68%) delete mode 100644 src/arch/x86_64/table/fixed.rs create mode 100644 src/mem/address.rs create mode 100644 src/mem/pointer.rs create mode 100644 tools/gentables/Cargo.toml create mode 100644 tools/gentables/src/main.rs create mode 100644 tools/gentables/src/x86_64.rs diff --git a/Cargo.toml b/Cargo.toml index 5d6c01f9..4abfd002 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,12 +14,13 @@ vfs = { path = "lib/vfs" } memfs = { path = "lib/memfs" } device-api = { path = "lib/device-api", features = ["derive"] } kernel-util = { path = "lib/kernel-util" } +memtables = { path = "lib/memtables" } atomic_enum = "0.2.0" bitflags = "2.3.3" linked_list_allocator = "0.10.5" spinning_top = "0.2.5" -# static_assertions = "1.1.0" +static_assertions = "1.1.0" tock-registers = "0.8.1" cfg-if = "1.0.0" git-version = "0.3.5" @@ -48,7 +49,7 @@ acpi-system = { git = "https://github.com/alnyan/acpi-system.git", version = "0. xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } [features] -default = [] +default = ["fb_console"] fb_console = [] aarch64_qemu = [] aarch64_orange_pi3 = [] diff --git a/lib/memtables/Cargo.toml b/lib/memtables/Cargo.toml new file mode 100644 index 00000000..948e37e0 --- /dev/null +++ b/lib/memtables/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "memtables" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bytemuck = { version = "1.14.0", features = ["derive"] } diff --git a/lib/memtables/src/lib.rs b/lib/memtables/src/lib.rs new file mode 100644 index 00000000..79caa00c --- /dev/null +++ b/lib/memtables/src/lib.rs @@ -0,0 +1,39 @@ +#![no_std] + +use bytemuck::{Pod, Zeroable}; + +pub const KERNEL_L3_COUNT: usize = 16; + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C, align(0x1000))] +pub struct RawTable { + pub data: [u64; 512], +} + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub struct FixedTables { + pub l0: RawTable, + + pub kernel_l1: RawTable, + pub kernel_l2: RawTable, + pub kernel_l3s: [RawTable; KERNEL_L3_COUNT], +} + +impl RawTable { + pub const fn zeroed() -> Self { + Self { data: [0; 512] } + } +} + +impl FixedTables { + pub const fn zeroed() -> Self { + Self { + l0: RawTable::zeroed(), + + kernel_l1: RawTable::zeroed(), + kernel_l2: RawTable::zeroed(), + kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], + } + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 8f1477ef..452d8c89 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -27,6 +27,8 @@ use device_api::{ ResetDevice, }; +use crate::mem::{device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, PhysicalAddress}; + cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod aarch64; @@ -60,24 +62,31 @@ pub trait Architecture { /// IRQ number type associated with the architecture type IrqNumber; - /// Initializes the memory management unit and sets up virtual memory management. - /// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP - /// system. - /// - /// # Safety - /// - /// Unsafe to call if the MMU has already been initialized. - unsafe fn init_mmu(&self, bsp: bool); - /// Starts up the application processors that may be present in the system. /// /// # Safety /// /// Only safe to call once during system init. - unsafe fn start_application_processors(&self); + unsafe fn start_application_processors(&self) {} /// Allocates a virtual mapping for the specified physical memory region - fn map_device_pages(&self, phys: usize, count: usize) -> Result; + unsafe fn map_device_memory( + &self, + base: PhysicalAddress, + size: usize, + ) -> Result; + unsafe fn unmap_device_memory(&self, map: RawDeviceMemoryMapping); + + fn map_physical_memory + Clone>( + &self, + it: I, + memory_start: PhysicalAddress, + memory_end: PhysicalAddress, + ) -> Result<(), Error>; + + fn virtualize(address: PhysicalAddress) -> Result; + + fn physicalize(address: usize) -> Result; // Architecture intrinsics @@ -102,36 +111,50 @@ pub trait Architecture { fn register_external_interrupt_controller( &self, intc: &'static dyn ExternalInterruptController, - ) -> Result<(), Error>; + ) -> Result<(), Error> { + Err(Error::NotImplemented) + } /// Adds a local interrupt controller to the system fn register_local_interrupt_controller( &self, intc: &'static dyn LocalInterruptController, - ) -> Result<(), Error>; + ) -> Result<(), Error> { + Err(Error::NotImplemented) + } /// Adds a monotonic timer to the system fn register_monotonic_timer( &self, timer: &'static dyn MonotonicTimestampProviderDevice, - ) -> Result<(), Error>; + ) -> Result<(), Error> { + Err(Error::NotImplemented) + } /// Adds a reset device to the system - fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error>; + fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { + Err(Error::NotImplemented) + } // TODO only supports 1 extintc per system /// Returns the primary external interrupt controller fn external_interrupt_controller( &'static self, - ) -> &'static dyn ExternalInterruptController; + ) -> &'static dyn ExternalInterruptController { + unimplemented!() + } /// Returns the local interrupt controller fn local_interrupt_controller( &'static self, - ) -> &'static dyn LocalInterruptController; + ) -> &'static dyn LocalInterruptController { + unimplemented!() + } /// Returns the monotonic timer - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice; + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { + unimplemented!() + } /// Sends a message to the requested set of CPUs through an interprocessor interrupt. /// @@ -143,7 +166,9 @@ pub trait Architecture { /// # Safety /// /// As the call may alter the flow of execution on CPUs, this function is unsafe. - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>; + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { + Ok(()) + } /// Performs a CPU reset. /// @@ -151,5 +176,7 @@ pub trait Architecture { /// /// The caller must ensure it is actually safe to reset, i.e. no critical processes will be /// aborted and no data will be lost. - unsafe fn reset(&self) -> !; + unsafe fn reset(&self) -> ! { + loop {} + } } diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 3a24c0f2..7234faf5 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -23,7 +23,7 @@ use crate::{ x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, Architecture, CpuMessage, ARCHITECTURE, }, - mem::{heap::GLOBAL_HEAP, ConvertAddress}, + mem::{address::FromRaw, heap::GLOBAL_HEAP, PhysicalAddress}, sync::IrqSafeSpinlock, util, }; @@ -79,14 +79,19 @@ unsafe impl Allocator for AcpiAllocator { impl acpi_system::Handler for AcpiHandlerImpl { unsafe fn map_slice(address: u64, length: u64) -> &'static [u8] { - if address + length < 0x100000000 { - core::slice::from_raw_parts( - (address as usize).virtualize() as *const u8, - length as usize, - ) - } else { - panic!("Unhandled address: {:#x}", address) - } + let slice = PhysicalAddress::from_raw(address).virtualize_slice::(length as usize); + + todo!(); + // PhysicalPointer::into_raw(slice) + + // if address + length < 0x100000000 { + // core::slice::from_raw_parts( + // (address as usize).virtualize() as *const u8, + // length as usize, + // ) + // } else { + // panic!("Unhandled address: {:#x}", address) + // } } fn io_read_u8(port: u16) -> u8 { @@ -123,71 +128,39 @@ impl acpi_system::Handler for AcpiHandlerImpl { } fn mem_read_u8(address: u64) -> u8 { - let value = unsafe { (address as *const u8).virtualize().read_volatile() }; - log::trace!("mem_read_u8 {:#x} <- {:#x}", address, value); - value + todo!() } fn mem_read_u16(address: u64) -> u16 { - let value = if address & 0x1 == 0 { - unsafe { (address as *const u16).virtualize().read_volatile() } - } else { - unsafe { (address as *const u16).virtualize().read_unaligned() } - }; - log::trace!("mem_read_u16 {:#x} <- {:#x}", address, value); - value + todo!() } fn mem_read_u32(address: u64) -> u32 { - let value = if address & 0x3 == 0 { - unsafe { (address as *const u32).virtualize().read_volatile() } - } else { - unsafe { (address as *const u32).virtualize().read_unaligned() } - }; - log::trace!("mem_read_u32 {:#x} <- {:#x}", address, value); - value + todo!() } fn mem_read_u64(address: u64) -> u64 { - let value = if address & 0x7 == 0 { - unsafe { (address as *const u64).virtualize().read_volatile() } - } else { - unsafe { (address as *const u64).virtualize().read_unaligned() } - }; - log::trace!("mem_read_u64 {:#x} <- {:#x}", address, value); - value + todo!() } fn mem_write_u8(address: u64, value: u8) { log::trace!("mem_write_u8 {:#x}, {:#x}", address, value); - unsafe { (address as *mut u8).virtualize().write_volatile(value) }; + todo!() } fn mem_write_u16(address: u64, value: u16) { log::trace!("mem_write_u16 {:#x}, {:#x}", address, value); - if address & 0x1 == 0 { - unsafe { (address as *mut u16).virtualize().write_volatile(value) }; - } else { - unsafe { (address as *mut u16).virtualize().write_unaligned(value) }; - } + todo!() } fn mem_write_u32(address: u64, value: u32) { log::trace!("mem_write_u32 {:#x}, {:#x}", address, value); - if address & 0x3 == 0 { - unsafe { (address as *mut u32).virtualize().write_volatile(value) }; - } else { - unsafe { (address as *mut u32).virtualize().write_unaligned(value) }; - } + todo!() } fn mem_write_u64(address: u64, value: u64) { log::trace!("mem_write_u64 {:#x}, {:#x}", address, value); - if address & 0x7 == 0 { - unsafe { (address as *mut u64).virtualize().write_volatile(value) }; - } else { - unsafe { (address as *mut u64).virtualize().write_unaligned(value) }; - } + todo!() } fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> { @@ -342,17 +315,15 @@ impl AcpiHandler for AcpiHandlerImpl { physical_address: usize, size: usize, ) -> PhysicalMapping { - if physical_address <= 0xFFFFFFFF { - PhysicalMapping::new( - physical_address, - NonNull::new_unchecked(physical_address.virtualize() as *mut T), - size, - size, - *self, - ) - } else { - todo!() - } + PhysicalMapping::new( + physical_address, + NonNull::new_unchecked( + PhysicalAddress::from_raw(physical_address).virtualize_raw() as *mut T + ), + size, + size, + *self, + ) } // Unmap nothing, these addresses are "virtualized" to high address space diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 736f98f5..a028c3f6 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -8,10 +8,19 @@ use device_api::{ }, Device, }; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_structs, + registers::{ReadWrite, WriteOnly}, +}; use crate::{ arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}, - mem::ConvertAddress, + mem::{ + address::FromRaw, + device::{DeviceMemoryIo, DeviceMemoryMapping}, + PhysicalAddress, + }, sync::IrqSafeSpinlock, }; @@ -38,12 +47,18 @@ struct IsaRedirection { trigger: IrqTrigger, } -struct Regs { - base: usize, +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x00 => Index: WriteOnly), + (0x04 => _0), + (0x10 => Data: ReadWrite), + (0x14 => @END), + } } struct Inner { - regs: Regs, + regs: DeviceMemoryIo<'static, Regs>, max_gsi: u32, } @@ -59,22 +74,14 @@ pub struct IoApic { impl Regs { #[inline] fn read(&self, reg: u32) -> u32 { - let ptr = self.base as *mut u32; - - unsafe { - ptr.write_volatile(reg & 0xFF); - ptr.add(4).read_volatile() - } + self.Index.set(reg); + self.Data.get() } #[inline] fn write(&self, reg: u32, value: u32) { - let ptr = self.base as *mut u32; - - unsafe { - ptr.write_volatile(reg & 0xFF); - ptr.add(4).write_volatile(value); - } + self.Index.set(reg); + self.Data.set(value); } } @@ -269,8 +276,12 @@ impl IoApic { } // TODO properly map this using DeviceMemory - let regs = Regs { - base: unsafe { (ioapic.address as usize).virtualize() }, + // let regs = Regs { + // base: unsafe { PhysicalAddress::from_raw(ioapic.address as u64).virtualize_raw() }, + // }; + // let mapping = unsafe { DeviceMemoryMapping::map(base, size) }; + let regs = unsafe { + DeviceMemoryIo::<'_, Regs>::map(PhysicalAddress::from_raw(ioapic.address as u64))? }; let max_gsi = (regs.read(REG_IOAPIC_VERSION) >> 16) & 0xFF; diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 092796fa..0ef9f1f3 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -17,7 +17,7 @@ use crate::{ x86_64::{registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT}, CpuMessage, }, - mem::ConvertAddress, + mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, task::Cpu, }; @@ -130,7 +130,7 @@ register_structs! { /// Per-processor local APIC interface pub struct LocalApic { - regs: &'static Regs, + regs: DeviceMemoryIo<'static, Regs>, } unsafe impl Send for LocalApic {} @@ -190,8 +190,7 @@ impl LocalApic { /// /// Only meant to be called once per processor during their init. pub unsafe fn new() -> Self { - let base = unsafe { Self::base().virtualize() }; - let regs = unsafe { &*(base as *const Regs) }; + let regs = DeviceMemoryIo::::map(Self::base()).unwrap(); let id = regs.Id.read(Id::ApicId); @@ -294,8 +293,8 @@ impl LocalApic { } #[inline] - fn base() -> usize { - MSR_IA32_APIC_BASE.read_base() as usize + fn base() -> PhysicalAddress { + PhysicalAddress::from_raw(MSR_IA32_APIC_BASE.read_base()) } #[inline] diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 9be041ff..284ee3d1 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,5 +1,5 @@ //! x86-64 boot and entry functions -use core::{arch::global_asm, sync::atomic::Ordering}; +use core::arch::global_asm; use tock_registers::interfaces::Writeable; use yboot_proto::{ @@ -8,26 +8,20 @@ use yboot_proto::{ }; use crate::{ - arch::{ - x86_64::{cpuid, exception, registers::MSR_IA32_KERNEL_GS_BASE, BootData}, - Architecture, ArchitectureImpl, ARCHITECTURE, - }, + arch::{x86_64::registers::MSR_IA32_KERNEL_GS_BASE, Architecture, ArchitectureImpl}, fs::devfs, - kernel_main, kernel_secondary_main, - mem::{ - heap, - phys::{self, PageUsage}, - ConvertAddress, KERNEL_VIRT_OFFSET, - }, + kernel_main, + mem::KERNEL_VIRT_OFFSET, task::runtime, }; -use super::smp::CPU_COUNT; +use super::{cpuid::init_cpuid, exception, ARCHITECTURE}; -// use super::ARCHITECTURE; +pub enum BootData { + YBoot(&'static LoadProtocolV1), +} const BOOT_STACK_SIZE: usize = 1024 * 1024; -const HEAP_PAGES: usize = 512; #[repr(C, align(0x20))] struct BootStack { @@ -41,7 +35,7 @@ static mut BSP_STACK: BootStack = BootStack { #[used] #[link_section = ".data.yboot"] -static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { +static YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { header: LoadProtocolHeader { kernel_magic: KERNEL_MAGIC, version: PROTOCOL_VERSION_1, @@ -65,32 +59,80 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { res_size: 0, }, }; +// +// +// unsafe extern "C" fn __x86_64_upper_entry() -> ! { +// } +// +// /// Application processor entry point +// pub extern "C" fn __x86_64_ap_entry() -> ! { +// let cpu_id = CPU_COUNT.load(Ordering::Acquire); +// +// MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); +// unsafe { +// core::arch::asm!("swapgs"); +// } +// MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); +// unsafe { +// core::arch::asm!("swapgs"); +// } +// +// // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base +// cpuid::feature_gate(); +// +// infoln!("cpu{} initializing", cpu_id); +// unsafe { +// ARCHITECTURE.init_mmu(false); +// core::arch::asm!("wbinvd"); +// +// // Cpu::init_local(LocalApic::new(), cpu_id as u32); +// // syscall::init_syscall(); +// exception::init_exceptions(cpu_id); +// +// ARCHITECTURE.init_platform(cpu_id); +// } +// +// CPU_COUNT.fetch_add(1, Ordering::Release); +// +// kernel_secondary_main() +// } -static UNINIT_CPU: usize = 0; - -unsafe extern "C" fn __x86_64_upper_entry() -> ! { - ArchitectureImpl::set_interrupt_mask(true); +unsafe fn init_dummy_cpu() { + // TODO this is incorrect + static UNINIT_CPU_INNER: usize = 0; + static UNINIT_CPU_PTR: &'static usize = &UNINIT_CPU_INNER; // Point %gs to a dummy structure so that Cpu::get_local() works properly even before the CPU // data structure is initialized - MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU_PTR as *const _ as u64); core::arch::asm!("swapgs"); - MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); + MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU_PTR as *const _ as u64); core::arch::asm!("swapgs"); +} - ARCHITECTURE.init_mmu(true); - core::arch::asm!("wbinvd"); +pub extern "C" fn __x86_64_ap_entry() -> ! { + loop {} +} + +extern "C" fn __x86_64_upper_entry() -> ! { + // Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry + unsafe { + init_dummy_cpu(); + } ARCHITECTURE.set_boot_data(BootData::YBoot(&YBOOT_DATA)); - ARCHITECTURE - .init_physical_memory() - .expect("Failed to initialize the physical memory manager"); - let heap_base = phys::alloc_pages_contiguous(HEAP_PAGES, PageUsage::Used) - .expect("Couldn't allocate memory for heap"); - heap::init_heap(heap_base.virtualize(), HEAP_PAGES * 0x1000); + // Gather available CPU features + init_cpuid(); - exception::init_exceptions(0); + // Setup memory management: kernel virtual memory tables, physical page manager and heap + unsafe { + ARCHITECTURE.init_memory_management(); + } + + unsafe { + exception::init_exceptions(0); + } // Initialize async executor queue runtime::init_task_queue(); @@ -98,52 +140,21 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! { devfs::init(); // Initializes: local CPU, platform devices (timers/serials/etc), debug output - ARCHITECTURE.init_platform(0); - - cpuid::feature_gate(); + unsafe { + ARCHITECTURE.init_platform(0); + } kernel_main() } -/// Application processor entry point -pub extern "C" fn __x86_64_ap_entry() -> ! { - let cpu_id = CPU_COUNT.load(Ordering::Acquire); - - MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); - unsafe { - core::arch::asm!("swapgs"); - } - MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); - unsafe { - core::arch::asm!("swapgs"); - } - - // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base - cpuid::feature_gate(); - - infoln!("cpu{} initializing", cpu_id); - unsafe { - ARCHITECTURE.init_mmu(false); - core::arch::asm!("wbinvd"); - - // Cpu::init_local(LocalApic::new(), cpu_id as u32); - // syscall::init_syscall(); - exception::init_exceptions(cpu_id); - - ARCHITECTURE.init_platform(cpu_id); - } - - CPU_COUNT.fetch_add(1, Ordering::Release); - - kernel_secondary_main() -} - global_asm!( r#" +// {boot_data} .global __x86_64_entry .section .text.entry __x86_64_entry: + cli mov ${yboot_loader_magic}, %edi cmp %edi, %eax je 2f @@ -166,6 +177,7 @@ __x86_64_entry: yboot_loader_magic = const LOADER_MAGIC, stack_size = const BOOT_STACK_SIZE, stack_bottom = sym BSP_STACK, + boot_data = sym YBOOT_DATA, entry = sym __x86_64_upper_entry, options(att_syntax) ); diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 6e74efcd..17e59cb1 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -4,10 +4,10 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use crate::{ - arch::x86_64::table::KERNEL_TABLES, + arch::x86_64::mem::KERNEL_TABLES, mem::{ - phys::{self, PageUsage}, - ConvertAddress, + address::{AsPhysicalAddress, IntoRaw}, + phys, PhysicalAddress, }, task::context::TaskContextImpl, }; @@ -65,11 +65,11 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, cr3: usize) { + fn init_common(&mut self, entry: usize, cr3: PhysicalAddress) { self.push(entry); // address for ret // End of common context - self.push(cr3); // %cr3 + self.push(cr3.into_raw()); // %cr3 self.push(0); // %rbp self.push(0); // %fs (TODO) @@ -89,9 +89,9 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 32; - let stack_base = unsafe { - phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() - }; + + let stack_base = + unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() }; let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -100,7 +100,7 @@ impl TaskContextImpl for TaskContext { stack.push(arg); stack.init_common(__x86_64_task_enter_kernel as _, unsafe { - KERNEL_TABLES.physical_address() + KERNEL_TABLES.as_physical_address() }); let sp = stack.build(); @@ -115,10 +115,15 @@ impl TaskContextImpl for TaskContext { }) } - fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result { + fn user( + entry: usize, + arg: usize, + cr3: PhysicalAddress, + user_stack_sp: usize, + ) -> Result { const USER_TASK_PAGES: usize = 8; - let stack_base = - unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; + + let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() }; let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 160d9174..40d88bcf 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -1,9 +1,13 @@ //! x86-64 CPUID interface -use tock_registers::interfaces::ReadWriteable; -use crate::arch::x86_64::registers::CR4; +use bitflags::bitflags; +use kernel_util::util::OneTimeInit; -use super::registers::XCR0; +bitflags! { + pub struct ProcessorFeatures: u64 { + const PDPE1GB = 1 << 0; + } +} unsafe fn cpuid(eax: u32, result: &mut [u32]) { core::arch::asm!( @@ -21,60 +25,19 @@ unsafe fn cpuid(eax: u32, result: &mut [u32]) { ); } -type RequiredBit = (u32, &'static str); +pub static PROCESSOR_FEATURES: OneTimeInit = OneTimeInit::new(); -const EAX1_ECX_REQUIRED_FEATURES: &[RequiredBit] = &[ - (1 << 0, "SSE3"), - (1 << 19, "SSE4.1"), - (1 << 20, "SSE4.2"), - // (1 << 24, "TSC"), - (1 << 26, "XSAVE"), - (1 << 28, "AVX"), -]; -const EAX1_EDX_REQUIRED_FEATURES: &[RequiredBit] = &[ - (1 << 0, "FPU"), - (1 << 3, "PSE"), - (1 << 4, "TSC (%edx)"), - (1 << 5, "MSR"), - (1 << 6, "PAE"), - (1 << 9, "APIC"), - (1 << 13, "PGE"), - (1 << 23, "MMX"), - (1 << 24, "FXSR"), - (1 << 25, "SSE"), - (1 << 26, "SSE2"), -]; - -fn enable_cr4_features() { - // TODO maybe also include FSGSBASE here? - CR4.modify(CR4::OSXSAVE::SET + CR4::OSFXSR::SET + CR4::PGE::SET); -} - -fn enable_xcr0_features() { - XCR0.modify(XCR0::X87::SET + XCR0::SSE::SET + XCR0::AVX::SET); -} - -/// Checks for the features required by the kernel and enables them -pub fn feature_gate() { - // TODO the compiler may have generated instructions from SSE/AVX sets by now, find some way to - // perform this as early as possible +pub fn init_cpuid() { + let mut features = ProcessorFeatures::empty(); let mut data = [0; 3]; + unsafe { - cpuid(1, &mut data); + cpuid(0x80000001, &mut data); } - for (bit, name) in EAX1_ECX_REQUIRED_FEATURES { - if data[2] & bit == 0 { - panic!("Required feature not supported: {}", name); - } - } - for (bit, name) in EAX1_EDX_REQUIRED_FEATURES { - if data[1] & bit == 0 { - panic!("Required feature not supported: {}", name); - } + if data[1] & (1 << 26) != 0 { + features |= ProcessorFeatures::PDPE1GB; } - // Enable the SSE/AVX features - enable_cr4_features(); - enable_xcr0_features(); + PROCESSOR_FEATURES.init(features); } diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs new file mode 100644 index 00000000..e7da3a31 --- /dev/null +++ b/src/arch/x86_64/mem/mod.rs @@ -0,0 +1,318 @@ +use core::{ + alloc::Layout, + ops::{Deref, DerefMut}, +}; + +use abi::error::Error; +use kernel_util::util::OneTimeInit; +use memtables::FixedTables; +use static_assertions::{const_assert_eq, const_assert_ne}; + +pub mod table; + +use crate::{ + arch::x86_64::mem::table::PageAttributes, + mem::{ + address::{FromRaw, IntoRaw, KernelImageObject}, + device::RawDeviceMemoryMapping, + table::EntryLevel, + PhysicalAddress, KERNEL_VIRT_OFFSET, + }, +}; + +use self::table::{PageEntry, PageTable, L0, L1, L2, L3}; + +const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; +const KERNEL_PHYS_BASE: usize = 0x400000; + +// Mapped at compile time +const KERNEL_L0_INDEX: usize = L0::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); + +// Must not be zero, should be at 4MiB +const_assert_ne!(KERNEL_START_L2_INDEX, 0); +// From static mapping +const_assert_eq!(KERNEL_L0_INDEX, 511); +const_assert_eq!(KERNEL_L1_INDEX, 0); + +// Mapped at boot +const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1; +const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; +const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; +const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1; + +const DEVICE_MAPPING_L3_COUNT: usize = 4; + +#[link_section = ".data.tables"] +pub static mut KERNEL_TABLES: KernelImageObject = + unsafe { KernelImageObject::new(FixedTables::zeroed()) }; + +// 2MiB for early mappings +const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK + | (KERNEL_L0_INDEX * L0::SIZE) + | (KERNEL_L1_INDEX * L1::SIZE) + | (EARLY_MAPPING_L2I * L2::SIZE); +static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); +// 1GiB for heap mapping +pub(super) const HEAP_MAPPING_OFFSET: usize = + CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (HEAP_MAPPING_L1I * L1::SIZE); +pub(super) static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); +// 1GiB for device MMIO mapping +const DEVICE_MAPPING_OFFSET: usize = + CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE); +static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); +static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = + [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; +// 512GiB for whole RAM mapping +pub(super) const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE); +pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); +pub(super) static mut RAM_MAPPING_L1: PageTable = PageTable::zeroed(); + +// Global limits +pub(super) const HEAP_SIZE_LIMIT: usize = L1::SIZE; + +// Early mappings +unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { + for l3i in 0..512 { + let mut taken = false; + for i in 0..count { + if EARLY_MAPPING_L3[i + l3i].is_present() { + taken = true; + break; + } + } + + if taken { + continue; + } + + for i in 0..count { + // TODO NX, NC + EARLY_MAPPING_L3[i + l3i] = + PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE); + } + + return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); + } + + loop {} +} + +unsafe fn unmap_early_page(address: usize) { + if address < EARLY_MAPPING_OFFSET || address >= EARLY_MAPPING_OFFSET + L2::SIZE { + loop {} + } + + let l3i = L3::index(address - EARLY_MAPPING_OFFSET); + + assert!(EARLY_MAPPING_L3[l3i].is_present()); + EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; +} + +// Device mappings +unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result { + // TODO don't map pages if already mapped + + 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { + continue 'l0; + } + } + + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + // TODO NX, NC + DEVICE_MAPPING_L3S[l2i][l3i] = + PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE); + } + + return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); + } + loop {} +} + +unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result { + 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { + for j in 0..count { + if DEVICE_MAPPING_L2[i + j].is_present() { + continue 'l0; + } + } + + for j in 0..count { + DEVICE_MAPPING_L2[i + j] = + PageEntry::::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE); + } + + debugln!( + "map l2s: base={:#x}, count={} -> {:#x}", + base, + count, + DEVICE_MAPPING_OFFSET + i * L2::SIZE + ); + return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); + } + loop {} +} + +pub(super) unsafe fn map_device_memory( + base: PhysicalAddress, + size: usize, +) -> Result { + debugln!("Map {}B @ {:#x}", size, base); + let l3_aligned = base.page_align_down::(); + let l3_offset = L3::page_offset(base.into_raw()); + let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE; + + if page_count > 256 { + // Large mapping, use L2 mapping instead + let l2_aligned = base.page_align_down::(); + let l2_offset = L2::page_offset(base.into_raw()); + let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE; + + let base_address = map_device_memory_l2(l2_aligned, page_count)?; + let address = base_address + l2_offset; + + Ok(RawDeviceMemoryMapping { + address, + base_address, + page_count, + page_size: L2::SIZE, + }) + } else { + let page_size = L3::SIZE; + + // Just map the pages directly + let base_address = map_device_memory_l3(l3_aligned, page_count)?; + let address = base_address + l3_offset; + + Ok(RawDeviceMemoryMapping { + address, + base_address, + page_count, + page_size: L3::SIZE, + }) + } +} + +pub(super) unsafe fn unmap_device_memory(map: RawDeviceMemoryMapping) { + loop {} +} + +pub(super) unsafe fn map_heap_block(index: usize, page: PhysicalAddress) { + if L2::page_offset(page.into_raw()) != 0 { + loop {} + } + assert!(index < 512); + + if HEAP_MAPPING_L2[index].is_present() { + loop {} + } + + // TODO NX + HEAP_MAPPING_L2[index] = PageEntry::::block(page, PageAttributes::WRITABLE); +} + +pub struct EarlyMapping<'a, T: ?Sized> { + value: &'a mut T, + page_count: usize, +} + +impl<'a, T: Sized> EarlyMapping<'a, T> { + pub unsafe fn map_slice( + physical: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); + + Ok(EarlyMapping { value, page_count }) + } +} + +impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { + fn drop(&mut self) { + let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); + + for i in 0..self.page_count { + let page = address + i * L3::SIZE; + + unsafe { + unmap_early_page(page); + } + } + } +} + +fn clone_kernel_tables(dst: &mut PageTable) { + unsafe { + dst[KERNEL_L0_INDEX] = PageEntry::from_raw(KERNEL_TABLES.l0.data[KERNEL_L0_INDEX]); + dst[RAM_MAPPING_L0I] = PageEntry::from_raw(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I]); + } +} + +/// Sets up the following memory map: +/// ...: KERNEL_TABLES.l0: +/// * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000 : RAM_MAPPING_L1 +/// * 0xFFFFFF8000000000 .. ... : KERNEL_TABLES.kernel_l1: +/// * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000 : KERNEL_TABLES.kernel_l2 +/// * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000 : --- +/// * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000 : EARLY_MAPPING_L3 +/// * 0xFFFFFF8000400000 .. ... : KERNEL_TABLES.kernel_l3s +/// * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000 : HEAP_MAPPING_L2 +/// * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000 : DEVICE_MAPPING_L2 +/// * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000 : DEVICE_MAPPING_L3S +/// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ... +pub unsafe fn init_fixed_tables() { + let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + let ram_mapping_l1_phys = &RAM_MAPPING_L1 as *const _ as usize - KERNEL_VIRT_OFFSET; + + for i in 0..DEVICE_MAPPING_L3_COUNT { + let device_mapping_l3_phys = PhysicalAddress::from_raw( + &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, + ); + DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE); + } + + KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64) + | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I] = + (heap_mapping_l2_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64) + | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + KERNEL_TABLES.l0.data[RAM_MAPPING_L0I] = + (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + let cr3 = &KERNEL_TABLES.l0 as *const _ as usize - KERNEL_VIRT_OFFSET; + core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); +} diff --git a/src/arch/x86_64/table/mod.rs b/src/arch/x86_64/mem/table.rs similarity index 68% rename from src/arch/x86_64/table/mod.rs rename to src/arch/x86_64/mem/table.rs index 868da138..eebf68f9 100644 --- a/src/arch/x86_64/table/mod.rs +++ b/src/arch/x86_64/mem/table.rs @@ -1,5 +1,3 @@ -//! x86-64 virtual memory management implementation - use core::{ marker::PhantomData, ops::{Index, IndexMut}, @@ -8,21 +6,24 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -mod fixed; - -pub use fixed::{init_fixed_tables, KERNEL_TABLES}; - -use crate::mem::{ - phys::{self, PageUsage}, - table::{ - EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, +use crate::{ + mem::{ + address::{AsPhysicalAddress, FromRaw}, + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + table::{ + EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, + }, + PhysicalAddress, }, - ConvertAddress, + sync::IrqSafeSpinlock, }; +use super::{clone_kernel_tables, KERNEL_TABLES}; + bitflags! { /// Describes how each page table entry is mapped - struct PageAttributes: u64 { + pub struct PageAttributes: u64 { /// When set, the mapping is considered valid and pointing somewhere const PRESENT = 1 << 0; /// For tables, allows writes to further translation levels, for pages/blocks, allows @@ -41,7 +42,7 @@ bitflags! { /// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space. #[repr(C)] pub struct AddressSpace { - l0: *mut PageTable, + inner: IrqSafeSpinlock>>, } /// Represents a single virtual address space mapping depending on its translation level @@ -80,6 +81,8 @@ impl NonTerminalEntryLevel for L2 { } impl const EntryLevel for L0 { + const SIZE: usize = 1 << 39; + fn index(addr: usize) -> usize { (addr >> 39) & 0x1FF } @@ -90,6 +93,8 @@ impl const EntryLevel for L0 { } impl const EntryLevel for L1 { + const SIZE: usize = 1 << 30; + fn index(addr: usize) -> usize { (addr >> 30) & 0x1FF } @@ -100,6 +105,8 @@ impl const EntryLevel for L1 { } impl const EntryLevel for L2 { + const SIZE: usize = 1 << 21; + fn index(addr: usize) -> usize { (addr >> 21) & 0x1FF } @@ -110,6 +117,8 @@ impl const EntryLevel for L2 { } impl const EntryLevel for L3 { + const SIZE: usize = 1 << 12; + fn index(addr: usize) -> usize { (addr >> 12) & 0x1FF } @@ -121,18 +130,18 @@ impl const EntryLevel for L3 { impl PageEntry { /// Constructs a mapping which points to a 4KiB page - fn page(phys: usize, attrs: PageAttributes) -> Self { + pub fn page(phys: PhysicalAddress, attrs: PageAttributes) -> Self { Self( - (phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::USER).bits(), + u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::USER).bits(), PhantomData, ) } /// Returns the physical address of the page this entry refers to, returning None if it does /// not - pub fn as_page(self) -> Option { + pub fn as_page(self) -> Option { if self.0 & PageAttributes::PRESENT.bits() != 0 { - Some((self.0 & !0xFFF) as usize) + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) } else { None } @@ -141,11 +150,18 @@ impl PageEntry { impl PageEntry { /// Constructs a mapping which points to a 2MiB block - fn block(phys: usize, attrs: PageAttributes) -> Self { + pub fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { Self( - (phys as u64) - | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK | PageAttributes::USER) - .bits(), + u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), + PhantomData, + ) + } +} + +impl PageEntry { + pub unsafe fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), PhantomData, ) } @@ -153,9 +169,9 @@ impl PageEntry { impl PageEntry { /// Constructs a mapping which points to a next-level table - fn table(phys: usize, attrs: PageAttributes) -> Self { + pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self { Self( - (phys as u64) + u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::WRITABLE @@ -167,11 +183,11 @@ impl PageEntry { /// Returns the physical address of the table this entry refers to, returning None if it /// does not - pub fn as_table(self) -> Option { + pub fn as_table(self) -> Option { if self.0 & PageAttributes::PRESENT.bits() != 0 && self.0 & PageAttributes::BLOCK.bits() == 0 { - Some((self.0 & !0xFFF) as usize) + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) } else { None } @@ -182,6 +198,10 @@ impl PageEntry { /// An entry that is not mapped pub const INVALID: Self = Self(0, PhantomData); + pub const unsafe fn from_raw(raw: u64) -> Self { + Self(raw, PhantomData) + } + /// Returns `true` if the entry contains a valid mapping to either a table or to a page/block pub fn is_present(&self) -> bool { self.0 & PageAttributes::PRESENT.bits() != 0 @@ -197,41 +217,49 @@ impl PageTable { } /// Allocates a new page table, filling it with non-preset entries - pub fn new_zeroed() -> Result<&'static mut Self, Error> { - let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() }; - let table = unsafe { &mut *(page as *mut Self) }; + pub fn new_zeroed<'a>() -> Result, Error> { + let physical = phys::alloc_page()?; + let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) }; + for i in 0..512 { table[i] = PageEntry::INVALID; } + Ok(table) } - /// Returns the physical address of this table - pub fn physical_address(&self) -> usize { - unsafe { (self.data.as_ptr() as usize).physicalize() } - } + // /// Returns the physical address of this table + // pub fn physical_address(&self) -> usize { + // unsafe { (self.data.as_ptr() as usize).physicalize() } + // } } -impl NextPageTable for PageTable { +impl NextPageTable for PageTable { type NextLevel = PageTable; + type TableRef = PhysicalRef<'static, Self::NextLevel>; + type TableRefMut = PhysicalRefMut<'static, Self::NextLevel>; - fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> { - let entry = self[index]; - - entry + fn get(&self, index: usize) -> Option { + self[index] .as_table() - .map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) }) + .map(|addr| unsafe { PhysicalRef::map(addr) }) } - fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> { + fn get_mut(&mut self, index: usize) -> Option { + self[index] + .as_table() + .map(|addr| unsafe { PhysicalRefMut::map(addr) }) + } + + fn get_mut_or_alloc(&mut self, index: usize) -> Result { let entry = self[index]; if let Some(table) = entry.as_table() { - Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) }) + Ok(unsafe { PhysicalRefMut::map(table) }) } else { let table = PageTable::new_zeroed()?; self[index] = PageEntry::::table( - table.physical_address(), + unsafe { table.as_physical_address() }, PageAttributes::WRITABLE | PageAttributes::USER, ); Ok(table) @@ -285,7 +313,7 @@ impl VirtualMemoryManager for AddressSpace { } for i in 0..len { - let page = phys::alloc_page(PageUsage::Used)?; + let page = phys::alloc_page()?; self.map_page(base + i * 0x1000, page, attrs)?; } @@ -295,7 +323,12 @@ impl VirtualMemoryManager for AddressSpace { Err(Error::OutOfMemory) } - fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> { + fn map_page( + &self, + virt: usize, + phys: PhysicalAddress, + attrs: MapAttributes, + ) -> Result<(), Error> { self.write_entry(virt, PageEntry::page(phys, attrs.into()), true) } @@ -318,49 +351,50 @@ impl VirtualMemoryManager for AddressSpace { impl AddressSpace { /// Allocates an empty address space with all entries marked as non-present pub fn new_empty() -> Result { - let l0 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable }; + let mut l0 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; for i in 0..512 { unsafe { - (*l0)[i] = PageEntry::INVALID; + l0[i] = PageEntry::INVALID; } } - unsafe { - KERNEL_TABLES.clone_into(&mut *l0); - } - Ok(Self { l0 }) - } + clone_kernel_tables(&mut *l0); - unsafe fn as_mut(&self) -> &'static mut PageTable { - self.l0.as_mut().unwrap() + Ok(Self { + inner: IrqSafeSpinlock::new(l0), + }) } // TODO return page size and attributes /// Returns the physical address to which the `virt` address is mapped - pub fn translate(&self, virt: usize) -> Option { + pub fn translate(&self, virt: usize) -> Option { + let l0 = self.inner.lock(); + let l0i = L0::index(virt); let l1i = L1::index(virt); let l2i = L2::index(virt); let l3i = L3::index(virt); - let l1 = unsafe { self.as_mut().get_mut(l0i) }?; - let l2 = l1.get_mut(l1i)?; - let l3 = l2.get_mut(l2i)?; + let l1 = l0.get(l0i)?; + let l2 = l1.get(l1i)?; + let l3 = l2.get(l2i)?; l3[l3i].as_page() } // Write a single 4KiB entry fn write_entry(&self, virt: usize, entry: PageEntry, overwrite: bool) -> Result<(), Error> { + let mut l0 = self.inner.lock(); + let l0i = L0::index(virt); let l1i = L1::index(virt); let l2i = L2::index(virt); let l3i = L3::index(virt); - let l1 = unsafe { self.as_mut().get_mut_or_alloc(l0i) }?; - let l2 = l1.get_mut_or_alloc(l1i)?; - let l3 = l2.get_mut_or_alloc(l2i)?; + let mut l1 = l0.get_mut_or_alloc(l0i)?; + let mut l2 = l1.get_mut_or_alloc(l1i)?; + let mut l3 = l2.get_mut_or_alloc(l2i)?; if l3[l3i].is_present() && !overwrite { todo!(); @@ -376,10 +410,7 @@ impl AddressSpace { } /// Returns the physical address of the root table - pub fn physical_address(&self) -> usize { - unsafe { (self.l0 as usize).physicalize() } + pub fn physical_address(&self) -> PhysicalAddress { + unsafe { self.inner.lock().as_physical_address() } } } - -unsafe impl Send for AddressSpace {} -unsafe impl Sync for AddressSpace {} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index b6621f82..2e10cdfa 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,176 +1,113 @@ -//! x86-64 architecture and platform implementation -use core::sync::atomic::Ordering; +use core::{mem::size_of, sync::atomic::Ordering}; use abi::error::Error; -use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; +use acpi_lib::{AcpiHandler, AcpiTable, AcpiTables, InterruptModel}; use alloc::boxed::Box; -use cpu::Cpu; use device_api::{ - input::KeyboardProducer, - interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, - timer::MonotonicTimestampProviderDevice, - Device, + input::KeyboardProducer, interrupt::ExternalInterruptController, + timer::MonotonicTimestampProviderDevice, Device, }; use git_version::git_version; use kernel_util::util::OneTimeInit; -use yboot_proto::{AvailableRegion, IterableMemoryMap}; +use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; + +mod acpi; +mod apic; +mod boot; +pub mod context; +pub mod cpu; +mod cpuid; +mod exception; +mod gdt; +mod intrinsics; +pub mod mem; +mod peripherals; +mod registers; +mod smp; +mod syscall; use crate::{ arch::x86_64::{ - apic::local::LocalApic, - peripherals::serial::ComPort, - table::{init_fixed_tables, KERNEL_TABLES}, + intrinsics::{IoPort, IoPortAccess}, + mem::{map_heap_block, table::L2, HEAP_MAPPING_OFFSET}, }, debug::{self, LogLevel}, device::{ self, - bus::pci::PciBusManager, display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, - tty::combined::CombinedTerminal, + tty::CombinedTerminal, }, fs::{ devfs::{self, CharDeviceType}, Initrd, INITRD_DATA, }, mem::{ + address::{FromRaw, IntoRaw}, + device::RawDeviceMemoryMapping, + heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, - ConvertAddress, + table::EntryLevel, + PhysicalAddress, }, - panic, sync::SpinFence, - CPU_INIT_FENCE, }; use self::{ acpi::{AcpiAllocator, AcpiHandlerImpl}, - apic::ioapic::IoApic, - intrinsics::{IoPort, IoPortAccess}, - peripherals::{i8253::I8253, ps2::PS2Controller}, + apic::{ioapic::IoApic, local::LocalApic}, + boot::BootData, + cpu::Cpu, + cpuid::{ProcessorFeatures, PROCESSOR_FEATURES}, + mem::{ + init_fixed_tables, + table::{PageAttributes, PageEntry, L1, L3}, + EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, + }, + peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort}, smp::CPU_COUNT, }; use super::{Architecture, CpuMessage}; -#[macro_use] -pub mod intrinsics; - -pub mod acpi; -pub mod apic; -pub mod boot; -pub mod context; -pub mod cpu; -pub mod cpuid; -pub mod exception; -pub mod gdt; -pub mod peripherals; -pub mod registers; -pub mod smp; -pub mod syscall; -pub mod table; - -/// x86-64 interrupt number wrapper -#[derive(Clone, Copy, PartialEq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum IrqNumber { - /// Legacy (ISA) interrupt. Can have value in range 0..16. Isa(u8), - /// Global system interrupt. Means an external interrupt for I/O APIC. Gsi(u8), } -/// Helper trait to provide abstract access to available memory regions -pub trait AbstractAvailableRegion { - /// Returns page-aligned physical start address of the region - fn start_address(&self) -> usize; - /// Returns the page count (rounded down) of this region - fn page_count(&self) -> usize; -} - -/// Helper trait to provide abstract access to memory maps -pub trait AbstractMemoryMap<'a>: 'a { - /// Available memory region type contained within this memory map - type AvailableRegion: AbstractAvailableRegion; - /// Iterator type returned by [Self::iter] - type Iter: Iterator + Clone; - - /// Returns the physical memory range which contains this memory map - fn reserved_range(&self) -> PhysicalMemoryRegion; - /// Returns an iterator over the available memory regions - fn iter(&self) -> Self::Iter; -} - -impl AbstractAvailableRegion for T { - fn start_address(&self) -> usize { - ::start_address(self) - } - - fn page_count(&self) -> usize { - ::page_count(self) - } -} - -impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T { - type AvailableRegion = T::Entry; - type Iter = T::Iter; - - fn reserved_range(&self) -> PhysicalMemoryRegion { - PhysicalMemoryRegion { - base: self.data_physical_base(), - size: (self.data_size() + 0xFFF) & !0xFFF, - } - } - - fn iter(&self) -> Self::Iter { - ::iter_with_offset(self, X86_64::KERNEL_VIRT_OFFSET) - } -} - -/// Describes which kind of bootloader data was provided to the kernel -pub enum BootData { - /// [yboot_proto::LoadProtocolV1] - YBoot(&'static yboot_proto::LoadProtocolV1), -} - -/// x86-64 architecture + platform implementation pub struct X86_64 { boot_data: OneTimeInit, acpi: OneTimeInit>, + // Display framebuffer: OneTimeInit, - fb_console: OneTimeInit, - combined_terminal: OneTimeInit, + fbconsole: OneTimeInit, + tty: OneTimeInit, ioapic: OneTimeInit, timer: OneTimeInit, } -/// x86-64 architecture implementation +static SHUTDOWN_FENCE: SpinFence = SpinFence::new(); + pub static ARCHITECTURE: X86_64 = X86_64 { boot_data: OneTimeInit::new(), acpi: OneTimeInit::new(), framebuffer: OneTimeInit::new(), - fb_console: OneTimeInit::new(), - combined_terminal: OneTimeInit::new(), + fbconsole: OneTimeInit::new(), + tty: OneTimeInit::new(), - // Devices ioapic: OneTimeInit::new(), timer: OneTimeInit::new(), }; -static SHUTDOWN_FENCE: SpinFence = SpinFence::new(); - impl Architecture for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; - type IrqNumber = IrqNumber; - unsafe fn init_mmu(&self, bsp: bool) { - if bsp { - init_fixed_tables(); - } - - let cr3 = KERNEL_TABLES.physical_address(); - core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); + fn cpu_count() -> usize { + CPU_COUNT.load(Ordering::Acquire) } unsafe fn set_interrupt_mask(mask: bool) { @@ -196,91 +133,76 @@ impl Architecture for X86_64 { } } - // CPU management - unsafe fn reset(&self) -> ! { - Self::set_interrupt_mask(true); - loop { - Self::wait_for_interrupt(); + #[inline] + unsafe fn map_device_memory( + &self, + base: PhysicalAddress, + size: usize, + ) -> Result { + mem::map_device_memory(base, size) + } + + #[inline] + unsafe fn unmap_device_memory(&self, map: RawDeviceMemoryMapping) { + mem::unmap_device_memory(map) + } + + fn map_physical_memory + Clone>( + &self, + it: I, + _memory_start: PhysicalAddress, + memory_end: PhysicalAddress, + ) -> Result<(), Error> { + let end_l1i = (IntoRaw::::into_raw(memory_end) + (1 << 30) - 1) >> 30; + + if end_l1i > 512 { + loop {} + } + + MEMORY_LIMIT.init(memory_end.into_raw()); + + // Check if 1GiB pages are supported + if PROCESSOR_FEATURES + .get() + .contains(ProcessorFeatures::PDPE1GB) + { + // Just map gigabytes of RAM + for l1i in 0..end_l1i { + // TODO NX + unsafe { + RAM_MAPPING_L1[l1i] = PageEntry::::block( + PhysicalAddress::from_raw(l1i << 30), + PageAttributes::WRITABLE, + ); + } + } + + Ok(()) + } else { + loop {} } } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { - if !CPU_INIT_FENCE.try_wait_all(1) { - // Don't send an IPI: SMP not initialized yet - return Ok(()); - } - - let Some(local_apic) = Cpu::get_local().map(|cpu| cpu.local_apic()) else { - panic!("Local APIC has not been initialized yet"); - }; - - local_apic.send_ipi(target, msg) - } - - unsafe fn start_application_processors(&self) { - if let Some(acpi) = self.acpi.try_get() { - let Some(pinfo) = acpi - .platform_info_in(AcpiAllocator) - .ok() - .and_then(|p| p.processor_info) - else { - return; - }; - - smp::start_ap_cores(&pinfo); + #[inline] + fn virtualize(address: PhysicalAddress) -> Result { + let raw: usize = address.into_raw(); + if raw < *mem::MEMORY_LIMIT.get() { + Ok(raw + RAM_MAPPING_OFFSET) + } else { + errorln!("Invalid physical address: {:#x}", address); + Err(Error::InvalidMemoryOperation) } } - fn cpu_count() -> usize { - CPU_COUNT.load(Ordering::Acquire) - } + #[inline] + fn physicalize(address: usize) -> Result { + if address < RAM_MAPPING_OFFSET || address - RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() + { + errorln!("Not a virtualized physical address: {:#x}", address); + return Err(Error::InvalidMemoryOperation); + } - // Memory - fn map_device_pages(&self, phys: usize, count: usize) -> Result { - unsafe { KERNEL_TABLES.map_device_pages(phys, count) } - } - - // Devices - fn register_monotonic_timer( - &self, - _timer: &'static dyn device_api::timer::MonotonicTimestampProviderDevice, - ) -> Result<(), Error> { - todo!() - } - - fn register_local_interrupt_controller( - &self, - _intc: &'static dyn device_api::interrupt::LocalInterruptController< - IpiMessage = CpuMessage, - >, - ) -> Result<(), Error> { - todo!() - } - - fn register_external_interrupt_controller( - &self, - _intc: &'static dyn device_api::interrupt::ExternalInterruptController< - IrqNumber = Self::IrqNumber, - >, - ) -> Result<(), Error> { - todo!() - } - - fn register_reset_device( - &self, - _reset: &'static dyn device_api::ResetDevice, - ) -> Result<(), Error> { - todo!() - } - - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { - self.timer.get() - } - - fn local_interrupt_controller( - &'static self, - ) -> &'static dyn LocalInterruptController { - todo!() + Ok(PhysicalAddress::from_raw(address - RAM_MAPPING_OFFSET)) } fn external_interrupt_controller( @@ -288,35 +210,220 @@ impl Architecture for X86_64 { ) -> &'static dyn ExternalInterruptController { self.ioapic.get() } + + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { + self.timer.get() + } } impl X86_64 { - fn set_boot_data(&self, boot_data: BootData) { - match &boot_data { + unsafe fn handle_ipi(&self, msg: CpuMessage) { + loop {} + } + + fn set_boot_data(&self, data: BootData) { + match data { BootData::YBoot(data) => { - // Setup initrd - Self::init_initrd( - data.initrd_address as usize, - (data.initrd_address + data.initrd_size) as usize, + // Reserve the memory map + unsafe { + reserve_region( + "mmap", + PhysicalMemoryRegion { + base: PhysicalAddress::from_raw(data.memory_map.address), + size: data.memory_map.len as usize * size_of::(), + }, + ); + } + + // Reserve initrd, if not NULL + if data.initrd_address != 0 && data.initrd_size != 0 { + let aligned_start = data.initrd_address & !0xFFF; + let aligned_end = (data.initrd_address + data.initrd_size + 0xFFF) & !0xFFF; + + unsafe { + reserve_region( + "initrd", + PhysicalMemoryRegion { + base: PhysicalAddress::from_raw(aligned_start), + size: (aligned_end - aligned_start) as usize, + }, + ); + } + } + } + } + + self.boot_data.init(data); + } + + unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) { + let mmap = EarlyMapping::::map_slice( + PhysicalAddress::from_raw(data.memory_map.address), + data.memory_map.len as usize, + ) + .unwrap(); + + phys::init_from_iter(mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion { + base: PhysicalAddress::from_raw(reg.start_address), + size: reg.page_count as usize * 0x1000, + })); + } + + unsafe fn init_memory_management(&self) { + const HEAP_PAGES: usize = 16; + + init_fixed_tables(); + + // Reserve lower 4MiB just in case + reserve_region( + "lowmem", + PhysicalMemoryRegion { + base: PhysicalAddress::ZERO, + size: 4 * 1024 * 1024, + }, + ); + + match self.boot_data.get() { + &BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data), + } + + // Setup heap + for i in 0..HEAP_PAGES { + // Allocate in 2MiB chunks + let l2_page = phys::alloc_2m_page().unwrap(); + + map_heap_block(i, l2_page); + } + + heap::init_heap(HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE); + } + + unsafe fn init_platform(&'static self, cpu_id: usize) { + Cpu::init_local(LocalApic::new(), cpu_id as _); + + if cpu_id == 0 { + match self.boot_data.get() { + &BootData::YBoot(data) => { + let start = PhysicalAddress::from_raw(data.initrd_address); + Self::init_initrd(start, start.add(data.initrd_size as usize)); + } + } + + self.init_acpi_from_boot_data(); + + Self::disable_8259(); + + self.timer.init(I8253::new()); + + let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)))); + debug::add_sink(com1_3.port_a(), LogLevel::Debug); + + self.init_framebuffer(); + + debug::init(); + + let ps2 = Box::leak(Box::new(PS2Controller::new( + IrqNumber::Isa(1), + IrqNumber::Isa(12), + 0x64, + 0x60, + ))); + ps2.init().unwrap(); + + infoln!( + "Yggdrasil v{} ({})", + env!("CARGO_PKG_VERSION"), + git_version!() + ); + + if let Some(acpi) = self.acpi.try_get() { + self.init_platform_from_acpi(acpi); + } + + self.timer.get().init_irq().unwrap(); + ps2.connect(self.tty.get()); + ps2.init_irq().unwrap(); + + device::register_device(self.ioapic.get()); + device::register_device(ps2); + + // TODO setup PCI devices + } else { + loop {} + } + } + + unsafe fn init_acpi_from_boot_data(&self) { + match self.boot_data.get() { + &BootData::YBoot(data) => self.init_acpi_from_rsdp(data.rsdp_address as usize), + } + } + + unsafe fn init_acpi_from_rsdp(&self, rsdp: usize) { + let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap(); + self.acpi.init(acpi_tables); + } + + unsafe fn init_platform_from_acpi(&self, acpi: &'static AcpiTables) { + let platform_info = acpi.platform_info_in(AcpiAllocator).unwrap(); + + let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { + panic!("The processor does not support APIC"); + }; + + self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap()); + + // TODO ACPI init + // acpi::init_acpi(acpi).unwrap(); + + // TODO MCFG + // if let Ok(mcfg) = acpi.find_table::() { + // for entry in mcfg.entries() { + // PciBusManager::add_segment_from_mcfg(entry).unwrap(); + // } + // } + } + + unsafe fn init_framebuffer(&'static self) { + match self.boot_data.get() { + &BootData::YBoot(data) => { + let info = &data.opt_framebuffer; + + self.framebuffer.init( + LinearFramebuffer::from_physical_bits( + PhysicalAddress::from_raw(info.res_address), + info.res_size as usize, + info.res_stride as usize, + info.res_width, + info.res_height, + ) + .unwrap(), ); } } - self.boot_data.init(boot_data); + self.fbconsole + .init(FramebufferConsole::from_framebuffer(self.framebuffer.get(), None).unwrap()); + debug::add_sink(self.fbconsole.get(), LogLevel::Info); + + self.tty.init(CombinedTerminal::new(self.fbconsole.get())); + + devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular).unwrap(); + console::add_console_autoflush(self.fbconsole.get()); } - fn init_initrd(initrd_start: usize, initrd_end: usize) { - if initrd_start == 0 || initrd_end <= initrd_start { + fn init_initrd(initrd_start: PhysicalAddress, initrd_end: PhysicalAddress) { + if initrd_start.is_zero() || initrd_end <= initrd_start { infoln!("No initrd loaded"); return; } - let start_aligned = initrd_start & !0xFFF; - let end_aligned = initrd_end & !0xFFF; + let start_aligned = initrd_start.page_align_down::(); + let end_aligned = initrd_start.page_align_up::(); let data = unsafe { core::slice::from_raw_parts( - initrd_start.virtualize() as *const _, + start_aligned.virtualize_raw() as *const u8, initrd_end - initrd_start, ) }; @@ -330,167 +437,6 @@ impl X86_64 { INITRD_DATA.init(initrd); } - fn init_acpi_from_boot_data(&self) { - match *self.boot_data.get() { - BootData::YBoot(data) => { - self.init_acpi_from_rsdp(data.rsdp_address as usize); - } - } - } - - fn init_acpi_from_rsdp(&self, address: usize) { - let acpi_tables = unsafe { AcpiTables::from_rsdp(AcpiHandlerImpl, address).unwrap() }; - self.acpi.init(acpi_tables); - } - - unsafe fn init_physical_memory(&self) -> Result<(), Error> { - // Reserve the lower 8MiB of memory - reserve_region( - "lower-memory", - PhysicalMemoryRegion { - base: 0, - size: 8 << 21, - }, - ); - - // Reserve initrd - if let Some(initrd) = INITRD_DATA.try_get() { - reserve_region( - "initrd", - PhysicalMemoryRegion { - base: initrd.phys_page_start, - size: initrd.phys_page_len, - }, - ); - } - - match *self.boot_data.get() { - BootData::YBoot(data) => { - let memory_map = &data.memory_map; - - reserve_region("memory-map", memory_map.reserved_range()); - - phys::init_from_iter(IterableMemoryMap::iter(memory_map).map(|r| { - PhysicalMemoryRegion { - base: AbstractAvailableRegion::start_address(r), - size: AbstractAvailableRegion::page_count(r) * 0x1000, - } - })) - .expect("Failed to initialize the physical memory manager"); - } - } - - Ok(()) - } - - unsafe fn init_platform_from_acpi(&self, acpi: &'static AcpiTables) { - let platform_info = acpi.platform_info_in(AcpiAllocator).unwrap(); - - let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { - panic!("Processor does not have an APIC"); - }; - - self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap()); - - acpi::init_acpi(acpi).unwrap(); - - // Enumerate PCIe root devices - // TODO can there be multiple MCFGs? - if let Ok(mcfg) = acpi.find_table::() { - for entry in mcfg.entries() { - PciBusManager::add_segment_from_mcfg(entry).unwrap(); - } - } - } - - unsafe fn init_framebuffer(&'static self) { - match *self.boot_data.get() { - BootData::YBoot(data) => { - let fb = &data.opt_framebuffer; - self.framebuffer.init( - LinearFramebuffer::from_physical_bits( - fb.res_address as _, - fb.res_size as _, - fb.res_stride as _, - fb.res_width as _, - fb.res_height as _, - ) - .unwrap(), - ); - } - } - - self.fb_console - .init(FramebufferConsole::from_framebuffer(self.framebuffer.get(), None).unwrap()); - - debug::add_sink(self.fb_console.get(), LogLevel::Info); - - // Add a terminal to the devfs - // TODO this is ugly - let combined_terminal = CombinedTerminal::new(self.fb_console.get()); - self.combined_terminal.init(combined_terminal); - - devfs::add_char_device(self.combined_terminal.get(), CharDeviceType::TtyRegular).unwrap(); - console::add_console_autoflush(self.fb_console.get()); - } - - unsafe fn init_platform(&'static self, cpu_id: usize) { - Cpu::init_local(LocalApic::new(), cpu_id as _); - - if cpu_id == 0 { - self.init_acpi_from_boot_data(); - - Self::disable_8259(); - - self.timer.init(I8253::new()); - - // Initialize debug output as soon as possible - let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)))); - debug::add_sink(com1_3.port_a(), LogLevel::Debug); - // devfs::add_char_device(com1_3.port_a(), CharDeviceType::TtySerial).unwrap(); - - self.init_framebuffer(); - debug::init(); - - let ps2 = Box::leak(Box::new(PS2Controller::new( - IrqNumber::Isa(1), - IrqNumber::Isa(12), - 0x64, - 0x60, - ))); - ps2.init().unwrap(); - - // Print some stuff now that the output is initialized - infoln!( - "Yggdrasil v{} ({})", - env!("CARGO_PKG_VERSION"), - git_version!() - ); - infoln!("Initializing x86_64 platform"); - - if let Some(acpi) = self.acpi.try_get() { - self.init_platform_from_acpi(acpi); - } - - // Enable IRQs for the devices - self.timer.get().init_irq().unwrap(); - ps2.connect(self.combined_terminal.get()); - ps2.init_irq().unwrap(); - - device::register_device(self.ioapic.get()); - // device::register_device(self.timer.get()); - device::register_device(ps2); - - // Initialize devices from PCI bus - PciBusManager::setup_bus_devices().unwrap(); - - infoln!("Device list:"); - for device in device::manager_lock().devices() { - infoln!("* {}", device.display_name()); - } - } - } - unsafe fn disable_8259() { infoln!("Disabling i8259 PIC"); // TODO should I make a module for 8259 if I don't even use it? @@ -512,21 +458,4 @@ impl X86_64 { pic_master_cmd.write(0x20); pic_slave_cmd.write(0x20); } - - unsafe fn handle_ipi(&self, msg: CpuMessage) { - match msg { - CpuMessage::Panic => panic::panic_secondary(), - CpuMessage::Shutdown => { - Self::set_interrupt_mask(true); - - let id = Cpu::local_id(); - infoln!("cpu{} shutdown", id); - SHUTDOWN_FENCE.signal(); - - loop { - Self::wait_for_interrupt(); - } - } - } - } } diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index 8735c8f6..b2820842 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -7,15 +7,18 @@ use core::{ use acpi_lib::platform::{ProcessorInfo, ProcessorState}; use crate::{ - arch::{x86_64::boot::__x86_64_ap_entry, Architecture, ArchitectureImpl}, + arch::{ + x86_64::{boot::__x86_64_ap_entry, mem::KERNEL_TABLES}, + Architecture, ArchitectureImpl, + }, mem::{ - phys::{self, PageUsage}, - ConvertAddress, + address::{AsPhysicalAddress, IntoRaw}, + phys, }, task::Cpu, }; -use super::{acpi::AcpiAllocator, table::KERNEL_TABLES}; +use super::acpi::AcpiAllocator; /// The number of CPUs present in the system pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); @@ -46,10 +49,11 @@ unsafe fn load_ap_bootstrap_code() { "Invalid bootstrap code placement: is not below 1MiB" ); - let src_slice = core::slice::from_raw_parts(src_ptr, size); - let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); + todo!(); + // let src_slice = core::slice::from_raw_parts(src_ptr, size); + // let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); - dst_slice.copy_from_slice(src_slice); + // dst_slice.copy_from_slice(src_slice); } unsafe fn load_ap_bootstrap_data(src: &ApBootstrapData) { @@ -62,11 +66,12 @@ unsafe fn load_ap_bootstrap_data(src: &ApBootstrapData) { "Invalid bootstrap data placement: is not below 1MiB" ); - let src_slice = core::slice::from_raw_parts(src_ptr, size); - let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); + todo!() + // let src_slice = core::slice::from_raw_parts(src_ptr, size); + // let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); - dst_slice.copy_from_slice(src_slice); - core::arch::asm!("wbinvd"); + // dst_slice.copy_from_slice(src_slice); + // core::arch::asm!("wbinvd"); } unsafe fn start_ap_core(apic_id: u32) { @@ -75,10 +80,10 @@ unsafe fn start_ap_core(apic_id: u32) { let bsp_cpu = Cpu::local(); let bsp_apic = bsp_cpu.local_apic(); - let cr3 = KERNEL_TABLES.physical_address(); - let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used) + let cr3 = KERNEL_TABLES.as_physical_address().into_raw(); + let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES) .unwrap() - .virtualize(); + .virtualize_raw(); let stack_size = AP_STACK_PAGES * 0x1000; let data = ApBootstrapData { diff --git a/src/arch/x86_64/table/fixed.rs b/src/arch/x86_64/table/fixed.rs deleted file mode 100644 index ed6cbc7a..00000000 --- a/src/arch/x86_64/table/fixed.rs +++ /dev/null @@ -1,161 +0,0 @@ -use abi::error::Error; - -use crate::{ - arch::x86_64::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3}, - mem::KERNEL_VIRT_OFFSET, -}; - -// Means 4 lower GiB are mapped -const KERNEL_PD_COUNT: usize = 4; -// Leave 1GiB gap just for fool safety -const DEVICE_MAPPING_L1I: usize = KERNEL_PD_COUNT + 1; -const DEVICE_VIRT_OFFSET: usize = (DEVICE_MAPPING_L1I << 30) + KERNEL_VIRT_OFFSET; - -/// Fixed tables for x86-64. Provide device mappings and static kernel mapping. -pub struct FixedTables { - // Common - l0: PageTable, - l1: PageTable, - - // Kernel mapping - kernel_l2: [PageTable; KERNEL_PD_COUNT], - // Device mapping - // 511 entries - device_l2: PageTable, - // 512 entries - device_l3: PageTable, - - device_l2i: usize, - device_l3i: usize, -} - -impl FixedTables { - /// Constructs a set of empty translation tables - pub const fn zeroed() -> Self { - Self { - // Global - l0: PageTable::zeroed(), - - // Higher-half common - l1: PageTable::zeroed(), - - // Kernel - kernel_l2: [PageTable::zeroed(); KERNEL_PD_COUNT], - - // Device - device_l2: PageTable::zeroed(), - device_l3: PageTable::zeroed(), - - device_l2i: 1, - device_l3i: 0, - } - } - - /// Maps a specified count of physical memory pages to the device virtual address space - pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result { - if count > 512 * 512 { - panic!("Unsupported device memory mapping size"); - } else if count > 512 { - let count = (count + 511) / 512; - // 2MiB mappings - if self.device_l2i + count > 512 { - return Err(Error::OutOfMemory); - } - - let virt = DEVICE_VIRT_OFFSET + (self.device_l2i << 21); - for i in 0..count { - self.device_l2[self.device_l2i + i] = - PageEntry::block(phys, PageAttributes::WRITABLE); - } - self.device_l2i += count; - - Ok(virt) - } else { - // 4KiB mappings - // Check if a mapping to that address already exists - if self.device_l3i >= count { - for i in 0..self.device_l3i { - let mut matches = true; - - for j in 0..count { - let page = phys + j * 0x1000; - let existing = self.device_l3[i].as_page().unwrap(); - - if page != existing { - matches = false; - break; - } - } - - if matches { - let virt = DEVICE_VIRT_OFFSET + (i << 12); - return Ok(virt); - } - } - } - - if self.device_l3i + count > 512 { - return Err(Error::OutOfMemory); - } - - let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12); - for i in 0..count { - self.device_l3[self.device_l3i + i] = - PageEntry::page(phys + i * 0x1000, PageAttributes::WRITABLE); - } - self.device_l3i += count; - - Ok(virt) - } - } - - /// Returns the physical address of the fixed PML4 - pub fn physical_address(&self) -> usize { - self.l0.physical_address() - } - - pub fn clone_into(&self, target: &mut PageTable) { - target[511] = self.l0[511]; - } -} - -/// Instance of fixed translation tables -pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed(); - -/// Initializes the fixed translation tables. -/// -/// # Safety -/// -/// Only meant to be called by BSP during early init. -pub unsafe fn init_fixed_tables() { - // Kernel L2 - for i in 0..512 * KERNEL_PD_COUNT { - let table_index = i / 512; - let table_offset = i % 512; - - KERNEL_TABLES.kernel_l2[table_index][table_offset] = - PageEntry::block(i << 21, PageAttributes::WRITABLE); - } - - // Device L2 - let addr = KERNEL_TABLES.device_l3.physical_address(); - KERNEL_TABLES.device_l2[0] = PageEntry::table(addr, PageAttributes::empty()); - - // Higher-half L1 - // Map kernel nGiB - for i in 0..KERNEL_PD_COUNT { - let addr = KERNEL_TABLES.kernel_l2[i].physical_address(); - KERNEL_TABLES.l1[i] = PageEntry::table(addr, PageAttributes::empty()); - } - - // Map device tables - let addr = KERNEL_TABLES.device_l2.physical_address(); - KERNEL_TABLES.l1[DEVICE_MAPPING_L1I] = PageEntry::table(addr, PageAttributes::empty()); - - // Global L0 - let addr = KERNEL_TABLES.l1.physical_address(); - - // Keep the lower mapping for AP bootstrapping - KERNEL_TABLES.l0[0] = PageEntry::table(addr, PageAttributes::empty()); - KERNEL_TABLES.l0[511] = PageEntry::table(addr, PageAttributes::empty()); -} diff --git a/src/debug.rs b/src/debug.rs index e691d4d2..8f297b1b 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -52,7 +52,7 @@ macro_rules! log_print_raw { macro_rules! log_print { ($level:expr, $($args:tt)+) => { - log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::task::Cpu::local_id(), file!(), line!(), format_args!($($args)+)) + log_print_raw!($level, "cpu{}:{}:{}: {}", /* $crate::task::Cpu::local_id() */ 0, file!(), line!(), format_args!($($args)+)) }; } diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 1b8927ad..ef5e57a2 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -1,20 +1,13 @@ //! Console device interfaces -use core::{mem::size_of, time::Duration}; + +use core::time::Duration; use abi::{error::Error, primitive_enum}; -use alloc::vec::Vec; +use alloc::{boxed::Box, vec, vec::Vec}; use bitflags::bitflags; use kernel_util::util::StaticVector; -use crate::{ - debug::DebugSink, - mem::{ - phys::{self, PageUsage}, - ConvertAddress, - }, - sync::IrqSafeSpinlock, - task::runtime, -}; +use crate::{debug::DebugSink, sync::IrqSafeSpinlock, task::runtime}; const CONSOLE_ROW_LEN: usize = 80; const MAX_CSI_ARGS: usize = 8; @@ -93,7 +86,7 @@ pub struct ConsoleRow { /// Buffer that contains text rows of the console with their attributes + tracks dirty rows which /// need to be flushed to the display pub struct ConsoleBuffer { - rows: &'static mut [ConsoleRow], + rows: Vec, height: u32, } @@ -253,48 +246,27 @@ impl ConsoleRow { impl ConsoleBuffer { /// Constructs a fixed-size console buffer pub fn new(height: u32) -> Result { - let size = size_of::() * (height as usize); - let page_count = (size + 0xFFF) / 0x1000; - let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?; - - let rows = unsafe { - core::slice::from_raw_parts_mut(pages.virtualize() as *mut ConsoleRow, height as usize) - }; + // let size = size_of::() * (height as usize); + let mut rows = vec![ConsoleRow::zeroed(); height as usize]; for row in rows.iter_mut() { row.clear(DEFAULT_BG_COLOR); } Ok(Self { rows, height }) - } + // let size = size_of::() * (height as usize); + // let page_count = (size + 0xFFF) / 0x1000; + // let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?; - /// Reallocates the internal buffer with a new size - pub fn reallocate(&mut self, new_height: u32) -> Result<(), Error> { - // TODO suppress debugging output here - if new_height <= self.height { - // Keep using the old buffer - return Ok(()); - } + // let rows = unsafe { + // core::slice::from_raw_parts_mut(pages.virtualize() as *mut ConsoleRow, height as usize) + // }; - let size = size_of::() * (new_height as usize); - let page_count = (size + 0xFFF) / 0x1000; - let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?; + // for row in rows.iter_mut() { + // row.clear(DEFAULT_BG_COLOR); + // } - let data = unsafe { - core::slice::from_raw_parts_mut( - pages.virtualize() as *mut ConsoleRow, - new_height as usize, - ) - }; - - // Copy rows from the old buffer - data[0..self.height as usize].copy_from_slice(self.rows); - data[self.height as usize..].fill(ConsoleRow::zeroed()); - - self.rows = data; - self.height = new_height; - - Ok(()) + // Ok(Self { rows, height }) } #[inline(never)] diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 5749643a..cd3dd189 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -5,7 +5,10 @@ use core::ops::{Index, IndexMut}; use abi::error::Error; use device_api::Device; -use crate::{mem::device::DeviceMemory, sync::IrqSafeSpinlock}; +use crate::{ + mem::{device::RawDeviceMemoryMapping, PhysicalAddress}, + sync::IrqSafeSpinlock, +}; use super::{DisplayDevice, DisplayDimensions}; @@ -34,18 +37,17 @@ impl LinearFramebuffer { /// /// Unsafe: the caller must ensure the validity of all the arguments. pub unsafe fn from_physical_bits( - base: usize, + base: PhysicalAddress, size: usize, stride: usize, width: u32, height: u32, ) -> Result { - // TODO this may get Dropped later - let mmio = unsafe { DeviceMemory::map("framebuffer", base, size) }?; + let base = unsafe { RawDeviceMemoryMapping::map(base, size) }?.leak(); let inner = Inner { dimensions: DisplayDimensions { width, height }, - base: mmio.base(), + base, stride, }; diff --git a/src/device/mod.rs b/src/device/mod.rs index 51a99ea2..06ce75c2 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -7,9 +7,9 @@ use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; #[cfg(target_arch = "aarch64")] pub mod devtree; -pub mod bus; +// pub mod bus; pub mod display; -pub mod power; +// pub mod power; pub mod serial; pub mod timer; pub mod tty; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 0f79554c..563c0510 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -7,18 +7,14 @@ use memfs::block::{self, BlockAllocator}; use vfs::VnodeRef; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::mem::{ - self, - phys::{self, PageUsage}, - ConvertAddress, -}; +use crate::mem::{self, phys, PhysicalAddress}; pub mod devfs; /// Describes in-memory filesystem image used as initial root pub struct Initrd { /// Page-aligned start address of the initrd - pub phys_page_start: usize, + pub phys_page_start: PhysicalAddress, /// Page-aligned length pub phys_page_len: usize, /// Safe reference to the initrd data slice @@ -35,14 +31,14 @@ unsafe impl BlockAllocator for FileBlockAllocator { fn alloc() -> Result, Error> { // TODO make this a static assertion assert_eq!(block::SIZE, 4096); - let page = phys::alloc_page(PageUsage::Used)?; - Ok(unsafe { NonNull::new_unchecked(page.virtualize() as *mut _) }) + let page = phys::alloc_page()?; + Ok(unsafe { NonNull::new_unchecked(page.virtualize_raw() as *mut _) }) } unsafe fn dealloc(block: NonNull) { let page = block.as_ptr() as usize; - assert!(page > mem::KERNEL_VIRT_OFFSET); - phys::free_page(page.physicalize()); + let physical = PhysicalAddress::from_virtualized(page); + phys::free_page(physical); } } diff --git a/src/main.rs b/src/main.rs index 80a9191c..c7171e96 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ //! osdev-x kernel crate #![feature( + step_trait, decl_macro, naked_functions, asm_const, @@ -13,7 +14,8 @@ linked_list_cursors, rustc_private, allocator_api, - async_fn_in_trait + async_fn_in_trait, + strict_provenance )] #![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] // #![warn(missing_docs)] @@ -21,11 +23,12 @@ #![no_std] #![no_main] -use sync::SpinFence; +use arch::Architecture; use crate::{ - arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, + arch::{ArchitectureImpl, ARCHITECTURE}, mem::heap, + sync::SpinFence, task::{spawn_kernel_closure, Cpu}, }; diff --git a/src/mem/address.rs b/src/mem/address.rs new file mode 100644 index 00000000..9636ebde --- /dev/null +++ b/src/mem/address.rs @@ -0,0 +1,185 @@ +use core::{ + fmt, + iter::Step, + marker::PhantomData, + ops::{Add, Deref, DerefMut, Sub}, +}; + +use crate::arch::{Architecture, ArchitectureImpl, ARCHITECTURE}; + +use super::{pointer::PhysicalPointer, table::EntryLevel, KERNEL_VIRT_OFFSET}; + +#[repr(transparent)] +pub struct KernelImageObject { + inner: T, +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[repr(transparent)] +pub struct PhysicalAddress(u64); + +#[const_trait] +pub trait FromRaw { + fn from_raw(value: T) -> Self; +} + +#[const_trait] +pub trait IntoRaw { + fn into_raw(self) -> T; +} + +pub trait AsPhysicalAddress { + unsafe fn as_physical_address(&self) -> PhysicalAddress; +} + +// KernelImageObject wrapper for objects inside the kernel + +impl KernelImageObject { + pub const unsafe fn new(inner: T) -> Self { + Self { inner } + } +} + +impl AsPhysicalAddress for KernelImageObject { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::from_raw(&self.inner as *const _ as usize - KERNEL_VIRT_OFFSET) + } +} + +impl Deref for KernelImageObject { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for KernelImageObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +// + +impl PhysicalAddress { + pub const ZERO: Self = Self(0); + + pub const MAX: Self = Self(u64::MAX); + pub const MIN: Self = Self(u64::MIN); + + pub const fn add(self, offset: usize) -> Self { + Self(self.0 + offset as u64) + } + + #[inline(always)] + pub const fn is_zero(self) -> bool { + self.0 == 0 + } + + pub const fn page_offset(self) -> usize { + L::page_offset(self.0 as usize) + } + + pub const fn page_align_down(self) -> Self { + Self(self.0 & !(L::SIZE as u64 - 1)) + } + + pub const fn page_align_up(self) -> Self { + Self((self.0 + L::SIZE as u64 - 1) & !(L::SIZE as u64 - 1)) + } + + pub unsafe fn from_virtualized(address: usize) -> Self { + ArchitectureImpl::physicalize(address).unwrap() + } + + pub fn virtualize_raw(self) -> usize { + ArchitectureImpl::virtualize(self).unwrap() + } + + pub fn virtualize(self) -> PhysicalPointer { + loop {} + } + + pub fn virtualize_slice(self, len: usize) -> PhysicalPointer<[T]> { + loop {} + } +} + +impl Add for PhysicalAddress { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl Sub for PhysicalAddress { + type Output = usize; + + fn sub(self, rhs: Self) -> Self::Output { + (self.0 - rhs.0) as usize + } +} + +// Conversions + +impl const FromRaw for PhysicalAddress { + fn from_raw(value: u64) -> Self { + Self(value) + } +} + +impl const FromRaw for PhysicalAddress { + fn from_raw(value: usize) -> Self { + Self(value as u64) + } +} + +impl const IntoRaw for PhysicalAddress { + fn into_raw(self) -> u64 { + self.0 + } +} + +impl const IntoRaw for PhysicalAddress { + fn into_raw(self) -> usize { + self.0 as usize + } +} + +impl From for u64 { + fn from(addr: PhysicalAddress) -> u64 { + addr.0 + } +} + +impl From for usize { + fn from(addr: PhysicalAddress) -> usize { + addr.0 as usize + } +} + +// Ranges + +impl Step for PhysicalAddress { + fn steps_between(start: &Self, end: &Self) -> Option { + loop {} + } + + fn forward_checked(start: Self, count: usize) -> Option { + start.0.checked_add(count as u64).map(Self) + } + + fn backward_checked(start: Self, count: usize) -> Option { + loop {} + } +} + +// fmt + +impl fmt::LowerHex for PhysicalAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} diff --git a/src/mem/device.rs b/src/mem/device.rs index ca7a2491..988ff62f 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -1,85 +1,80 @@ //! Facilities for mapping devices to virtual address space -use core::{marker::PhantomData, mem::size_of, ops::Deref}; +use core::{marker::PhantomData, mem::size_of, ops::Deref, sync::atomic::AtomicUsize}; use abi::error::Error; +use alloc::sync::Arc; use crate::arch::{Architecture, ARCHITECTURE}; -/// Generic MMIO access mapping +use super::PhysicalAddress; + +#[derive(Debug)] +pub struct RawDeviceMemoryMapping { + pub address: usize, + pub base_address: usize, + pub page_size: usize, + pub page_count: usize, +} + #[derive(Clone, Debug)] -#[allow(unused)] -pub struct DeviceMemory { - name: &'static str, - base: usize, - size: usize, +pub struct DeviceMemoryMapping { + inner: Arc, + address: usize, } -/// MMIO wrapper for `T` -pub struct DeviceMemoryIo { - mmio: DeviceMemory, - _pd: PhantomData, +#[derive(Clone, Debug)] +pub struct DeviceMemoryIo<'a, T: ?Sized> { + inner: Arc, + value: &'a T, } -impl DeviceMemory { - /// Maps the device to some virtual memory address and constructs a wrapper for that range. - /// - /// # Safety - /// - /// The caller is responsible for making sure the (phys, size) range is valid and actually - /// points to some device's MMIO. The caller must also make sure no aliasing for that range is - /// possible. - pub unsafe fn map(name: &'static str, phys: usize, size: usize) -> Result { - let aligned_base = phys & !0xFFF; - let base_offset = phys & 0xFFF; - let aligned_size = (size + 0xFFF) & !0xFFF; - - let base = ARCHITECTURE.map_device_pages(aligned_base, aligned_size / 0x1000)?; - let base = base + base_offset; - - debugln!("Mapped {}@{:#x} to {:#x}", name, phys, base); - - Ok(Self { name, base, size }) - } - - /// Returns the base address of this mapping +impl RawDeviceMemoryMapping { #[inline] - pub fn base(&self) -> usize { - self.base + pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { + ARCHITECTURE.map_device_memory(base, size) + } + + pub fn leak(self) -> usize { + let address = self.address; + core::mem::forget(self); + address } } -impl DeviceMemoryIo { - /// Maps the `T` struct at `phys` to some virtual memory address and provides a [Deref]able - /// wrapper to it. - /// - /// # Safety - /// - /// The caller is responsible for making sure the `phys` address points to a MMIO region which - /// is at least `size_of::()` and no aliasing for that region is possible. - pub unsafe fn map(name: &'static str, phys: usize) -> Result { - DeviceMemory::map(name, phys, size_of::()).map(|t| Self::new(t)) - } - - /// Constructs a device MMIO wrapper from given [DeviceMemory] mapping. - /// - /// # Safety - /// - /// The caller must ensure `mmio` actually points to a device of type `T`. - pub unsafe fn new(mmio: DeviceMemory) -> Self { - assert!(mmio.size >= size_of::()); - // TODO check align - - Self { - mmio, - _pd: PhantomData, - } +impl Drop for RawDeviceMemoryMapping { + fn drop(&mut self) { + loop {} } } -impl Deref for DeviceMemoryIo { +impl DeviceMemoryMapping { + pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { + let inner = RawDeviceMemoryMapping::map(base, size)?; + loop {} + } +} + +impl<'a, T: Sized> DeviceMemoryIo<'a, T> { + pub unsafe fn from_raw(raw: DeviceMemoryMapping) -> DeviceMemoryIo<'a, T> { + // TODO + loop {} + } + + pub unsafe fn map(base: PhysicalAddress) -> Result, Error> { + let inner = RawDeviceMemoryMapping::map(base, size_of::())?; + let value = &*(inner.address as *const T); + + Ok(DeviceMemoryIo { + inner: Arc::new(inner), + value, + }) + } +} + +impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> { type Target = T; fn deref(&self) -> &Self::Target { - unsafe { &*(self.mmio.base as *const T) } + self.value } } diff --git a/src/mem/heap.rs b/src/mem/heap.rs index fca4f7ce..cb3bd678 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -35,7 +35,7 @@ unsafe impl GlobalAlloc for KernelAllocator { match self.inner.lock().allocate_first_fit(layout) { Ok(v) => v.as_ptr(), Err(e) => { - errorln!("Failed to allocate {:?}: {:?}", layout, e); + // errorln!("Failed to allocate {:?}: {:?}", layout, e); null_mut() } } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 9eda78af..518a2048 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,57 +1,75 @@ -//! Memory management utilities and types -// use core::{alloc::Layout, mem::size_of}; +// //! Memory management utilities and types +// // use core::{alloc::Layout, mem::size_of}; +// +// use core::{alloc::Layout, ffi::c_void, mem::size_of}; +// +// use abi::error::Error; +// +// // use abi::error::Error; +// // +// use crate::arch::{Architecture, ArchitectureImpl /*, PlatformImpl*/}; +// +// use self::table::AddressSpace; +// // +// // use self::table::AddressSpace; +// +// pub mod device; -use core::{alloc::Layout, ffi::c_void, mem::size_of}; +use core::{alloc::Layout, ffi::c_void, mem::size_of, ops::Add}; use abi::error::Error; -// use abi::error::Error; -// -use crate::arch::{Architecture, ArchitectureImpl /*, PlatformImpl*/}; - -use self::table::AddressSpace; -// -// use self::table::AddressSpace; +use crate::arch::{Architecture, ArchitectureImpl}; +pub mod address; pub mod device; pub mod heap; pub mod phys; +pub mod pointer; pub mod table; -/// Kernel's physical load address -// pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; -/// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + -/// [KERNEL_VIRT_OFFSET]) +pub use address::PhysicalAddress; + +use self::table::AddressSpace; + pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; -/// Interface for converting between address spaces. -/// -/// # Safety -/// -/// An incorrect implementation can produce invalid address. -pub unsafe trait ConvertAddress { - /// Convert the address into a virtual one - /// - /// # Panics - /// - /// Panics if the address is already a virtual one - /// - /// # Safety - /// - /// An incorrect implementation can produce invalid address. - unsafe fn virtualize(self) -> Self; - /// Convert the address into a physical one - /// - /// # Panics - /// - /// Panics if the address is already a physical one - /// - /// # Safety - /// - /// An incorrect implementation can produce invalid address. - unsafe fn physicalize(self) -> Self; -} - +// pub mod phys; +// +// /// Kernel's physical load address +// // pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; +// /// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + +// /// [KERNEL_VIRT_OFFSET]) +// pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; +// +// /// Interface for converting between address spaces. +// /// +// /// # Safety +// /// +// /// An incorrect implementation can produce invalid address. +// pub unsafe trait ConvertAddress { +// /// Convert the address into a virtual one +// /// +// /// # Panics +// /// +// /// Panics if the address is already a virtual one +// /// +// /// # Safety +// /// +// /// An incorrect implementation can produce invalid address. +// unsafe fn virtualize(self) -> Self; +// /// Convert the address into a physical one +// /// +// /// # Panics +// /// +// /// Panics if the address is already a physical one +// /// +// /// # Safety +// /// +// /// An incorrect implementation can produce invalid address. +// unsafe fn physicalize(self) -> Self; +// } +// /// Helper trait to allow cross-address space access to pointers pub trait ForeignPointer: Sized { /// Perform a volatile pointer write without dropping the old value. @@ -120,52 +138,6 @@ pub trait ForeignPointer: Sized { ) -> Result<&'a mut [Self], Error>; } -unsafe impl ConvertAddress for usize { - #[inline(always)] - unsafe fn virtualize(self) -> Self { - #[cfg(debug_assertions)] - if self > KERNEL_VIRT_OFFSET { - todo!(); - } - - self + KERNEL_VIRT_OFFSET - } - - #[inline(always)] - unsafe fn physicalize(self) -> Self { - #[cfg(debug_assertions)] - if self < KERNEL_VIRT_OFFSET { - todo!(); - } - - self - KERNEL_VIRT_OFFSET - } -} - -unsafe impl ConvertAddress for *mut T { - #[inline(always)] - unsafe fn virtualize(self) -> Self { - (self as usize).virtualize() as Self - } - - #[inline(always)] - unsafe fn physicalize(self) -> Self { - (self as usize).physicalize() as Self - } -} - -unsafe impl ConvertAddress for *const T { - #[inline(always)] - unsafe fn virtualize(self) -> Self { - (self as usize).virtualize() as Self - } - - #[inline(always)] - unsafe fn physicalize(self) -> Self { - (self as usize).physicalize() as Self - } -} - impl ForeignPointer for T { unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { // TODO check align @@ -182,7 +154,7 @@ impl ForeignPointer for T { .translate(start_page) .expect("Address is not mapped in the target address space"); - let virt_ptr = (phys_page + page_offset).virtualize() as *mut T; + let virt_ptr = phys_page.add(page_offset).virtualize_raw() as *mut T; virt_ptr.write_volatile(value); } diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index c08e8f3f..db28c645 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -3,89 +3,132 @@ use core::mem::size_of; use abi::error::Error; -use super::{Page, PageUsage, PhysicalMemoryStats}; +use crate::mem::{ + address::{FromRaw, IntoRaw}, + pointer::{PhysicalRef, PhysicalRefMut}, + PhysicalAddress, +}; + +pub type BitmapWord = u64; + +pub(super) const BITMAP_WORD_SIZE: usize = size_of::() * 8; +pub(super) const BITMAP_PAGE_COUNT: usize = 256; + +const HUGE_PAGE_WORD_COUNT: usize = 512 / BITMAP_WORD_SIZE; + +pub(super) const TRACKED_PAGE_LIMIT: usize = (BITMAP_PAGE_COUNT * 4096) * 8; /// Physical memory management interface pub struct PhysicalMemoryManager { - pages: &'static mut [Page], - offset: usize, - stats: PhysicalMemoryStats, + bitmap: PhysicalRefMut<'static, [u64]>, + last_free_bit: usize, + page_count: usize, } impl PhysicalMemoryManager { - /// Constructs a [PhysicalMemoryManager] with page tracking array placed at given - /// `base`..`base+size` range. Physical addresses allocated are offset by the given value. - /// - /// # Safety - /// - /// Addresses are not checked. The caller is responsible for making sure (base, size) ranges do - /// not alias/overlap, they're accessible through virtual memory and that the offset is a - /// meaningful value. - pub unsafe fn new(offset: usize, base: usize, size: usize) -> PhysicalMemoryManager { - // TODO check alignment - let page_count = size / size_of::(); - let pages = core::slice::from_raw_parts_mut(base as *mut _, page_count); + pub unsafe fn new( + bitmap_phys_base: PhysicalAddress, + page_count: usize, + ) -> PhysicalMemoryManager { + // let bitmap_addr = bitmap_phys_base.virtualize(); + let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE; + let mut bitmap = PhysicalRefMut::<'static, u64>::map_slice(bitmap_phys_base, bitmap_len); + // let bitmap = core::slice::from_raw_parts_mut(bitmap_addr as *mut BitmapWord, bitmap_len); - for page in pages.iter_mut() { - *page = Page { - usage: PageUsage::Reserved, - refcount: 0, - }; - } + bitmap.fill(BitmapWord::MAX); - PhysicalMemoryManager { - pages, - offset, - stats: PhysicalMemoryStats { - available_pages: 0, - used_pages: 0, - }, + Self { + bitmap, + page_count, + last_free_bit: 0, } } + #[inline] + fn mark_alloc(&mut self, index: usize) { + self.bitmap[index / BITMAP_WORD_SIZE] |= 1 << (index & (BITMAP_WORD_SIZE - 1)); + } + + #[inline] + fn mark_free(&mut self, index: usize) { + self.bitmap[index / BITMAP_WORD_SIZE] &= !(1 << (index & (BITMAP_WORD_SIZE - 1))); + } + + #[inline(always)] + fn is_alloc(&self, index: usize) -> bool { + self.bitmap[index / BITMAP_WORD_SIZE] & (1 << (index & (BITMAP_WORD_SIZE - 1))) != 0 + } + /// Allocates a single page, marking it as used with `usage` - pub fn alloc_page(&mut self, usage: PageUsage) -> Result { - assert_ne!(usage, PageUsage::Available); - assert_ne!(usage, PageUsage::Reserved); - - for index in 0..self.pages.len() { - if self.pages[index].usage == PageUsage::Available { - self.pages[index].usage = PageUsage::Used; - self.stats.add_allocated_pages(1, usage); - return Ok(index * 4096 + self.offset); + pub fn alloc_page(&mut self) -> Result { + for i in self.last_free_bit..self.page_count { + if self.is_alloc(i) { + continue; } + + self.last_free_bit = i + 1; + self.mark_alloc(i); + + return Ok(PhysicalAddress::from_raw(i * 0x1000)); } - Err(Error::OutOfMemory) + if self.last_free_bit != 0 { + self.last_free_bit = 0; + self.alloc_page() + } else { + loop {} + } } - /// Allocates a contiguous range of physical pages, marking it as used with `usage` - pub fn alloc_contiguous_pages( - &mut self, - count: usize, - usage: PageUsage, - ) -> Result { - assert_ne!(usage, PageUsage::Available); - assert_ne!(usage, PageUsage::Reserved); - assert_ne!(count, 0); + pub fn alloc_2m_page(&mut self) -> Result { + let aligned_bit = self.last_free_bit & !511; - 'l0: for i in 0..self.pages.len() { - for j in 0..count { - if self.pages[i + j].usage != PageUsage::Available { + 'l0: for i in (aligned_bit..self.page_count).step_by(512) { + for j in 0..HUGE_PAGE_WORD_COUNT { + if self.bitmap[i / BITMAP_WORD_SIZE] != 0 { continue 'l0; } } - for j in 0..count { - let page = &mut self.pages[i + j]; - assert!(page.usage == PageUsage::Available); - page.usage = usage; - page.refcount = 1; + + for j in 0..HUGE_PAGE_WORD_COUNT { + self.bitmap[i / BITMAP_WORD_SIZE + j] = BitmapWord::MAX; } - self.stats.add_allocated_pages(count, usage); - return Ok(self.offset + i * 0x1000); + self.last_free_bit = i + 512; + + return Ok(PhysicalAddress::from_raw(i * 0x1000)); } - Err(Error::OutOfMemory) + if self.last_free_bit != 0 { + self.last_free_bit = 0; + self.alloc_2m_page() + } else { + loop {} + } + } + + /// Allocates a contiguous range of physical pages, marking it as used with `usage` + pub fn alloc_contiguous_pages(&mut self, count: usize) -> Result { + 'l0: for i in self.last_free_bit..self.page_count { + for j in 0..count { + if self.is_alloc(i + j) { + continue 'l0; + } + } + + for j in 0..count { + self.mark_alloc(i + j); + } + self.last_free_bit = i + count; + + return Ok(PhysicalAddress::from_raw(i * 0x1000)); + } + + if self.last_free_bit != 0 { + self.last_free_bit = 0; + self.alloc_contiguous_pages(count) + } else { + loop {} + } } /// Deallocates a physical memory page. @@ -93,15 +136,11 @@ impl PhysicalMemoryManager { /// # Safety /// /// `addr` must be a page-aligned physical address previously allocated by this implementation. - pub unsafe fn free_page(&mut self, addr: usize) { - assert!(addr > self.offset); - let index = (addr - self.offset) / 0x1000; - let page = &mut self.pages[index]; - assert_eq!(page.usage, PageUsage::Used); + pub unsafe fn free_page(&mut self, page: PhysicalAddress) { + let index = IntoRaw::::into_raw(page) / 0x1000; - self.stats.add_freed_pages(1, page.usage); - - page.usage = PageUsage::Available; + assert!(self.is_alloc(index)); + self.mark_free(index); } /// Marks a previously reserved page as available. @@ -109,19 +148,10 @@ impl PhysicalMemoryManager { /// # Panics /// /// Will panic if the address does not point to a valid, reserved (and unallocated) page. - pub fn add_available_page(&mut self, addr: usize) { - assert!(addr >= self.offset); - let index = (addr - self.offset) / 4096; + pub fn add_available_page(&mut self, page: PhysicalAddress) { + let index = IntoRaw::::into_raw(page) / 0x1000; - assert_eq!(self.pages[index].usage, PageUsage::Reserved); - assert_eq!(self.pages[index].refcount, 0); - - self.stats.add_available_pages(1); - self.pages[index].usage = PageUsage::Available; - } - - /// Returns a reference to physical memory stats info - pub fn stats(&self) -> &PhysicalMemoryStats { - &self.stats + assert!(self.is_alloc(index)); + self.mark_free(index); } } diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 54ce445c..70c1da90 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,135 +1,164 @@ -//! Physical memory management facilities -use core::{iter::StepBy, mem::size_of, ops::Range}; +use core::{iter::StepBy, ops::Range}; use abi::error::Error; use kernel_util::util::OneTimeInit; use crate::{ - debug::LogLevel, - mem::{ - phys::reserved::{is_reserved, reserve_region}, - ConvertAddress, /*, KERNEL_PHYS_BASE */ - }, + arch::{Architecture, ARCHITECTURE}, + mem::phys::reserved::is_reserved, sync::IrqSafeSpinlock, }; -use self::manager::PhysicalMemoryManager; +use self::{ + manager::{PhysicalMemoryManager, BITMAP_PAGE_COUNT, BITMAP_WORD_SIZE, TRACKED_PAGE_LIMIT}, + reserved::reserve_region, +}; -// Enumerating lots of pages is slow and I'm too lazy right now to write a better algorithm, so -// capping the page count helps -const PHYS_MEMORY_PAGE_CAP: usize = 65536; +use super::{address::FromRaw, PhysicalAddress}; -pub mod manager; +// //! Physical memory management facilities +// use core::{iter::StepBy, mem::size_of, ops::Range}; +// +// use abi::error::Error; +// use kernel_util::util::OneTimeInit; +// +// use crate::{ +// debug::LogLevel, +// mem::{ +// phys::reserved::{is_reserved, reserve_region}, +// ConvertAddress, /*, KERNEL_PHYS_BASE */ +// }, +// sync::IrqSafeSpinlock, +// }; +// +// use self::manager::PhysicalMemoryManager; +// +// // Enumerating lots of pages is slow and I'm too lazy right now to write a better algorithm, so +// // capping the page count helps +// const PHYS_MEMORY_PAGE_CAP: usize = 65536; +// + +// 8 * 4096 bits per page, 1 page per bit +const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PAGE_LIMIT * 4096); + +mod manager; pub mod reserved; - -/// Contains information about the physical memory usage -#[derive(Clone, Copy, Debug)] -pub struct PhysicalMemoryStats { - /// Number of pages available for allocation - pub available_pages: usize, - /// Number of pages being used - pub used_pages: usize, -} - -/// Represents the way in which the page is used (or not) -#[derive(PartialEq, Clone, Copy, Debug)] -#[repr(u32)] -pub enum PageUsage { - /// Page is not available for allocation or use - Reserved = 0, - /// Regular page available for allocation - Available, - /// Page is used by some kernel facility - Used, -} - -/// Page descriptor structure for the page management array -#[repr(C)] -pub struct Page { - usage: PageUsage, - refcount: u32, -} - +// +// /// Contains information about the physical memory usage +// #[derive(Clone, Copy, Debug)] +// pub struct PhysicalMemoryStats { +// /// Number of pages available for allocation +// pub available_pages: usize, +// /// Number of pages being used +// pub used_pages: usize, +// } +// +// /// Represents the way in which the page is used (or not) +// #[derive(PartialEq, Clone, Copy, Debug)] +// #[repr(u32)] +// pub enum PageUsage { +// /// Page is not available for allocation or use +// Reserved = 0, +// /// Regular page available for allocation +// Available, +// /// Page is used by some kernel facility +// Used, +// } +// +// /// Page descriptor structure for the page management array +// #[repr(C)] +// pub struct Page { +// usage: PageUsage, +// refcount: u32, +// } +// /// Defines an usable memory region #[derive(Clone, Copy, Debug)] pub struct PhysicalMemoryRegion { /// Start of the region - pub base: usize, + pub base: PhysicalAddress, /// Length of the region pub size: usize, } impl PhysicalMemoryRegion { /// Returns the end address of the region - pub const fn end(&self) -> usize { - self.base + self.size + pub const fn end(&self) -> PhysicalAddress { + self.base.add(self.size) } /// Returns an address range covered by the region - pub fn range(&self) -> Range { + pub fn range(&self) -> Range { self.base..self.end() } - /// Provides an iterator over the pages in the region - pub fn pages(&self) -> StepBy> { - self.range().step_by(0x1000) + pub fn clamp(self) -> Option<(PhysicalAddress, PhysicalAddress)> { + let start = self.base.min(MEMORY_UPPER_LIMIT); + let end = self.end().min(MEMORY_UPPER_LIMIT); + + if start < end { + Some((start, end)) + } else { + None + } } } - -impl PhysicalMemoryStats { - /// Handles "alloc" cases of the memory manager - pub fn add_allocated_pages(&mut self, count: usize, _usage: PageUsage) { - assert!(self.available_pages >= count); - self.available_pages -= count; - self.used_pages += count; - } - - /// Handles "free" cases of the memory manager - pub fn add_freed_pages(&mut self, count: usize, _usage: PageUsage) { - assert!(self.used_pages >= count); - self.used_pages -= count; - self.available_pages += count; - } - - /// Increases the available pages counter - pub fn add_available_pages(&mut self, count: usize) { - self.available_pages += count; - } - - /// Prints out the statistics into specified log level - pub fn dump(&self, level: LogLevel) { - log_print_raw!(level, "+++ Physical memory stats +++\n"); - log_print_raw!( - level, - "Available: {}K ({} pages)\n", - self.available_pages * 4, - self.available_pages - ); - log_print_raw!( - level, - "Used: {}K ({} pages)\n", - self.used_pages * 4, - self.used_pages - ); - log_print_raw!(level, "-----------------------------\n"); - } -} - +// +// impl PhysicalMemoryStats { +// /// Handles "alloc" cases of the memory manager +// pub fn add_allocated_pages(&mut self, count: usize, _usage: PageUsage) { +// assert!(self.available_pages >= count); +// self.available_pages -= count; +// self.used_pages += count; +// } +// +// /// Handles "free" cases of the memory manager +// pub fn add_freed_pages(&mut self, count: usize, _usage: PageUsage) { +// assert!(self.used_pages >= count); +// self.used_pages -= count; +// self.available_pages += count; +// } +// +// /// Increases the available pages counter +// pub fn add_available_pages(&mut self, count: usize) { +// self.available_pages += count; +// } +// +// /// Prints out the statistics into specified log level +// pub fn dump(&self, level: LogLevel) { +// log_print_raw!(level, "+++ Physical memory stats +++\n"); +// log_print_raw!( +// level, +// "Available: {}K ({} pages)\n", +// self.available_pages * 4, +// self.available_pages +// ); +// log_print_raw!( +// level, +// "Used: {}K ({} pages)\n", +// self.used_pages * 4, +// self.used_pages +// ); +// log_print_raw!(level, "-----------------------------\n"); +// } +// } +// /// Global physical memory manager pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); /// Allocates a single physical page from the global manager -pub fn alloc_page(usage: PageUsage) -> Result { - PHYSICAL_MEMORY.get().lock().alloc_page(usage) +pub fn alloc_page() -> Result { + PHYSICAL_MEMORY.get().lock().alloc_page() } /// Allocates a contiguous range of physical pages from the global manager -pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result { - PHYSICAL_MEMORY - .get() - .lock() - .alloc_contiguous_pages(count, usage) +pub fn alloc_pages_contiguous(count: usize) -> Result { + PHYSICAL_MEMORY.get().lock().alloc_contiguous_pages(count) +} + +pub fn alloc_2m_page() -> Result { + PHYSICAL_MEMORY.get().lock().alloc_2m_page() } /// Deallocates a physical memory page. @@ -137,26 +166,26 @@ pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result>( it: I, -) -> Option<(usize, usize)> { - let mut start = usize::MAX; - let mut end = usize::MIN; +) -> Option<(PhysicalAddress, PhysicalAddress)> { + let mut start = PhysicalAddress::MAX; + let mut end = PhysicalAddress::MIN; - for reg in it { - if reg.base < start { - start = reg.base; + for (reg_start, reg_end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { + if reg_start < start { + start = reg_start; } - if reg.base + reg.size > end { - end = reg.base + reg.size; + if reg_end > end { + end = reg_end; } } - if start == usize::MAX || end == usize::MIN { + if start == PhysicalAddress::MAX || end == PhysicalAddress::MIN { None } else { Some((start, end)) @@ -166,12 +195,12 @@ fn physical_memory_range>( fn find_contiguous_region>( it: I, count: usize, -) -> Option { - for region in it { +) -> Option { + for (reg_start, reg_end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { let mut collected = 0; let mut base_addr = None; - for addr in region.pages() { + for addr in (reg_start..reg_end).step_by(0x1000) { if is_reserved(addr) { collected = 0; base_addr = None; @@ -188,7 +217,7 @@ fn find_contiguous_region>( } todo!() } - +// /// Initializes physical memory manager from given available memory region iterator. /// /// 1. Finds a non-reserved range to place the page tracking array. @@ -201,68 +230,102 @@ fn find_contiguous_region>( pub unsafe fn init_from_iter + Clone>( it: I, ) -> Result<(), Error> { + // Map the physical memory let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap(); + + ARCHITECTURE.map_physical_memory(it.clone(), phys_start, phys_end)?; + let total_count = (phys_end - phys_start) / 0x1000; - let pages_array_size = total_count * size_of::(); + let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / BITMAP_WORD_SIZE; + let page_bitmap_page_count = (page_bitmap_size + 0xFFF) / 0x1000; - debugln!("Initializing physical memory manager"); - debugln!("Total tracked pages: {}", total_count); - - // Reserve memory regions from which allocation is forbidden reserve_region("kernel", kernel_physical_memory_region()); - let pages_array_base = find_contiguous_region(it.clone(), (pages_array_size + 0xFFF) / 0x1000) - .ok_or(Error::OutOfMemory)?; - - debugln!( - "Placing page tracking at {:#x}", - pages_array_base.virtualize() - ); + let page_bitmap_phys_base = find_contiguous_region(it.clone(), page_bitmap_page_count).unwrap(); reserve_region( - "pages", + "page-bitmap", PhysicalMemoryRegion { - base: pages_array_base, - size: (pages_array_size + 0xFFF) & !0xFFF, + base: page_bitmap_phys_base, + size: page_bitmap_page_count * 0x1000, }, ); - let mut manager = - PhysicalMemoryManager::new(phys_start, pages_array_base.virtualize(), pages_array_size); - let mut page_count = 0; + let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, total_count); - for region in it { - if page_count >= PHYS_MEMORY_PAGE_CAP { - break; - } - - for page in region.pages() { + for (start, end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { + for page in (start..end).step_by(0x1000) { if is_reserved(page) { continue; } manager.add_available_page(page); - page_count += 1; - - if page_count >= PHYS_MEMORY_PAGE_CAP { - break; - } } } - infoln!("{} available pages ({}KiB)", page_count, page_count * 4); - PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager)); + Ok(()) } - +// +// debugln!("Initializing physical memory manager"); +// debugln!("Total tracked pages: {}", total_count); +// +// // Reserve memory regions from which allocation is forbidden +// reserve_region("kernel", kernel_physical_memory_region()); +// +// let pages_array_base = find_contiguous_region(it.clone(), (pages_array_size + 0xFFF) / 0x1000) +// .ok_or(Error::OutOfMemory)?; +// +// debugln!( +// "Placing page tracking at {:#x}", +// pages_array_base.virtualize() +// ); +// +// reserve_region( +// "pages", +// PhysicalMemoryRegion { +// base: pages_array_base, +// size: (pages_array_size + 0xFFF) & !0xFFF, +// }, +// ); +// +// let mut manager = +// PhysicalMemoryManager::new(phys_start, pages_array_base.virtualize(), pages_array_size); +// let mut page_count = 0; +// +// for region in it { +// if page_count >= PHYS_MEMORY_PAGE_CAP { +// break; +// } +// +// for page in region.pages() { +// if is_reserved(page) { +// continue; +// } +// +// manager.add_available_page(page); +// page_count += 1; +// +// if page_count >= PHYS_MEMORY_PAGE_CAP { +// break; +// } +// } +// } +// +// infoln!("{} available pages ({}KiB)", page_count, page_count * 4); +// +// PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager)); +// Ok(()) +// } +// fn kernel_physical_memory_region() -> PhysicalMemoryRegion { extern "C" { static __kernel_phys_start: u8; static __kernel_size: u8; } - let base = absolute_address!(__kernel_phys_start); + let base = PhysicalAddress::from_raw(absolute_address!(__kernel_phys_start)); let size = absolute_address!(__kernel_size); PhysicalMemoryRegion { base, size } diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 697c77d5..099a2309 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -2,6 +2,8 @@ use kernel_util::util::StaticVector; +use crate::mem::PhysicalAddress; + use super::PhysicalMemoryRegion; static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); @@ -12,18 +14,18 @@ static mut RESERVED_MEMORY: StaticVector = StaticVector /// /// Can only be called from initialization code **before** physical memory manager is initialized. pub unsafe fn reserve_region(reason: &str, region: PhysicalMemoryRegion) { - debugln!( - "Reserve {:?} memory: {:#x}..{:#x}", - reason, - region.base, - region.end() - ); + // debugln!( + // "Reserve {:?} memory: {:#x}..{:#x}", + // reason, + // region.base, + // region.end() + // ); RESERVED_MEMORY.push(region); } /// Returns `true` if `addr` refers to any reserved memory region -pub fn is_reserved(addr: usize) -> bool { +pub fn is_reserved(addr: PhysicalAddress) -> bool { for region in unsafe { RESERVED_MEMORY.iter() } { if region.range().contains(&addr) { return true; diff --git a/src/mem/pointer.rs b/src/mem/pointer.rs new file mode 100644 index 00000000..85c10fe1 --- /dev/null +++ b/src/mem/pointer.rs @@ -0,0 +1,132 @@ +use core::{ + alloc::Layout, + fmt, + ops::{Deref, DerefMut}, +}; + +use super::{address::AsPhysicalAddress, PhysicalAddress}; + +#[derive(Clone, Copy, PartialEq, PartialOrd, Debug, Hash)] +#[repr(transparent)] +pub struct PhysicalPointer { + pointer: *mut T, +} + +#[repr(transparent)] +pub struct PhysicalRef<'a, T: ?Sized> { + value: &'a T, +} + +#[repr(transparent)] +pub struct PhysicalRefMut<'a, T: ?Sized> { + value: &'a mut T, +} + +// PhysicalPointer wrapper for direct access to any memory location + +impl PhysicalPointer { + pub fn into_address(self) -> usize { + self.pointer.addr() + } +} + +impl PhysicalPointer { + pub unsafe fn write_unaligned(self, value: T) { + self.write_unaligned(value); + } +} + +impl AsPhysicalAddress for PhysicalPointer { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + todo!() + } +} + +// PhysicalRefMut wrapper for safe mutable access to physical addresses + +impl<'a, T: Sized> PhysicalRefMut<'a, T> { + pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRefMut<'a, T> { + let value = virtualize_raw(physical); + PhysicalRefMut { value } + } + + pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRefMut<'a, [T]> { + let value = virtualize_slice_raw(physical, len); + PhysicalRefMut { value } + } +} + +impl PhysicalRefMut<'_, T> { + #[inline] + pub fn as_address(&self) -> usize { + (self.value as *const T).addr() + } +} + +impl AsPhysicalAddress for PhysicalRefMut<'_, T> { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::from_virtualized(self.as_address()) + } +} + +impl Deref for PhysicalRefMut<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl DerefMut for PhysicalRefMut<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl fmt::Pointer for PhysicalRefMut<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Pointer::fmt(&self.value, f) + } +} + +// PhysicalRef: same as PhysicalRefMut, except immutable + +impl<'a, T: Sized> PhysicalRef<'a, T> { + pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRef<'a, T> { + let value = virtualize_raw(physical); + PhysicalRef { value } + } +} + +impl PhysicalRef<'_, T> { + #[inline] + pub fn as_address(&self) -> usize { + (self.value as *const T).addr() + } +} + +impl AsPhysicalAddress for PhysicalRef<'_, T> { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::from_virtualized(self.as_address()) + } +} + +impl Deref for PhysicalRef<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +unsafe fn virtualize_raw<'a, T: Sized>(physical: PhysicalAddress) -> &'a mut T { + // TODO check align + let address = physical.virtualize_raw(); + &mut *(address as *mut T) +} + +unsafe fn virtualize_slice_raw<'a, T: Sized>(physical: PhysicalAddress, len: usize) -> &'a mut [T] { + // TODO check align + let address = physical.virtualize_raw(); + core::slice::from_raw_parts_mut(address as *mut T, len) +} diff --git a/src/mem/table.rs b/src/mem/table.rs index fd40e306..50c21fb9 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -1,13 +1,17 @@ //! Virtual memory table interface +use core::ops::{Deref, DerefMut}; + use abi::error::Error; use bitflags::bitflags; use cfg_if::cfg_if; +use super::PhysicalAddress; + cfg_if! { if #[cfg(target_arch = "aarch64")] { pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::table::{AddressSpace, PageEntry, PageTable}; + pub use crate::arch::x86_64::mem::table::{AddressSpace, PageEntry, PageTable}; } } @@ -36,7 +40,12 @@ pub trait VirtualMemoryManager { ) -> Result; /// Insert a single 4KiB-page translation mapping into the table - fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error>; + fn map_page( + &self, + virt: usize, + phys: PhysicalAddress, + attrs: MapAttributes, + ) -> Result<(), Error>; /// Releases the virtual memory region from the address space and the pages it refers to fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error>; @@ -46,17 +55,23 @@ pub trait VirtualMemoryManager { pub trait NextPageTable { /// Type for the next-level page table type NextLevel; + type TableRef: Deref; + type TableRefMut: DerefMut; /// Tries looking up a next-level table at given index, allocating and mapping one if it is not /// present there - fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error>; + fn get_mut_or_alloc(&mut self, index: usize) -> Result; /// Returns a mutable reference to a next-level table at `index`, if present - fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel>; + fn get_mut(&mut self, index: usize) -> Option; + + fn get(&self, index: usize) -> Option; } /// Interface for a single level of address translation #[const_trait] pub trait EntryLevel: Copy { + const SIZE: usize; + /// Returns the index into a page table for a given address fn index(addr: usize) -> usize; /// Returns the offset of an address from the page start at current level diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 44d09dea..aeaff070 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -1,4 +1,6 @@ //! ELF binary format support +use core::ops::DerefMut; + use elf::{ abi::{PF_W, PF_X, PT_LOAD}, endian::AnyEndian, @@ -8,9 +10,9 @@ use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::mem::{ - phys::{self, PageUsage}, + phys, + pointer::PhysicalRefMut, table::{AddressSpace, MapAttributes, VirtualMemoryManager}, - ConvertAddress, }; #[derive(Clone, Copy)] @@ -64,7 +66,7 @@ fn load_bytes( elf_attrs: u32, ) -> Result<(), Error> where - F: FnMut(usize, &mut [u8]) -> Result<(), Error>, + F: FnMut(usize, PhysicalRefMut<'_, [u8]>) -> Result<(), Error>, { // TODO check for crazy addresses here @@ -87,19 +89,22 @@ where let virt_page = dst_page_aligned + page_idx * 0x1000; assert_eq!(virt_page & 0xFFF, 0); - if space.translate(virt_page).is_some() { - // Handle these cases + if let Some(page) = space.translate(virt_page) { + // TODO Handle these cases + warnln!("Page {:#x} is already mapped to {:#x}", virt_page, page); todo!(); } - let phys_page = phys::alloc_page(PageUsage::Used)?; + let phys_page = phys::alloc_page()?; space.map_page(virt_page, phys_page, attrs)?; + debugln!("Map {:#x} -> {:#x}", virt_page, phys_page); - let dst_slice = unsafe { - let addr = (phys_page + page_off).virtualize(); + let dst_slice = unsafe { PhysicalRefMut::map_slice(phys_page.add(page_off), count) }; + // let dst_slice = unsafe { + // let addr = (phys_page + page_off).virtualize(); - core::slice::from_raw_parts_mut(addr as *mut u8, count) - }; + // core::slice::from_raw_parts_mut(addr as *mut u8, count) + // }; src(off, dst_slice)?; @@ -127,10 +132,10 @@ pub fn load_elf_from_file(space: &AddressSpace, file: FileRef) -> Result Result Result<(), Er debugln!("arg data size = {}", args_size); - let phys_page = phys::alloc_page(PageUsage::Used)?; + let phys_page = phys::alloc_page()?; // TODO check if this doesn't overwrite anything space.map_page( virt, phys_page, - MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, // PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL, + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; - let write = unsafe { phys_page.virtualize() }; + let write = phys_page.virtualize_raw(); let mut offset = args_ptr_size; @@ -83,7 +83,7 @@ fn setup_binary>( let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; for i in 0..USER_STACK_PAGES { - let phys = phys::alloc_page(PageUsage::Used)?; + let phys = phys::alloc_page()?; space.map_page( virt_stack_base + i * 0x1000, phys, diff --git a/src/task/context.rs b/src/task/context.rs index 6808d302..dadfd1c1 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -4,7 +4,7 @@ use abi::{arch::SavedFrame, error::Error, process::ExitCode}; use alloc::boxed::Box; use cfg_if::cfg_if; -use crate::task::process::Process; +use crate::{mem::PhysicalAddress, task::process::Process}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -56,7 +56,12 @@ pub trait TaskContextImpl: Sized { /// Constructs a user thread context. The caller is responsible for allocating the userspace /// stack and setting up a valid address space for the context. - fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result; + fn user( + entry: usize, + arg: usize, + cr3: PhysicalAddress, + user_stack_sp: usize, + ) -> Result; /// Performs an entry into a context. /// diff --git a/src/task/process.rs b/src/task/process.rs index e58bfddc..e79f4cd8 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -82,7 +82,7 @@ pub struct Process { } /// Guard type that provides [Process] operations only available for current processes -pub struct CurrentProcess(Arc); +pub struct CurrentProcess(Arc, IrqGuard); impl Process { /// Creates a process from raw architecture-specific [TaskContext]. @@ -447,11 +447,8 @@ impl CurrentProcess { /// # Safety /// /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. - pub unsafe fn new(inner: Arc) -> Self { - // XXX - // assert_eq!(DAIF.read(DAIF::I), 1); - assert!(ArchitectureImpl::interrupt_mask()); - Self(inner) + pub unsafe fn new(inner: Arc, guard: IrqGuard) -> Self { + Self(inner, guard) } /// Configures signal entry information for the process diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs index 90fde118..ded29ee2 100644 --- a/src/task/runtime/task_queue.rs +++ b/src/task/runtime/task_queue.rs @@ -3,7 +3,11 @@ use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; use kernel_util::util::OneTimeInit; -use crate::{sync::IrqGuard, task::process::Process}; +use crate::{ + arch::{Architecture, ArchitectureImpl}, + sync::IrqGuard, + task::process::Process, +}; use super::task::Task; @@ -41,6 +45,7 @@ impl TaskQueue { pub fn dequeue(&self) -> Result, Error> { let process = Process::current(); + assert!(ArchitectureImpl::interrupt_mask()); loop { if let Some(task) = self.task_queue.pop() { return Ok(task); diff --git a/src/task/sched.rs b/src/task/sched.rs index aac2ea4e..9aaf760c 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -8,7 +8,7 @@ use kernel_util::util::OneTimeInit; use crate::{ // arch::aarch64::{context::TaskContext, cpu::Cpu}, arch::{Architecture, ArchitectureImpl}, - sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, + sync::{IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard}, }; use super::{ @@ -276,11 +276,12 @@ impl CpuQueue { /// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu] /// is called. pub fn current_process(&self) -> Option { + let guard = IrqGuard::acquire(); self.inner .lock() .current .clone() - .map(|p| unsafe { CurrentProcess::new(p) }) + .map(|p| unsafe { CurrentProcess::new(p, guard) }) } /// Returns a queue for given CPU index diff --git a/tools/gentables/Cargo.toml b/tools/gentables/Cargo.toml new file mode 100644 index 00000000..ae7f7609 --- /dev/null +++ b/tools/gentables/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "gentables" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +memtables = { path = "../../lib/memtables" } + +bytemuck = "1.14.0" +elf = "0.7.2" +thiserror = "1.0.48" +clap = { version = "4.4.2", features = ["derive"] } +bitflags = "2.4.0" diff --git a/tools/gentables/src/main.rs b/tools/gentables/src/main.rs new file mode 100644 index 00000000..0ec86b02 --- /dev/null +++ b/tools/gentables/src/main.rs @@ -0,0 +1,186 @@ +#![feature(offset_of)] + +use std::{ + fs::OpenOptions, + io::{Read, Seek, SeekFrom, Write}, + ops::Range, + path::{Path, PathBuf}, + process::ExitCode, +}; + +use clap::Parser; +use elf::{ + abi::{EM_X86_64, PT_LOAD}, + endian::AnyEndian, + ElfStream, +}; +use memtables::FixedTables; +use thiserror::Error; + +use crate::x86_64::X8664Builder; + +mod x86_64; + +#[derive(Error, Debug)] +pub enum GenError { + #[error("I/O error: {0}")] + IoError(#[from] std::io::Error), + #[error("ELF parse error: {0}")] + ElfParseError(#[from] elf::ParseError), + + #[error("Image's arhitecture is not supported")] + UnsupportedArchitecture, + #[error("Could not determine the kernel image address range (possibly incorrect segments?)")] + NoKernelImageRange, + #[error("Kernel image is too large: {0:#x?} ({1}B). Maximum size: {2}B")] + KernelTooLarge(Range, u64, u64), + #[error("Kernel image is missing a required symbol: {0:?}")] + MissingSymbol(&'static str), + #[error("Kernel image is missing a required section: {0:?}")] + MissingSection(&'static str), + #[error("Incorrect tables section placement: {0:#x}")] + IncorrectTablesPlacement(u64), +} + +#[derive(Parser)] +struct Args { + image: PathBuf, +} + +pub struct GenData { + pub kernel_start: u64, + pub kernel_end: u64, + + pub table_offset: u64, + pub table_physical_address: u64, + pub kernel_virt_offset: u64, +} + +fn kernel_image_range( + elf: &mut ElfStream, + kernel_virt_offset: u64, +) -> Result<(u64, u64), GenError> { + let mut start = u64::MAX; + let mut end = u64::MIN; + + for segment in elf.segments() { + if segment.p_type != PT_LOAD || segment.p_vaddr != segment.p_paddr + kernel_virt_offset { + continue; + } + + let aligned_start = segment.p_vaddr & !0xFFF; + let aligned_end = (segment.p_vaddr + segment.p_memsz + 0xFFF) & !0xFFF; + + if aligned_end > end { + end = aligned_end; + } + + if aligned_start < start { + start = aligned_start; + } + } + + if start < end { + Ok((start, end)) + } else { + Err(GenError::NoKernelImageRange) + } +} + +fn kernel_virt_offset(elf: &mut ElfStream) -> Result { + let (symtab, symstrtab) = elf + .symbol_table()? + .ok_or_else(|| GenError::MissingSection(".symtab"))?; + + for sym in symtab { + let name = symstrtab.get(sym.st_name as _)?; + + if name == "KERNEL_VIRT_OFFSET" { + // TODO symbol checks + return Ok(sym.st_value); + } + } + + Err(GenError::MissingSymbol("KERNEL_VIRT_OFFSET")) +} + +fn find_tables(elf: &mut ElfStream) -> Result<(u64, u64), GenError> { + let (shdrs, strtab) = elf.section_headers_with_strtab()?; + let strtab = strtab.ok_or_else(|| GenError::MissingSection(".strtab"))?; + + for shdr in shdrs { + let name = strtab.get(shdr.sh_name as _)?; + + if name == ".data.tables" { + // TODO section checks + return Ok((shdr.sh_offset, shdr.sh_addr)); + } + } + + Err(GenError::MissingSection(".data.tables")) +} + +fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> { + let mut elf = ElfStream::::open_stream(file)?; + + let kernel_virt_offset = kernel_virt_offset(&mut elf)?; + let (kernel_start, kernel_end) = kernel_image_range(&mut elf, kernel_virt_offset)?; + let (table_offset, table_virt_addr) = find_tables(&mut elf)?; + let table_physical_address = table_virt_addr + .checked_sub(kernel_virt_offset) + .ok_or_else(|| GenError::IncorrectTablesPlacement(table_virt_addr))?; + + println!("Kernel image range: {:#x?}", kernel_start..kernel_end); + println!("KERNEL_VIRT_OFFSET = {:#x}", kernel_virt_offset); + + match elf.ehdr.e_machine { + EM_X86_64 => X8664Builder::new( + elf, + GenData { + kernel_virt_offset, + kernel_start, + kernel_end, + table_offset, + table_physical_address, + }, + )? + .build(), + _ => todo!(), + } +} + +fn write_tables( + mut file: F, + offset: u64, + tables: FixedTables, +) -> Result<(), GenError> { + let bytes = bytemuck::bytes_of(&tables); + file.seek(SeekFrom::Start(offset))?; + file.write_all(bytes)?; + Ok(()) +} + +fn gentables>(image: P) -> Result<(), GenError> { + let mut file = OpenOptions::new() + .read(true) + .write(true) + .truncate(false) + .open(image)?; + + let (tables, file_offset) = build_tables(&mut file)?; + write_tables(file, file_offset, tables)?; + + Ok(()) +} + +fn main() -> ExitCode { + let args = Args::parse(); + + match gentables(&args.image) { + Ok(()) => ExitCode::SUCCESS, + Err(err) => { + eprintln!("{}: {}", args.image.display(), err); + ExitCode::FAILURE + } + } +} diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs new file mode 100644 index 00000000..8ce01464 --- /dev/null +++ b/tools/gentables/src/x86_64.rs @@ -0,0 +1,189 @@ +use core::fmt; +use std::{ + io::{Read, Seek}, + mem::offset_of, +}; + +use bitflags::bitflags; +use bytemuck::Zeroable; +use elf::{ + abi::{PF_W, PF_X, PT_LOAD}, + endian::AnyEndian, + ElfStream, +}; +use memtables::{FixedTables, KERNEL_L3_COUNT}; + +use crate::{GenData, GenError}; + +bitflags! { + #[derive(Clone, Copy)] + struct PageFlags: u64 { + const PRESENT = 1 << 0; + const WRITABLE = 1 << 1; + const NX = 1 << 63; + } +} + +pub struct X8664Builder { + elf: ElfStream, + data: GenData, + tables: FixedTables, + + l0i: usize, + l1i: usize, + start_l2i: usize, + end_l2i: usize, +} + +impl PageFlags { + fn from_elf(flags: u32) -> Self { + let mut out = Self::empty(); + if flags & PF_W != 0 { + out |= Self::WRITABLE; + } + if flags & PF_X == 0 { + out |= Self::NX; + } + out + } +} + +impl fmt::Display for PageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "r{}{}", + if self.contains(Self::WRITABLE) { + 'w' + } else { + '-' + }, + if self.contains(Self::NX) { '-' } else { 'x' } + ) + } +} + +impl X8664Builder { + pub fn new(elf: ElfStream, data: GenData) -> Result { + let l2_aligned_start = data.kernel_start & !0x1FFFFF; + let l2_aligned_end = (data.kernel_end + 0x1FFFFF) & !0x1FFFFF; + + if l2_aligned_end <= l2_aligned_start { + todo!(); + } + + if (l2_aligned_end - l2_aligned_start) as usize >= KERNEL_L3_COUNT * 0x200000 { + return Err(GenError::KernelTooLarge( + l2_aligned_start..l2_aligned_end, + l2_aligned_end - l2_aligned_start, + (KERNEL_L3_COUNT * 0x20000) as u64, + )); + } + + let l0i = (data.kernel_start >> 39) as usize & 0x1FF; + let l1i = (data.kernel_start >> 30) as usize & 0x1FF; + let start_l2i = (l2_aligned_start >> 21) as usize & 0x1FF; + let end_l2i = (l2_aligned_end >> 21) as usize & 0x1FF; + + Ok(Self { + elf, + data, + tables: FixedTables::zeroed(), + + l0i, + l1i, + start_l2i, + end_l2i, + }) + } + + pub fn build(mut self) -> Result<(FixedTables, u64), GenError> { + // L0 -> L1 + let l1_physical_address = + self.data.table_physical_address + offset_of!(FixedTables, kernel_l1) as u64; + self.tables.l0.data[self.l0i] = + l1_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); + + // L1 -> L2 + let l2_physical_address = + self.data.table_physical_address + offset_of!(FixedTables, kernel_l2) as u64; + self.tables.kernel_l1.data[self.l1i] = + l2_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); + + // L2 -> L3s + for i in 0..KERNEL_L3_COUNT { + let l3_physical_address = self.data.table_physical_address + + (offset_of!(FixedTables, kernel_l3s) + 0x1000 * i) as u64; + + self.tables.kernel_l2.data[i + self.start_l2i] = + l3_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); + } + + for (i, segment) in self.elf.segments().into_iter().enumerate() { + if segment.p_type != PT_LOAD + || segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset + { + continue; + } + + let aligned_virt_start = segment.p_vaddr & !0xFFF; + let aligned_virt_end = (segment.p_vaddr + segment.p_memsz + 0xFFF) & !0xFFF; + let aligned_phys_start = segment.p_paddr & !0xFFF; + let count = (aligned_virt_end - aligned_virt_start) / 0x1000; + + let flags = PageFlags::from_elf(segment.p_flags); + + println!( + "{}: {:#x?} -> {:#x} {}", + i, + aligned_virt_start..aligned_virt_end, + aligned_phys_start, + flags + ); + + Self::map_segment( + self.start_l2i, + &mut self.tables, + aligned_virt_start, + aligned_phys_start, + count as usize, + flags, + )?; + } + + Ok((self.tables, self.data.table_offset)) + } + + fn map_segment( + l2i_offset: usize, + tables: &mut FixedTables, + vaddr: u64, + paddr: u64, + count: usize, + flags: PageFlags, + ) -> Result<(), GenError> { + for index in 0..count { + let address = vaddr + index as u64 * 0x1000; + let page = paddr + index as u64 * 0x1000; + + let entry = page | (PageFlags::PRESENT | flags).bits(); + + let l2i = (address >> 21) as usize & 0x1FF - l2i_offset; + let l3i = (address >> 12) as usize & 0x1FF; + + let l3 = &mut tables.kernel_l3s[l2i]; + + if l3.data[l3i] != 0 { + if l3.data[l3i] != entry { + todo!(); + } else { + continue; + } + } + + l3.data[l3i] = entry; + } + + Ok(()) + } +} From 399e9531e7b680202e50c815681446531488c81a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 15 Sep 2023 00:02:14 +0300 Subject: [PATCH 073/211] x86-64: make SMP work with new mm --- src/arch/mod.rs | 7 ++- src/arch/x86_64/acpi.rs | 40 +++++++++----- src/arch/x86_64/apic/local.rs | 6 +-- src/arch/x86_64/boot/ap_boot.S | 4 +- src/arch/x86_64/boot/mod.rs | 75 +++++++++++--------------- src/arch/x86_64/cpu.rs | 10 +++- src/arch/x86_64/cpuid.rs | 81 +++++++++++++++++++++++++--- src/arch/x86_64/intrinsics.rs | 5 ++ src/arch/x86_64/mem/mod.rs | 30 +++++++++-- src/arch/x86_64/mem/table.rs | 4 +- src/arch/x86_64/mod.rs | 39 +++++++++----- src/arch/x86_64/smp.rs | 93 +++++++++++++++----------------- src/device/bus/pci/mod.rs | 9 ++-- src/device/bus/pci/space/ecam.rs | 64 +++++++++------------- src/device/mod.rs | 2 +- src/mem/address.rs | 30 +++++++++-- src/mem/device.rs | 14 ++++- src/mem/mod.rs | 30 ++++++++++- src/mem/phys/manager.rs | 8 +-- src/mem/pointer.rs | 37 ++++++++++++- src/mem/table.rs | 11 ++++ src/proc/elf.rs | 2 +- 22 files changed, 407 insertions(+), 194 deletions(-) diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 452d8c89..42ec4b16 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -27,7 +27,10 @@ use device_api::{ ResetDevice, }; -use crate::mem::{device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, PhysicalAddress}; +use crate::mem::{ + device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, table::KernelAddressSpace, + PhysicalAddress, +}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -75,7 +78,7 @@ pub trait Architecture { base: PhysicalAddress, size: usize, ) -> Result; - unsafe fn unmap_device_memory(&self, map: RawDeviceMemoryMapping); + unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping); fn map_physical_memory + Clone>( &self, diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 7234faf5..61e3b0e5 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -1,6 +1,7 @@ //! x86-64 implementation of ACPI management interfaces use core::{ alloc::{AllocError, Allocator, GlobalAlloc, Layout}, + mem::{align_of, size_of}, ptr::NonNull, sync::atomic::Ordering, time::Duration, @@ -23,7 +24,10 @@ use crate::{ x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, Architecture, CpuMessage, ARCHITECTURE, }, - mem::{address::FromRaw, heap::GLOBAL_HEAP, PhysicalAddress}, + mem::{ + address::FromRaw, heap::GLOBAL_HEAP, pointer::PhysicalRef, read_memory, write_memory, + PhysicalAddress, + }, sync::IrqSafeSpinlock, util, }; @@ -78,10 +82,14 @@ unsafe impl Allocator for AcpiAllocator { } impl acpi_system::Handler for AcpiHandlerImpl { - unsafe fn map_slice(address: u64, length: u64) -> &'static [u8] { - let slice = PhysicalAddress::from_raw(address).virtualize_slice::(length as usize); + type MappedSlice = PhysicalRef<'static, [u8]>; + + unsafe fn map_slice(address: u64, length: u64) -> Self::MappedSlice { + PhysicalRef::map_slice( + PhysicalAddress::from_raw(address), + length.try_into().unwrap(), + ) - todo!(); // PhysicalPointer::into_raw(slice) // if address + length < 0x100000000 { @@ -128,39 +136,47 @@ impl acpi_system::Handler for AcpiHandlerImpl { } fn mem_read_u8(address: u64) -> u8 { - todo!() + let value = unsafe { read_memory(PhysicalAddress::from_raw(address)) }; + log::trace!("mem_read_u8 {:#x} -> {:#x}", address, value); + value } fn mem_read_u16(address: u64) -> u16 { - todo!() + let value = unsafe { read_memory(PhysicalAddress::from_raw(address)) }; + log::trace!("mem_read_u16 {:#x} -> {:#x}", address, value); + value } fn mem_read_u32(address: u64) -> u32 { - todo!() + let value = unsafe { read_memory(PhysicalAddress::from_raw(address)) }; + log::trace!("mem_read_u32 {:#x} -> {:#x}", address, value); + value } fn mem_read_u64(address: u64) -> u64 { - todo!() + let value = unsafe { read_memory(PhysicalAddress::from_raw(address)) }; + log::trace!("mem_read_u64 {:#x} -> {:#x}", address, value); + value } fn mem_write_u8(address: u64, value: u8) { log::trace!("mem_write_u8 {:#x}, {:#x}", address, value); - todo!() + unsafe { write_memory(PhysicalAddress::from_raw(address), value) } } fn mem_write_u16(address: u64, value: u16) { log::trace!("mem_write_u16 {:#x}, {:#x}", address, value); - todo!() + unsafe { write_memory(PhysicalAddress::from_raw(address), value) } } fn mem_write_u32(address: u64, value: u32) { log::trace!("mem_write_u32 {:#x}, {:#x}", address, value); - todo!() + unsafe { write_memory(PhysicalAddress::from_raw(address), value) } } fn mem_write_u64(address: u64, value: u64) { log::trace!("mem_write_u64 {:#x}, {:#x}", address, value); - todo!() + unsafe { write_memory(PhysicalAddress::from_raw(address), value) } } fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> { diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 0ef9f1f3..3cb014e0 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -14,7 +14,7 @@ use tock_registers::{ use crate::{ arch::{ - x86_64::{registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT}, + x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT}, CpuMessage, }, mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, @@ -241,14 +241,14 @@ impl LocalApic { /// # Safety /// /// Unsafe: only meant to be called by the BSP during SMP init. - pub unsafe fn wakeup_cpu(&self, apic_id: u32, entry_vector: usize) { + pub unsafe fn wakeup_cpu(&self, apic_id: u32, entry_vector: PhysicalAddress) { infoln!("Waking up apic{}, entry = {:#x}", apic_id, entry_vector); while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { core::hint::spin_loop(); } - let entry_vector = entry_vector >> 12; + let entry_vector = entry_vector.page_index::(); // INIT assert self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); diff --git a/src/arch/x86_64/boot/ap_boot.S b/src/arch/x86_64/boot/ap_boot.S index aaa40571..25630dd5 100644 --- a/src/arch/x86_64/boot/ap_boot.S +++ b/src/arch/x86_64/boot/ap_boot.S @@ -46,10 +46,11 @@ ap_start_32: mov eax, dword [0x6000 + 0x00] mov cr3, eax - ; Enable EFER.LME + ; Enable EFER.LME + EFER.NXE mov ecx, 0xC0000080 rdmsr or eax, 1 << 8 + or eax, 1 << 11 wrmsr ; Enable paging @@ -79,6 +80,7 @@ ap_start_64: ; Jump to kernel entry mov rax, qword [0x6000 + 0x18] + jmp rax align 4 diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 284ee3d1..93f0c3d9 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,5 +1,5 @@ //! x86-64 boot and entry functions -use core::arch::global_asm; +use core::{arch::global_asm, sync::atomic::Ordering}; use tock_registers::interfaces::Writeable; use yboot_proto::{ @@ -8,9 +8,12 @@ use yboot_proto::{ }; use crate::{ - arch::{x86_64::registers::MSR_IA32_KERNEL_GS_BASE, Architecture, ArchitectureImpl}, + arch::{ + x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, + ArchitectureImpl, + }, fs::devfs, - kernel_main, + kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, task::runtime, }; @@ -59,43 +62,6 @@ static YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { res_size: 0, }, }; -// -// -// unsafe extern "C" fn __x86_64_upper_entry() -> ! { -// } -// -// /// Application processor entry point -// pub extern "C" fn __x86_64_ap_entry() -> ! { -// let cpu_id = CPU_COUNT.load(Ordering::Acquire); -// -// MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); -// unsafe { -// core::arch::asm!("swapgs"); -// } -// MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64); -// unsafe { -// core::arch::asm!("swapgs"); -// } -// -// // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base -// cpuid::feature_gate(); -// -// infoln!("cpu{} initializing", cpu_id); -// unsafe { -// ARCHITECTURE.init_mmu(false); -// core::arch::asm!("wbinvd"); -// -// // Cpu::init_local(LocalApic::new(), cpu_id as u32); -// // syscall::init_syscall(); -// exception::init_exceptions(cpu_id); -// -// ARCHITECTURE.init_platform(cpu_id); -// } -// -// CPU_COUNT.fetch_add(1, Ordering::Release); -// -// kernel_secondary_main() -// } unsafe fn init_dummy_cpu() { // TODO this is incorrect @@ -110,10 +76,6 @@ unsafe fn init_dummy_cpu() { core::arch::asm!("swapgs"); } -pub extern "C" fn __x86_64_ap_entry() -> ! { - loop {} -} - extern "C" fn __x86_64_upper_entry() -> ! { // Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry unsafe { @@ -147,6 +109,31 @@ extern "C" fn __x86_64_upper_entry() -> ! { kernel_main() } +/// Application processor entry point +pub extern "C" fn __x86_64_ap_entry() -> ! { + let cpu_id = CPU_COUNT.load(Ordering::Acquire); + + unsafe { + init_dummy_cpu(); + } + + // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base + + infoln!("cpu{} initializing", cpu_id); + + unsafe { + // Cpu::init_local(LocalApic::new(), cpu_id as u32); + // syscall::init_syscall(); + exception::init_exceptions(cpu_id); + + ARCHITECTURE.init_platform(cpu_id); + } + + CPU_COUNT.fetch_add(1, Ordering::Release); + + kernel_secondary_main() +} + global_asm!( r#" // {boot_data} diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 57c1a7e7..5759fe01 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -7,7 +7,12 @@ use tock_registers::interfaces::Writeable; use crate::{ arch::{ - x86_64::{gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, + x86_64::{ + cpuid::{self, PROCESSOR_FEATURES}, + gdt, + registers::MSR_IA32_KERNEL_GS_BASE, + syscall, + }, CpuMessage, }, sync::IrqSafeSpinlock, @@ -67,6 +72,9 @@ impl Cpu { pub unsafe fn init_local(local_apic: LocalApic, id: u32) { infoln!("Initialize CPU with id {}", id); + // Initialize CPU features + cpuid::enable_features(); + let tss_address = gdt::init(); let this = Box::new(Cpu { diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 40d88bcf..71d93742 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -2,6 +2,9 @@ use bitflags::bitflags; use kernel_util::util::OneTimeInit; +use tock_registers::interfaces::ReadWriteable; + +use super::registers::{CR4, XCR0}; bitflags! { pub struct ProcessorFeatures: u64 { @@ -9,7 +12,27 @@ bitflags! { } } -unsafe fn cpuid(eax: u32, result: &mut [u32]) { +bitflags! { + pub struct EcxFeatures: u32 { + const XSAVE = 1 << 26; + const AVX = 1 << 28; + } +} + +bitflags! { + pub struct EdxFeatures: u32 { + const FXSR = 1 << 24; + const PGE = 1 << 13; + } +} + +bitflags! { + pub struct ExtEdxFeatures: u32 { + const PDPE1GB = 1 << 26; + } +} + +unsafe fn raw_cpuid(eax: u32, result: &mut [u32]) { core::arch::asm!( r#" push %rbx @@ -25,19 +48,65 @@ unsafe fn cpuid(eax: u32, result: &mut [u32]) { ); } +fn cpuid_features() -> (EcxFeatures, EdxFeatures) { + let mut raw = [0; 3]; + + unsafe { + raw_cpuid(0x1, &mut raw); + } + + ( + EcxFeatures::from_bits_truncate(raw[2]), + EdxFeatures::from_bits_truncate(raw[1]), + ) +} + +fn cpuid_ext_features() -> ExtEdxFeatures { + let mut raw = [0; 3]; + + unsafe { + raw_cpuid(0x80000001, &mut raw); + } + + ExtEdxFeatures::from_bits_truncate(raw[1]) +} + pub static PROCESSOR_FEATURES: OneTimeInit = OneTimeInit::new(); pub fn init_cpuid() { let mut features = ProcessorFeatures::empty(); - let mut data = [0; 3]; - unsafe { - cpuid(0x80000001, &mut data); - } + let ext_edx = cpuid_ext_features(); - if data[1] & (1 << 26) != 0 { + if ext_edx.contains(ExtEdxFeatures::PDPE1GB) { features |= ProcessorFeatures::PDPE1GB; } PROCESSOR_FEATURES.init(features); } + +pub fn enable_features() { + let (ecx, edx) = cpuid_features(); + + if !ecx.contains(EcxFeatures::XSAVE) { + panic!("XSAVE feature is required"); + } + + if !edx.contains(EdxFeatures::FXSR) { + panic!("FXSR feature is required"); + } + + if !edx.contains(EdxFeatures::PGE) { + todo!("PGE feature (currently) is not optional"); + } + + CR4.modify(CR4::OSXSAVE::SET + CR4::OSFXSR::SET + CR4::PGE::SET); + + // XXX? SSE is supported on all x86-64s + XCR0.modify(XCR0::X87::SET + XCR0::SSE::SET); + + if ecx.contains(EcxFeatures::AVX) { + // Enable AVX + XCR0.modify(XCR0::AVX::SET); + } +} diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index cfffb596..6c11490f 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -129,3 +129,8 @@ pub unsafe fn outw(port: u16, value: u16) { pub unsafe fn outl(port: u16, value: u32) { core::arch::asm!("outl %eax, %dx", in("dx") port, in("eax") value, options(att_syntax)); } + +#[inline] +pub unsafe fn flush_tlb_entry(address: usize) { + core::arch::asm!("invlpg ({0})", in(reg) address, options(att_syntax)); +} diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index e7da3a31..177bf25d 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -11,11 +11,11 @@ use static_assertions::{const_assert_eq, const_assert_ne}; pub mod table; use crate::{ - arch::x86_64::mem::table::PageAttributes, + arch::x86_64::{intrinsics, mem::table::PageAttributes}, mem::{ address::{FromRaw, IntoRaw, KernelImageObject}, device::RawDeviceMemoryMapping, - table::EntryLevel, + table::{EntryLevel, KernelAddressSpace, MapAttributes}, PhysicalAddress, KERNEL_VIRT_OFFSET, }, }; @@ -166,7 +166,7 @@ pub(super) unsafe fn map_device_memory( base: PhysicalAddress, size: usize, ) -> Result { - debugln!("Map {}B @ {:#x}", size, base); + // debugln!("Map {}B @ {:#x}", size, base); let l3_aligned = base.page_align_down::(); let l3_offset = L3::page_offset(base.into_raw()); let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE; @@ -202,8 +202,26 @@ pub(super) unsafe fn map_device_memory( } } -pub(super) unsafe fn unmap_device_memory(map: RawDeviceMemoryMapping) { - loop {} +pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { + // debugln!( + // "Unmap {}B @ {:#x}", + // map.page_count * map.page_size, + // map.base_address + // ); + match map.page_size { + L3::SIZE => { + for i in 0..map.page_count { + let page = map.base_address + i * L3::SIZE; + let l2i = L2::index(page); + let l3i = L3::index(page); + assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; + intrinsics::flush_tlb_entry(page); + } + } + L2::SIZE => todo!(), + _ => unimplemented!(), + } } pub(super) unsafe fn map_heap_block(index: usize, page: PhysicalAddress) { @@ -313,6 +331,8 @@ pub unsafe fn init_fixed_tables() { KERNEL_TABLES.l0.data[RAM_MAPPING_L0I] = (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + // TODO ENABLE EFER.NXE + let cr3 = &KERNEL_TABLES.l0 as *const _ as usize - KERNEL_VIRT_OFFSET; core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); } diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index eebf68f9..3b73e228 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -7,6 +7,7 @@ use abi::error::Error; use bitflags::bitflags; use crate::{ + arch::x86_64::intrinsics, mem::{ address::{AsPhysicalAddress, FromRaw}, phys, @@ -401,9 +402,8 @@ impl AddressSpace { } l3[l3i] = entry; - unsafe { - core::arch::asm!("invlpg ({0})", in(reg) virt, options(att_syntax)); + intrinsics::flush_tlb_entry(virt); } Ok(()) diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 2e10cdfa..e417b8f2 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,7 +1,8 @@ +// TODO fix all TODOs use core::{mem::size_of, sync::atomic::Ordering}; use abi::error::Error; -use acpi_lib::{AcpiHandler, AcpiTable, AcpiTables, InterruptModel}; +use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use device_api::{ input::KeyboardProducer, interrupt::ExternalInterruptController, @@ -9,6 +10,7 @@ use device_api::{ }; use git_version::git_version; use kernel_util::util::OneTimeInit; +use memtables::FixedTables; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; mod acpi; @@ -34,6 +36,7 @@ use crate::{ debug::{self, LogLevel}, device::{ self, + bus::pci::PciBusManager, display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, tty::CombinedTerminal, }, @@ -61,7 +64,7 @@ use self::{ mem::{ init_fixed_tables, table::{PageAttributes, PageEntry, L1, L3}, - EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, + EarlyMapping, KERNEL_TABLES, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, }, peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort}, smp::CPU_COUNT, @@ -106,6 +109,20 @@ impl Architecture for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; type IrqNumber = IrqNumber; + unsafe fn start_application_processors(&self) { + if let Some(acpi) = self.acpi.try_get() { + let Some(pinfo) = acpi + .platform_info_in(AcpiAllocator) + .ok() + .and_then(|p| p.processor_info) + else { + return; + }; + + smp::start_ap_cores(&pinfo); + } + } + fn cpu_count() -> usize { CPU_COUNT.load(Ordering::Acquire) } @@ -143,7 +160,7 @@ impl Architecture for X86_64 { } #[inline] - unsafe fn unmap_device_memory(&self, map: RawDeviceMemoryMapping) { + unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) { mem::unmap_device_memory(map) } @@ -347,9 +364,7 @@ impl X86_64 { device::register_device(self.ioapic.get()); device::register_device(ps2); - // TODO setup PCI devices - } else { - loop {} + PciBusManager::setup_bus_devices().unwrap(); } } @@ -373,15 +388,13 @@ impl X86_64 { self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap()); - // TODO ACPI init // acpi::init_acpi(acpi).unwrap(); - // TODO MCFG - // if let Ok(mcfg) = acpi.find_table::() { - // for entry in mcfg.entries() { - // PciBusManager::add_segment_from_mcfg(entry).unwrap(); - // } - // } + if let Ok(mcfg) = acpi.find_table::() { + for entry in mcfg.entries() { + PciBusManager::add_segment_from_mcfg(entry).unwrap(); + } + } } unsafe fn init_framebuffer(&'static self) { diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index b2820842..e2687c38 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -1,19 +1,26 @@ //! x86-64 multiprocessing implementation -use core::{ - mem::size_of, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; use crate::{ arch::{ - x86_64::{boot::__x86_64_ap_entry, mem::KERNEL_TABLES}, + x86_64::{ + boot::__x86_64_ap_entry, + intrinsics::flush_tlb_entry, + mem::{ + table::{PageAttributes, L1, L2}, + KERNEL_TABLES, + }, + }, Architecture, ArchitectureImpl, }, mem::{ - address::{AsPhysicalAddress, IntoRaw}, + address::{AsPhysicalAddress, FromRaw, IntoRaw}, phys, + pointer::PhysicalRefMut, + table::{PageEntry, PageTable}, + PhysicalAddress, }, task::Cpu, }; @@ -26,61 +33,26 @@ pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86_64_ap_boot.bin")); const AP_STACK_PAGES: usize = 8; -const AP_BOOTSTRAP_DATA: usize = 0x6000; -const AP_BOOTSTRAP_CODE: usize = 0x7000; +const AP_BOOTSTRAP_DATA: PhysicalAddress = PhysicalAddress::from_raw(0x6000usize); +const AP_BOOTSTRAP_CODE: PhysicalAddress = PhysicalAddress::from_raw(0x7000usize); +const AP_ADDRESS_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(0x100000usize); +#[repr(C)] #[allow(dead_code)] struct ApBootstrapData { - cr3: usize, + cr3: PhysicalAddress, stack_base: usize, stack_size: usize, entry: usize, } -unsafe fn load_ap_bootstrap_code() { - let src_ptr = AP_BOOTSTRAP_BIN.as_ptr(); - let dst_ptr = AP_BOOTSTRAP_CODE as *mut u8; - - let size = AP_BOOTSTRAP_BIN.len(); - - assert!(size != 0, "Empty bootstrap code"); - assert!( - AP_BOOTSTRAP_CODE + size < 0x100000, - "Invalid bootstrap code placement: is not below 1MiB" - ); - - todo!(); - // let src_slice = core::slice::from_raw_parts(src_ptr, size); - // let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); - - // dst_slice.copy_from_slice(src_slice); -} - -unsafe fn load_ap_bootstrap_data(src: &ApBootstrapData) { - let src_ptr = src as *const _ as *const u8; - let dst_ptr = AP_BOOTSTRAP_DATA as *mut u8; - let size = size_of::(); - - assert!( - AP_BOOTSTRAP_DATA + size < 0x100000, - "Invalid bootstrap data placement: is not below 1MiB" - ); - - todo!() - // let src_slice = core::slice::from_raw_parts(src_ptr, size); - // let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size); - - // dst_slice.copy_from_slice(src_slice); - // core::arch::asm!("wbinvd"); -} - unsafe fn start_ap_core(apic_id: u32) { assert!(ArchitectureImpl::interrupt_mask()); let bsp_cpu = Cpu::local(); let bsp_apic = bsp_cpu.local_apic(); - let cr3 = KERNEL_TABLES.as_physical_address().into_raw(); + let cr3 = KERNEL_TABLES.as_physical_address(); let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES) .unwrap() .virtualize_raw(); @@ -93,11 +65,13 @@ unsafe fn start_ap_core(apic_id: u32) { entry: __x86_64_ap_entry as usize, }; - load_ap_bootstrap_data(&data); + let mut data_ref = PhysicalRefMut::::map(AP_BOOTSTRAP_DATA); + *data_ref = data; let cpu_count = CPU_COUNT.load(Ordering::Acquire); // Send an IPI to wake up the AP + core::arch::asm!("wbinvd"); bsp_apic.wakeup_cpu(apic_id, AP_BOOTSTRAP_CODE); while cpu_count == CPU_COUNT.load(Ordering::Acquire) { @@ -119,11 +93,32 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo) { return; } - load_ap_bootstrap_code(); + // Temporarily identity-map the lowest 2MiB + let mut identity_l1 = PageTable::::new_zeroed().unwrap(); + let mut identity_l2 = PageTable::::new_zeroed().unwrap(); + + identity_l1[0] = + PageEntry::::table(identity_l2.as_physical_address(), PageAttributes::WRITABLE); + identity_l2[0] = PageEntry::::block(PhysicalAddress::ZERO, PageAttributes::WRITABLE); + + assert_eq!(KERNEL_TABLES.l0.data[0], 0); + KERNEL_TABLES.l0.data[0] = IntoRaw::::into_raw(identity_l1.as_physical_address()) + | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + // Load AP_BOOTSTRAP_CODE + let mut code_ref = PhysicalRefMut::map_slice(AP_BOOTSTRAP_CODE, AP_BOOTSTRAP_BIN.len()); + code_ref.copy_from_slice(AP_BOOTSTRAP_BIN); for ap in aps.iter() { if ap.is_ap && ap.state == ProcessorState::WaitingForSipi { start_ap_core(ap.local_apic_id); } } + + // Remove the identity-map + identity_l2[0] = PageEntry::INVALID; + flush_tlb_entry(0); + KERNEL_TABLES.l0.data[0] = 0; + + // TODO drop the tables } diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index 5eb90692..c5ee8ad5 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -13,7 +13,10 @@ pub use space::{ ecam::PciEcam, PciConfigSpace, PciConfigurationSpace, PciLegacyConfigurationSpace, }; -use crate::sync::IrqSafeSpinlock; +use crate::{ + mem::{address::FromRaw, PhysicalAddress}, + sync::IrqSafeSpinlock, +}; bitflags! { /// Command register of the PCI configuration space @@ -77,7 +80,7 @@ pub struct PciBusSegment { segment_number: u8, bus_number_start: u8, bus_number_end: u8, - ecam_phys_base: Option, + ecam_phys_base: Option, devices: Vec, } @@ -192,7 +195,7 @@ impl PciBusManager { segment_number: entry.pci_segment_group as u8, bus_number_start: entry.bus_number_start, bus_number_end: entry.bus_number_end, - ecam_phys_base: Some(entry.base_address as usize), + ecam_phys_base: Some(PhysicalAddress::from_raw(entry.base_address)), devices: Vec::new(), }; diff --git a/src/device/bus/pci/space/ecam.rs b/src/device/bus/pci/space/ecam.rs index 6e8baccb..35781baf 100644 --- a/src/device/bus/pci/space/ecam.rs +++ b/src/device/bus/pci/space/ecam.rs @@ -1,7 +1,7 @@ //! PCI Express ECAM interface use yggdrasil_abi::error::Error; -use crate::mem::{device::DeviceMemory, ConvertAddress}; +use crate::mem::{device::DeviceMemoryMapping, PhysicalAddress}; use super::{PciAddress, PciConfigurationSpace}; @@ -9,27 +9,13 @@ use super::{PciAddress, PciConfigurationSpace}; #[derive(Debug)] #[repr(transparent)] pub struct PciEcam { - mapping: DeviceMemory, -} - -// Only used for probing -#[derive(Debug)] -#[repr(transparent)] -struct PciRawEcam { - virt_addr: usize, -} - -impl PciConfigurationSpace for PciRawEcam { - fn read_u32(&self, offset: usize) -> u32 { - assert_eq!(offset & 3, 0); - unsafe { ((self.virt_addr + offset) as *const u32).read_volatile() } - } + mapping: DeviceMemoryMapping, } impl PciConfigurationSpace for PciEcam { fn read_u32(&self, offset: usize) -> u32 { assert_eq!(offset & 3, 0); - unsafe { ((self.mapping.base() + offset) as *const u32).read_volatile() } + unsafe { ((self.mapping.address() + offset) as *const u32).read_volatile() } } } @@ -41,11 +27,9 @@ impl PciEcam { /// The `phys_addr` must be a valid ECAM address. The address must not alias any other mapped /// regions. The address must be aligned to a 4KiB boundary and be valid for accesses within a /// 4KiB-sized range. - pub unsafe fn map(phys_addr: usize) -> Result { - // TODO check align - let mapping = DeviceMemory::map("pcie-ecam", phys_addr, 0x1000)?; - - Ok(PciEcam { mapping }) + pub unsafe fn map(phys_addr: PhysicalAddress) -> Result { + let mapping = DeviceMemoryMapping::map(phys_addr, 0x1000)?; + Ok(Self { mapping }) } /// Checks if the ECAM contains a valid device configuration space, mapping and returning a @@ -55,29 +39,33 @@ impl PciEcam { /// /// See [PciEcam::map]. pub unsafe fn probe_raw_parts( - segment_phys_addr: usize, + segment_phys_addr: PhysicalAddress, bus_offset: u8, address: PciAddress, ) -> Result, Error> { - let phys_addr = segment_phys_addr - + ((address.bus - bus_offset) as usize * 256 + let phys_addr = segment_phys_addr.add( + ((address.bus - bus_offset) as usize * 256 + address.device as usize * 8 + address.function as usize) - * 0x1000; + * 0x1000, + ); + let this = Self::map(phys_addr)?; - if phys_addr + 0xFFF < 0x100000000 { - // Probe without allocating a mapping - let raw = PciRawEcam { - virt_addr: phys_addr.virtualize(), - }; + Ok(if this.is_valid() { Some(this) } else { None }) - if !raw.is_valid() { - return Ok(None); - } + // if phys_addr + 0xFFF < 0x100000000 { + // // Probe without allocating a mapping + // let raw = PciRawEcam { + // virt_addr: phys_addr.virtualize(), + // }; - Self::map(phys_addr).map(Some) - } else { - todo!() - } + // if !raw.is_valid() { + // return Ok(None); + // } + + // Self::map(phys_addr).map(Some) + // } else { + // todo!() + // } } } diff --git a/src/device/mod.rs b/src/device/mod.rs index 06ce75c2..1dbd0a6f 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -7,7 +7,7 @@ use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; #[cfg(target_arch = "aarch64")] pub mod devtree; -// pub mod bus; +pub mod bus; pub mod display; // pub mod power; pub mod serial; diff --git a/src/mem/address.rs b/src/mem/address.rs index 9636ebde..626ea3e6 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -2,9 +2,12 @@ use core::{ fmt, iter::Step, marker::PhantomData, + mem::align_of, ops::{Add, Deref, DerefMut, Sub}, }; +use bytemuck::{Pod, Zeroable}; + use crate::arch::{Architecture, ArchitectureImpl, ARCHITECTURE}; use super::{pointer::PhysicalPointer, table::EntryLevel, KERNEL_VIRT_OFFSET}; @@ -89,6 +92,15 @@ impl PhysicalAddress { Self((self.0 + L::SIZE as u64 - 1) & !(L::SIZE as u64 - 1)) } + pub const fn page_index(self) -> usize { + L::index(self.0 as usize) + } + + #[inline] + pub const fn is_aligned_for(self) -> bool { + self.0 as usize % align_of::() == 0 + } + pub unsafe fn from_virtualized(address: usize) -> Self { ArchitectureImpl::physicalize(address).unwrap() } @@ -97,12 +109,22 @@ impl PhysicalAddress { ArchitectureImpl::virtualize(self).unwrap() } - pub fn virtualize(self) -> PhysicalPointer { - loop {} + pub unsafe fn virtualize(self) -> PhysicalPointer { + if !self.is_aligned_for::() { + todo!(); + } + + let base = self.virtualize_raw(); + PhysicalPointer::from_raw(base as *mut T) } - pub fn virtualize_slice(self, len: usize) -> PhysicalPointer<[T]> { - loop {} + pub unsafe fn virtualize_slice(self, len: usize) -> PhysicalPointer<[T]> { + if !self.is_aligned_for::() { + todo!(); + } + + let base = self.virtualize_raw(); + PhysicalPointer::from_raw_parts(base as *mut T, len) } } diff --git a/src/mem/device.rs b/src/mem/device.rs index 988ff62f..2c3b1176 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -43,14 +43,24 @@ impl RawDeviceMemoryMapping { impl Drop for RawDeviceMemoryMapping { fn drop(&mut self) { - loop {} + unsafe { + ARCHITECTURE.unmap_device_memory(self); + } } } impl DeviceMemoryMapping { pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { let inner = RawDeviceMemoryMapping::map(base, size)?; - loop {} + let address = inner.address; + Ok(Self { + inner: Arc::new(inner), + address, + }) + } + + pub fn address(&self) -> usize { + self.address } } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 518a2048..94fcc0b3 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -15,7 +15,12 @@ // // pub mod device; -use core::{alloc::Layout, ffi::c_void, mem::size_of, ops::Add}; +use core::{ + alloc::Layout, + ffi::c_void, + mem::{align_of, size_of}, + ops::Add, +}; use abi::error::Error; @@ -30,10 +35,31 @@ pub mod table; pub use address::PhysicalAddress; -use self::table::AddressSpace; +use self::{device::DeviceMemoryMapping, table::AddressSpace}; pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; +pub unsafe fn read_memory(address: PhysicalAddress) -> T { + let io = DeviceMemoryMapping::map(address, size_of::()).unwrap(); + let address = io.address(); + + if address % align_of::() == 0 { + (address as *const T).read_volatile() + } else { + (address as *const T).read_unaligned() + } +} + +pub unsafe fn write_memory(address: PhysicalAddress, value: T) { + let io = DeviceMemoryMapping::map(address, size_of::()).unwrap(); + let address = io.address(); + + if address % align_of::() == 0 { + (address as *mut T).write_volatile(value) + } else { + (address as *mut T).write_unaligned(value) + } +} // pub mod phys; // // /// Kernel's physical load address diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index db28c645..5af2b58c 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -5,7 +5,7 @@ use abi::error::Error; use crate::mem::{ address::{FromRaw, IntoRaw}, - pointer::{PhysicalRef, PhysicalRefMut}, + pointer::PhysicalRefMut, PhysicalAddress, }; @@ -76,7 +76,7 @@ impl PhysicalMemoryManager { self.last_free_bit = 0; self.alloc_page() } else { - loop {} + Err(Error::OutOfMemory) } } @@ -102,7 +102,7 @@ impl PhysicalMemoryManager { self.last_free_bit = 0; self.alloc_2m_page() } else { - loop {} + Err(Error::OutOfMemory) } } @@ -127,7 +127,7 @@ impl PhysicalMemoryManager { self.last_free_bit = 0; self.alloc_contiguous_pages(count) } else { - loop {} + Err(Error::OutOfMemory) } } diff --git a/src/mem/pointer.rs b/src/mem/pointer.rs index 85c10fe1..d54b99a9 100644 --- a/src/mem/pointer.rs +++ b/src/mem/pointer.rs @@ -1,6 +1,7 @@ use core::{ alloc::Layout, fmt, + mem::align_of, ops::{Deref, DerefMut}, }; @@ -31,8 +32,37 @@ impl PhysicalPointer { } impl PhysicalPointer { + #[inline(always)] + pub fn is_aligned(&self) -> bool { + self.pointer.addr() % align_of::() == 0 + } + + #[inline(always)] + pub const unsafe fn from_raw(pointer: *mut T) -> PhysicalPointer { + PhysicalPointer { pointer } + } + + #[inline(always)] + pub unsafe fn from_raw_parts(base: *mut T, len: usize) -> PhysicalPointer<[T]> { + PhysicalPointer { + pointer: core::ptr::slice_from_raw_parts_mut(base, len), + } + } + pub unsafe fn write_unaligned(self, value: T) { - self.write_unaligned(value); + self.pointer.write_unaligned(value) + } + + pub unsafe fn write_volatile(self, value: T) { + self.pointer.write_volatile(value) + } + + pub unsafe fn read_unaligned(self) -> T { + self.pointer.read_unaligned() + } + + pub unsafe fn read_volatile(self) -> T { + self.pointer.read_volatile() } } @@ -96,6 +126,11 @@ impl<'a, T: Sized> PhysicalRef<'a, T> { let value = virtualize_raw(physical); PhysicalRef { value } } + + pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRef<'a, [T]> { + let value = virtualize_slice_raw(physical, len); + PhysicalRef { value } + } } impl PhysicalRef<'_, T> { diff --git a/src/mem/table.rs b/src/mem/table.rs index 50c21fb9..d15bc766 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -51,6 +51,17 @@ pub trait VirtualMemoryManager { fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error>; } +pub trait KernelAddressSpace { + type Mapping; + + fn map_page( + &self, + virt: usize, + physical: PhysicalAddress, + attrs: MapAttributes, + ) -> Result; +} + /// Interface for non-terminal tables to retrieve the next level of address translation tables pub trait NextPageTable { /// Type for the next-level page table diff --git a/src/proc/elf.rs b/src/proc/elf.rs index aeaff070..4948923a 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -97,7 +97,7 @@ where let phys_page = phys::alloc_page()?; space.map_page(virt_page, phys_page, attrs)?; - debugln!("Map {:#x} -> {:#x}", virt_page, phys_page); + // debugln!("Map {:#x} -> {:#x}", virt_page, phys_page); let dst_slice = unsafe { PhysicalRefMut::map_slice(phys_page.add(page_off), count) }; // let dst_slice = unsafe { From 5351ccf03820155a2b866fa8fde0d3024d3b4ae6 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 15 Sep 2023 00:09:26 +0300 Subject: [PATCH 074/211] x86-64: fix some loop {}s --- src/arch/x86_64/mem/mod.rs | 16 +++++++++------- src/arch/x86_64/mod.rs | 9 ++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 177bf25d..7aabcd9c 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -15,7 +15,7 @@ use crate::{ mem::{ address::{FromRaw, IntoRaw, KernelImageObject}, device::RawDeviceMemoryMapping, - table::{EntryLevel, KernelAddressSpace, MapAttributes}, + table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET, }, }; @@ -96,12 +96,12 @@ unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result= EARLY_MAPPING_OFFSET + L2::SIZE { - loop {} + panic!("Tried to unmap invalid early mapping: {:#x}", address); } let l3i = L3::index(address - EARLY_MAPPING_OFFSET); @@ -135,7 +135,8 @@ unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result Result { @@ -159,7 +160,8 @@ unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result::into_raw(memory_end) + (1 << 30) - 1) >> 30; if end_l1i > 512 { - loop {} + todo!( + "Cannot handle {}GiB of RAM", + end_l1i * L1::SIZE / (1024 * 1024 * 1024) + ); } MEMORY_LIMIT.init(memory_end.into_raw()); @@ -196,7 +199,7 @@ impl Architecture for X86_64 { Ok(()) } else { - loop {} + todo!(); } } @@ -235,7 +238,7 @@ impl Architecture for X86_64 { impl X86_64 { unsafe fn handle_ipi(&self, msg: CpuMessage) { - loop {} + todo!() } fn set_boot_data(&self, data: BootData) { From 03180a15618f39e7d0f6d530ab4460243d3d41d1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 15 Sep 2023 23:23:20 +0300 Subject: [PATCH 075/211] x86-64: make init return Result --- src/arch/x86_64/boot/mod.rs | 12 +++-- src/arch/x86_64/mod.rs | 88 ++++++++++++++++++++++--------------- src/mem/device.rs | 4 +- 3 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 93f0c3d9..9d97e756 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -89,7 +89,9 @@ extern "C" fn __x86_64_upper_entry() -> ! { // Setup memory management: kernel virtual memory tables, physical page manager and heap unsafe { - ARCHITECTURE.init_memory_management(); + ARCHITECTURE + .init_memory_management() + .expect("Could not initialize memory management"); } unsafe { @@ -103,7 +105,9 @@ extern "C" fn __x86_64_upper_entry() -> ! { // Initializes: local CPU, platform devices (timers/serials/etc), debug output unsafe { - ARCHITECTURE.init_platform(0); + ARCHITECTURE + .init_platform(0) + .expect("Could not initialize the platform"); } kernel_main() @@ -126,7 +130,9 @@ pub extern "C" fn __x86_64_ap_entry() -> ! { // syscall::init_syscall(); exception::init_exceptions(cpu_id); - ARCHITECTURE.init_platform(cpu_id); + ARCHITECTURE + .init_platform(cpu_id) + .expect("Could not initialize the platform (AP)"); } CPU_COUNT.fetch_add(1, Ordering::Release); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 41ea60f0..521b338c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -276,20 +276,19 @@ impl X86_64 { self.boot_data.init(data); } - unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) { + unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) -> Result<(), Error> { let mmap = EarlyMapping::::map_slice( PhysicalAddress::from_raw(data.memory_map.address), data.memory_map.len as usize, - ) - .unwrap(); + )?; phys::init_from_iter(mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion { base: PhysicalAddress::from_raw(reg.start_address), size: reg.page_count as usize * 0x1000, - })); + })) } - unsafe fn init_memory_management(&self) { + unsafe fn init_memory_management(&self) -> Result<(), Error> { const HEAP_PAGES: usize = 16; init_fixed_tables(); @@ -304,21 +303,23 @@ impl X86_64 { ); match self.boot_data.get() { - &BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data), + &BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data)?, } // Setup heap for i in 0..HEAP_PAGES { // Allocate in 2MiB chunks - let l2_page = phys::alloc_2m_page().unwrap(); + let l2_page = phys::alloc_2m_page()?; map_heap_block(i, l2_page); } heap::init_heap(HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE); + + Ok(()) } - unsafe fn init_platform(&'static self, cpu_id: usize) { + unsafe fn init_platform(&'static self, cpu_id: usize) -> Result<(), Error> { Cpu::init_local(LocalApic::new(), cpu_id as _); if cpu_id == 0 { @@ -329,7 +330,7 @@ impl X86_64 { } } - self.init_acpi_from_boot_data(); + self.init_acpi_from_boot_data()?; Self::disable_8259(); @@ -338,7 +339,7 @@ impl X86_64 { let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)))); debug::add_sink(com1_3.port_a(), LogLevel::Debug); - self.init_framebuffer(); + self.init_framebuffer()?; debug::init(); @@ -348,7 +349,7 @@ impl X86_64 { 0x64, 0x60, ))); - ps2.init().unwrap(); + ps2.init()?; infoln!( "Yggdrasil v{} ({})", @@ -357,75 +358,90 @@ impl X86_64 { ); if let Some(acpi) = self.acpi.try_get() { - self.init_platform_from_acpi(acpi); + self.init_platform_from_acpi(acpi)?; } - self.timer.get().init_irq().unwrap(); + self.timer.get().init_irq()?; ps2.connect(self.tty.get()); - ps2.init_irq().unwrap(); + ps2.init_irq()?; device::register_device(self.ioapic.get()); device::register_device(ps2); - PciBusManager::setup_bus_devices().unwrap(); + PciBusManager::setup_bus_devices()?; } + + Ok(()) } - unsafe fn init_acpi_from_boot_data(&self) { + unsafe fn init_acpi_from_boot_data(&self) -> Result<(), Error> { match self.boot_data.get() { &BootData::YBoot(data) => self.init_acpi_from_rsdp(data.rsdp_address as usize), } } - unsafe fn init_acpi_from_rsdp(&self, rsdp: usize) { - let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap(); + unsafe fn init_acpi_from_rsdp(&self, rsdp: usize) -> Result<(), Error> { + let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).map_err(|err| { + errorln!("Could not initialize ACPI tables: {:?}", err); + Error::InvalidArgument + })?; self.acpi.init(acpi_tables); + Ok(()) } - unsafe fn init_platform_from_acpi(&self, acpi: &'static AcpiTables) { - let platform_info = acpi.platform_info_in(AcpiAllocator).unwrap(); + unsafe fn init_platform_from_acpi( + &self, + acpi: &'static AcpiTables, + ) -> Result<(), Error> { + let platform_info = acpi.platform_info_in(AcpiAllocator).map_err(|err| { + errorln!("Could not get ACPI platform info: {:?}", err); + Error::InvalidArgument + })?; let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else { panic!("The processor does not support APIC"); }; - self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap()); + self.ioapic.init(IoApic::from_acpi(&apic_info)?); // acpi::init_acpi(acpi).unwrap(); if let Ok(mcfg) = acpi.find_table::() { for entry in mcfg.entries() { - PciBusManager::add_segment_from_mcfg(entry).unwrap(); + PciBusManager::add_segment_from_mcfg(entry)?; } } + + Ok(()) } - unsafe fn init_framebuffer(&'static self) { + unsafe fn init_framebuffer(&'static self) -> Result<(), Error> { match self.boot_data.get() { &BootData::YBoot(data) => { let info = &data.opt_framebuffer; - self.framebuffer.init( - LinearFramebuffer::from_physical_bits( - PhysicalAddress::from_raw(info.res_address), - info.res_size as usize, - info.res_stride as usize, - info.res_width, - info.res_height, - ) - .unwrap(), - ); + self.framebuffer.init(LinearFramebuffer::from_physical_bits( + PhysicalAddress::from_raw(info.res_address), + info.res_size as usize, + info.res_stride as usize, + info.res_width, + info.res_height, + )?); } } - self.fbconsole - .init(FramebufferConsole::from_framebuffer(self.framebuffer.get(), None).unwrap()); + self.fbconsole.init(FramebufferConsole::from_framebuffer( + self.framebuffer.get(), + None, + )?); debug::add_sink(self.fbconsole.get(), LogLevel::Info); self.tty.init(CombinedTerminal::new(self.fbconsole.get())); - devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular).unwrap(); + devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular)?; console::add_console_autoflush(self.fbconsole.get()); + + Ok(()) } fn init_initrd(initrd_start: PhysicalAddress, initrd_end: PhysicalAddress) { diff --git a/src/mem/device.rs b/src/mem/device.rs index 2c3b1176..df7f1b9f 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -1,5 +1,7 @@ //! Facilities for mapping devices to virtual address space -use core::{marker::PhantomData, mem::size_of, ops::Deref, sync::atomic::AtomicUsize}; +// TODO +#![allow(unused)] +use core::{mem::size_of, ops::Deref}; use abi::error::Error; use alloc::sync::Arc; From 6cedfa7c4a30a1175390430a50c08b13b5f719da Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 1 Oct 2023 16:58:46 +0300 Subject: [PATCH 076/211] x86-64/mm: better VMA allocator --- Cargo.toml | 1 + lib/vmalloc/Cargo.toml | 13 ++ lib/vmalloc/src/allocator.rs | 349 +++++++++++++++++++++++++++++++++ lib/vmalloc/src/lib.rs | 64 ++++++ src/arch/mod.rs | 6 +- src/arch/x86_64/acpi.rs | 12 -- src/arch/x86_64/apic/ioapic.rs | 6 +- src/arch/x86_64/boot/mod.rs | 5 +- src/arch/x86_64/context.rs | 5 +- src/arch/x86_64/cpu.rs | 7 +- src/arch/x86_64/mem/mod.rs | 6 +- src/arch/x86_64/mem/process.rs | 136 +++++++++++++ src/arch/x86_64/mem/table.rs | 173 +++------------- src/arch/x86_64/mod.rs | 12 +- src/arch/x86_64/smp.rs | 4 +- src/device/display/console.rs | 2 +- src/fs/mod.rs | 2 +- src/mem/address.rs | 13 +- src/mem/heap.rs | 2 +- src/mem/mod.rs | 82 ++------ src/mem/phys/manager.rs | 4 +- src/mem/phys/mod.rs | 150 +------------- src/mem/phys/reserved.rs | 9 +- src/mem/pointer.rs | 1 - src/mem/process.rs | 232 ++++++++++++++++++++++ src/mem/table.rs | 45 ----- src/proc/elf.rs | 59 +++--- src/proc/exec.rs | 35 ++-- src/syscall/mod.rs | 17 +- src/task/process.rs | 11 +- 30 files changed, 933 insertions(+), 530 deletions(-) create mode 100644 lib/vmalloc/Cargo.toml create mode 100644 lib/vmalloc/src/allocator.rs create mode 100644 lib/vmalloc/src/lib.rs create mode 100644 src/arch/x86_64/mem/process.rs create mode 100644 src/mem/process.rs diff --git a/Cargo.toml b/Cargo.toml index 4abfd002..4f800e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ memfs = { path = "lib/memfs" } device-api = { path = "lib/device-api", features = ["derive"] } kernel-util = { path = "lib/kernel-util" } memtables = { path = "lib/memtables" } +vmalloc = { path = "lib/vmalloc" } atomic_enum = "0.2.0" bitflags = "2.3.3" diff --git a/lib/vmalloc/Cargo.toml b/lib/vmalloc/Cargo.toml new file mode 100644 index 00000000..a5062b55 --- /dev/null +++ b/lib/vmalloc/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "vmalloc" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } + +[dev-dependencies] +itertools = "0.11.0" +proptest = "1.2.0" diff --git a/lib/vmalloc/src/allocator.rs b/lib/vmalloc/src/allocator.rs new file mode 100644 index 00000000..43398b25 --- /dev/null +++ b/lib/vmalloc/src/allocator.rs @@ -0,0 +1,349 @@ +use core::cmp::Ordering; + +use alloc::collections::{linked_list::CursorMut, LinkedList}; +use yggdrasil_abi::error::Error; + +use crate::VirtualMemoryRange; + +#[derive(PartialEq, Clone, Debug, Copy)] +struct AllocatorNode { + range: VirtualMemoryRange, + used: bool, +} + +impl PartialOrd for AllocatorNode { + fn partial_cmp(&self, other: &Self) -> Option { + self.range.start_pfn.partial_cmp(&other.range.start_pfn) + } +} + +pub struct TreeAllocator { + ranges: LinkedList, +} + +impl AllocatorNode { + pub const fn free(start_pfn: usize, end_pfn: usize) -> Self { + Self { + range: VirtualMemoryRange { start_pfn, end_pfn }, + used: false, + } + } + + #[cfg(test)] + pub const fn used(start_pfn: usize, end_pfn: usize) -> Self { + Self { + range: VirtualMemoryRange { start_pfn, end_pfn }, + used: true, + } + } + + #[inline] + pub const fn pfn_count(&self) -> usize { + self.range.end_pfn - self.range.start_pfn + } +} + +impl TreeAllocator { + pub fn new(start_pfn: usize, end_pfn: usize) -> Self { + let mut ranges = LinkedList::new(); + ranges.push_back(AllocatorNode::free(start_pfn, end_pfn)); + Self { ranges } + } + + fn find_region_mut bool>(&mut self, f: F) -> CursorMut { + let mut cursor = self.ranges.cursor_front_mut(); + while let Some(range) = cursor.current() { + if f(range) { + break; + } + cursor.move_next(); + } + cursor + } + + fn coalesce_regions(&mut self) { + let mut cursor = self.ranges.cursor_front_mut(); + + loop { + let Some(&mut next) = cursor.peek_next() else { + break; + }; + let current = cursor.current().unwrap(); + + if current.used == next.used { + debug_assert_eq!(current.range.end_pfn, next.range.start_pfn); + current.range.end_pfn = next.range.end_pfn; + + cursor.move_next(); + cursor.remove_current(); + cursor.move_prev(); + } else { + cursor.move_next(); + } + } + } + + fn set_range( + &mut self, + start_pfn: usize, + pfn_count: usize, + old_state: bool, + new_state: bool, + ) -> Result<(), Error> { + let insert = VirtualMemoryRange { + start_pfn, + end_pfn: start_pfn + pfn_count, + }; + let mut cursor = self.find_region_mut(|r| r.used == old_state && r.range.contains(&insert)); + let range = cursor.current().ok_or(Error::AlreadyExists)?; + + let start_pfn = range.range.start_pfn; + let end_pfn = range.range.end_pfn; + + match (insert.start_pfn == start_pfn, insert.end_pfn == end_pfn) { + // No split + (true, true) => { + range.used = new_state; + } + // Split start + (true, false) => { + range.used = new_state; + range.range.end_pfn = insert.end_pfn; + + cursor.insert_after(AllocatorNode { + range: VirtualMemoryRange { + start_pfn: insert.end_pfn, + end_pfn, + }, + used: old_state, + }); + } + // Split end + (false, true) => { + range.range.end_pfn = insert.start_pfn; + + cursor.insert_after(AllocatorNode { + range: VirtualMemoryRange { + start_pfn: insert.start_pfn, + end_pfn, + }, + used: new_state, + }); + } + // Split in the middle + (false, false) => { + range.range = insert; + range.used = new_state; + + cursor.insert_after(AllocatorNode { + range: VirtualMemoryRange { + start_pfn: insert.end_pfn, + end_pfn, + }, + used: old_state, + }); + cursor.insert_before(AllocatorNode { + range: VirtualMemoryRange { + start_pfn, + end_pfn: insert.start_pfn, + }, + used: old_state, + }); + } + } + + self.coalesce_regions(); + + Ok(()) + } + + pub fn insert(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { + self.set_range(start_pfn, pfn_count, false, true) + } + + pub fn free(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { + self.set_range(start_pfn, pfn_count, true, false) + } + + pub fn allocate(&mut self, pfn_count: usize) -> Option { + let mut cursor = self.find_region_mut(|r| !r.used && r.pfn_count() >= pfn_count); + let range = cursor.current()?; + + let start_pfn = range.range.start_pfn; + let end_pfn = range.range.end_pfn; + + range.used = true; + + if range.pfn_count() > pfn_count { + range.range.end_pfn = start_pfn + pfn_count; + + // Split the range + cursor.insert_after(AllocatorNode::free(start_pfn + pfn_count, end_pfn)); + } + + self.coalesce_regions(); + + Some(start_pfn) + } + + pub fn ranges(&self) -> impl Iterator + '_ { + self.ranges.iter().map(|r| (r.used, r.range)) + } +} + +#[cfg(test)] +mod tests { + use alloc::collections::LinkedList; + + use super::{AllocatorNode, TreeAllocator}; + + extern crate std; + + #[test] + fn deallocation() { + let ranges = LinkedList::from_iter([ + AllocatorNode::free(0, 12), + AllocatorNode::used(12, 24), + AllocatorNode::free(24, 32), + AllocatorNode::used(32, 64), + AllocatorNode::free(64, 128), + ]); + let mut alloc = TreeAllocator { ranges }; + + // No-split dealloc + assert_eq!(alloc.free(12, 12), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::free(0, 32), + AllocatorNode::used(32, 64), + AllocatorNode::free(64, 128), + ]); + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Split at the start dealloc + assert_eq!(alloc.free(32, 8), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::free(0, 40), + AllocatorNode::used(40, 64), + AllocatorNode::free(64, 128), + ]); + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Split at the end dealloc + assert_eq!(alloc.free(56, 8), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::free(0, 40), + AllocatorNode::used(40, 56), + AllocatorNode::free(56, 128), + ]); + + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Split in the middle + assert_eq!(alloc.free(42, 4), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::free(0, 40), + AllocatorNode::used(40, 42), + AllocatorNode::free(42, 46), + AllocatorNode::used(46, 56), + AllocatorNode::free(56, 128), + ]); + + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Whole region free + assert_eq!(alloc.free(40, 2), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::free(0, 46), + AllocatorNode::used(46, 56), + AllocatorNode::free(56, 128), + ]); + + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + assert_eq!(alloc.free(46, 10), Ok(())); + let expected = LinkedList::from_iter([AllocatorNode::free(0, 128)]); + + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + } + + #[test] + fn allocation() { + let ranges = LinkedList::from_iter([ + AllocatorNode::free(0, 12), + AllocatorNode::used(12, 24), + AllocatorNode::free(24, 32), + AllocatorNode::used(32, 64), + ]); + let mut alloc = TreeAllocator { ranges }; + + // Non-splitting allocation + assert_eq!(alloc.allocate(12), Some(0)); + + // Splitting allocation + assert_eq!(alloc.allocate(4), Some(24)); + + // Non-splitting allocation + assert_eq!(alloc.allocate(4), Some(28)); + + // Out of memory + assert_eq!(alloc.allocate(1), None); + + let expected = LinkedList::from_iter([AllocatorNode::used(0, 64)]); + + itertools::assert_equal(alloc.ranges, expected); + } + + #[test] + fn insertion() { + let ranges = LinkedList::from_iter([ + AllocatorNode::free(0, 12), + AllocatorNode::used(12, 24), + AllocatorNode::free(24, 32), + AllocatorNode::used(32, 64), + AllocatorNode::free(64, 128), + ]); + let mut alloc = TreeAllocator { ranges }; + + // No split + assert_eq!(alloc.insert(0, 12), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::used(0, 24), + AllocatorNode::free(24, 32), + AllocatorNode::used(32, 64), + AllocatorNode::free(64, 128), + ]); + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Split at the start + assert_eq!(alloc.insert(24, 4), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::used(0, 28), + AllocatorNode::free(28, 32), + AllocatorNode::used(32, 64), + AllocatorNode::free(64, 128), + ]); + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Split at the end + assert_eq!(alloc.insert(30, 2), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::used(0, 28), + AllocatorNode::free(28, 30), + AllocatorNode::used(30, 64), + AllocatorNode::free(64, 128), + ]); + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + + // Split in the middle + assert_eq!(alloc.insert(72, 16), Ok(())); + let expected = LinkedList::from_iter([ + AllocatorNode::used(0, 28), + AllocatorNode::free(28, 30), + AllocatorNode::used(30, 64), + AllocatorNode::free(64, 72), + AllocatorNode::used(72, 88), + AllocatorNode::free(88, 128), + ]); + itertools::assert_equal(alloc.ranges.iter(), expected.iter()); + } +} diff --git a/lib/vmalloc/src/lib.rs b/lib/vmalloc/src/lib.rs new file mode 100644 index 00000000..b7b6b04f --- /dev/null +++ b/lib/vmalloc/src/lib.rs @@ -0,0 +1,64 @@ +#![no_std] +#![feature(linked_list_cursors, let_chains, btree_extract_if)] + +extern crate alloc; + +use allocator::TreeAllocator; +use yggdrasil_abi::error::Error; + +pub(crate) mod allocator; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct VirtualMemoryRange { + start_pfn: usize, + end_pfn: usize, +} + +pub struct VirtualMemoryAllocator { + inner: TreeAllocator, +} + +impl VirtualMemoryRange { + pub fn start_pfn(&self) -> usize { + self.start_pfn + } + + pub fn end_pfn(&self) -> usize { + self.end_pfn + } + + pub fn pfn_count(&self) -> usize { + self.end_pfn - self.start_pfn + } + + pub fn contains(&self, other: &Self) -> bool { + other.start_pfn >= self.start_pfn && other.end_pfn <= self.end_pfn + } +} + +impl VirtualMemoryAllocator { + pub fn new(start_pfn: usize, end_pfn: usize) -> Self { + Self { + inner: TreeAllocator::new(start_pfn, end_pfn), + } + } + + pub fn allocate(&mut self, pfn_count: usize) -> Result { + self.inner.allocate(pfn_count).ok_or(Error::OutOfMemory) + } + + pub fn free(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { + self.inner.free(start_pfn, pfn_count) + } + + pub fn insert(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { + self.inner.insert(start_pfn, pfn_count) + } + + pub fn ranges(&self) -> impl Iterator + '_ { + self.inner.ranges() + } +} + +#[cfg(test)] +mod tests {} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 42ec4b16..956f15c4 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -27,10 +27,7 @@ use device_api::{ ResetDevice, }; -use crate::mem::{ - device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, table::KernelAddressSpace, - PhysicalAddress, -}; +use crate::mem::{device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, PhysicalAddress}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -58,6 +55,7 @@ pub enum CpuMessage { } /// Interface for an architecture-specific facilities +#[allow(unused)] pub trait Architecture { /// Address, to which "zero" address is mapped in the virtual address space const KERNEL_VIRT_OFFSET: usize; diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 61e3b0e5..80ce12d6 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -1,7 +1,6 @@ //! x86-64 implementation of ACPI management interfaces use core::{ alloc::{AllocError, Allocator, GlobalAlloc, Layout}, - mem::{align_of, size_of}, ptr::NonNull, sync::atomic::Ordering, time::Duration, @@ -89,17 +88,6 @@ impl acpi_system::Handler for AcpiHandlerImpl { PhysicalAddress::from_raw(address), length.try_into().unwrap(), ) - - // PhysicalPointer::into_raw(slice) - - // if address + length < 0x100000000 { - // core::slice::from_raw_parts( - // (address as usize).virtualize() as *const u8, - // length as usize, - // ) - // } else { - // panic!("Unhandled address: {:#x}", address) - // } } fn io_read_u8(port: u16) -> u8 { diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index a028c3f6..93d8f3d2 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -16,11 +16,7 @@ use tock_registers::{ use crate::{ arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}, - mem::{ - address::FromRaw, - device::{DeviceMemoryIo, DeviceMemoryMapping}, - PhysicalAddress, - }, + mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, sync::IrqSafeSpinlock, }; diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 9d97e756..679dafdc 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -8,10 +8,7 @@ use yboot_proto::{ }; use crate::{ - arch::{ - x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, - ArchitectureImpl, - }, + arch::x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, fs::devfs, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 17e59cb1..e26c13f6 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -90,8 +90,7 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 32; - let stack_base = - unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() }; + let stack_base = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -123,7 +122,7 @@ impl TaskContextImpl for TaskContext { ) -> Result { const USER_TASK_PAGES: usize = 8; - let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() }; + let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 5759fe01..b0f548e7 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -7,12 +7,7 @@ use tock_registers::interfaces::Writeable; use crate::{ arch::{ - x86_64::{ - cpuid::{self, PROCESSOR_FEATURES}, - gdt, - registers::MSR_IA32_KERNEL_GS_BASE, - syscall, - }, + x86_64::{cpuid, gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, CpuMessage, }, sync::IrqSafeSpinlock, diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 7aabcd9c..b52bf5ff 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -8,6 +8,7 @@ use kernel_util::util::OneTimeInit; use memtables::FixedTables; use static_assertions::{const_assert_eq, const_assert_ne}; +pub mod process; pub mod table; use crate::{ @@ -69,9 +70,6 @@ pub(super) const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPI pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); pub(super) static mut RAM_MAPPING_L1: PageTable = PageTable::zeroed(); -// Global limits -pub(super) const HEAP_SIZE_LIMIT: usize = L1::SIZE; - // Early mappings unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { for l3i in 0..512 { @@ -189,8 +187,6 @@ pub(super) unsafe fn map_device_memory( page_size: L2::SIZE, }) } else { - let page_size = L3::SIZE; - // Just map the pages directly let base_address = map_device_memory_l3(l3_aligned, page_count)?; let address = base_address + l3_offset; diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs new file mode 100644 index 00000000..cde7e096 --- /dev/null +++ b/src/arch/x86_64/mem/process.rs @@ -0,0 +1,136 @@ +use yggdrasil_abi::error::Error; + +use crate::{ + arch::x86_64::intrinsics, + mem::{ + address::AsPhysicalAddress, + phys, + pointer::PhysicalRefMut, + process::ProcessAddressSpaceManager, + table::{EntryLevel, MapAttributes, NextPageTable}, + PhysicalAddress, + }, +}; + +use super::{ + clone_kernel_tables, + table::{PageEntry, PageTable, L0, L1, L2, L3}, +}; + +/// Represents a process or kernel address space. Because x86-64 does not have cool stuff like +/// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space. +#[repr(C)] +pub struct ProcessAddressSpaceImpl { + l0: PhysicalRefMut<'static, PageTable>, +} + +impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { + const PAGE_SIZE: usize = L3::SIZE; + const LOWER_LIMIT_PFN: usize = 8; + // 16GiB VM limit + const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE; + + fn new() -> Result { + let mut l0 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; + + for i in 0..512 { + l0[i] = PageEntry::INVALID; + } + + clone_kernel_tables(&mut *l0); + + Ok(Self { l0 }) + } + + #[inline] + unsafe fn map_page( + &mut self, + address: usize, + physical: PhysicalAddress, + flags: MapAttributes, + ) -> Result<(), Error> { + self.write_l3_entry(address, PageEntry::page(physical, flags.into()), false) + } + + unsafe fn unmap_page(&mut self, address: usize) -> Result { + self.pop_l3_entry(address) + } + + #[inline] + fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> { + self.read_l3_entry(address) + .ok_or(Error::InvalidMemoryOperation) + } +} + +impl ProcessAddressSpaceImpl { + // Write a single 4KiB entry + fn write_l3_entry( + &mut self, + virt: usize, + entry: PageEntry, + overwrite: bool, + ) -> Result<(), Error> { + let l0i = L0::index(virt); + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let mut l1 = self.l0.get_mut_or_alloc(l0i)?; + let mut l2 = l1.get_mut_or_alloc(l1i)?; + let mut l3 = l2.get_mut_or_alloc(l2i)?; + + if l3[l3i].is_present() && !overwrite { + todo!(); + } + + l3[l3i] = entry; + unsafe { + intrinsics::flush_tlb_entry(virt); + } + + Ok(()) + } + + fn pop_l3_entry(&mut self, virt: usize) -> Result { + let l0i = L0::index(virt); + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + // TODO somehow drop tables if they're known to be empty? + let mut l1 = self.l0.get_mut(l0i).ok_or(Error::DoesNotExist)?; + let mut l2 = l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?; + + l3[l3i] = PageEntry::INVALID; + unsafe { + intrinsics::flush_tlb_entry(virt); + } + + Ok(page) + } + + fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { + let l0i = L0::index(virt); + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l1 = self.l0.get(l0i)?; + let l2 = l1.get(l1i)?; + let l3 = l2.get(l2i)?; + + let page = l3[l3i].as_page()?; + + Some((page, l3[l3i].attributes().into())) + } +} + +impl AsPhysicalAddress for ProcessAddressSpaceImpl { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + self.l0.as_physical_address() + } +} diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index 3b73e228..a70084f7 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -6,22 +6,14 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -use crate::{ - arch::x86_64::intrinsics, - mem::{ - address::{AsPhysicalAddress, FromRaw}, - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - table::{ - EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, - }, - PhysicalAddress, - }, - sync::IrqSafeSpinlock, +use crate::mem::{ + address::{AsPhysicalAddress, FromRaw}, + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, + PhysicalAddress, }; -use super::{clone_kernel_tables, KERNEL_TABLES}; - bitflags! { /// Describes how each page table entry is mapped pub struct PageAttributes: u64 { @@ -39,15 +31,8 @@ bitflags! { } } -/// Represents a process or kernel address space. Because x86-64 does not have cool stuff like -/// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space. -#[repr(C)] -pub struct AddressSpace { - inner: IrqSafeSpinlock>>, -} - /// Represents a single virtual address space mapping depending on its translation level -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] #[repr(transparent)] pub struct PageEntry(u64, PhantomData); @@ -59,16 +44,16 @@ pub struct PageTable { } /// Translation level 0 (PML4): Entry is 512GiB table -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct L0; /// Translation level 1 (PDPT): Entry is 1GiB table -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct L1; /// Translation level 2 (Page directory): Entry is 2MiB block/table -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct L2; /// Translation level 3 (Page table): Entry is 4KiB page -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct L3; impl NonTerminalEntryLevel for L0 { @@ -193,6 +178,10 @@ impl PageEntry { None } } + + pub fn is_block(self) -> bool { + self.0 & PageAttributes::BLOCK.bits() != 0 + } } impl PageEntry { @@ -203,6 +192,10 @@ impl PageEntry { Self(raw, PhantomData) } + pub fn attributes(&self) -> PageAttributes { + PageAttributes::from_bits_retain(self.0) + } + /// Returns `true` if the entry contains a valid mapping to either a table or to a page/block pub fn is_present(&self) -> bool { self.0 & PageAttributes::PRESENT.bits() != 0 @@ -292,125 +285,17 @@ impl From for PageAttributes { } } -impl VirtualMemoryManager for AddressSpace { - fn allocate( - &self, - hint: Option, - len: usize, - attrs: MapAttributes, - ) -> Result { - if hint.is_some() { - todo!(); - } - - const TRY_ALLOC_START: usize = 0x100000000; - const TRY_ALLOC_END: usize = 0xF00000000; - - 'l0: for base in (TRY_ALLOC_START..TRY_ALLOC_END - len * 0x1000).step_by(0x1000) { - for i in 0..len { - if self.translate(base + i * 0x1000).is_some() { - continue 'l0; - } - } - - for i in 0..len { - let page = phys::alloc_page()?; - self.map_page(base + i * 0x1000, page, attrs)?; - } - - return Ok(base); - } - - Err(Error::OutOfMemory) - } - - fn map_page( - &self, - virt: usize, - phys: PhysicalAddress, - attrs: MapAttributes, - ) -> Result<(), Error> { - self.write_entry(virt, PageEntry::page(phys, attrs.into()), true) - } - - fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> { - for page in (addr..addr + len).step_by(0x1000) { - let Some(phys) = self.translate(page) else { - todo!(); - }; - - self.write_entry(page, PageEntry::INVALID, true)?; - unsafe { - phys::free_page(phys); +impl From for MapAttributes { + fn from(value: PageAttributes) -> Self { + let mut res = MapAttributes::empty(); + if value.contains(PageAttributes::USER) { + res |= MapAttributes::USER_READ; + if value.contains(PageAttributes::WRITABLE) { + res |= MapAttributes::USER_WRITE; } } - - Ok(()) - } -} - -impl AddressSpace { - /// Allocates an empty address space with all entries marked as non-present - pub fn new_empty() -> Result { - let mut l0 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; - - for i in 0..512 { - unsafe { - l0[i] = PageEntry::INVALID; - } - } - - clone_kernel_tables(&mut *l0); - - Ok(Self { - inner: IrqSafeSpinlock::new(l0), - }) - } - - // TODO return page size and attributes - /// Returns the physical address to which the `virt` address is mapped - pub fn translate(&self, virt: usize) -> Option { - let l0 = self.inner.lock(); - - let l0i = L0::index(virt); - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); - - let l1 = l0.get(l0i)?; - let l2 = l1.get(l1i)?; - let l3 = l2.get(l2i)?; - - l3[l3i].as_page() - } - - // Write a single 4KiB entry - fn write_entry(&self, virt: usize, entry: PageEntry, overwrite: bool) -> Result<(), Error> { - let mut l0 = self.inner.lock(); - - let l0i = L0::index(virt); - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); - - let mut l1 = l0.get_mut_or_alloc(l0i)?; - let mut l2 = l1.get_mut_or_alloc(l1i)?; - let mut l3 = l2.get_mut_or_alloc(l2i)?; - - if l3[l3i].is_present() && !overwrite { - todo!(); - } - - l3[l3i] = entry; - unsafe { - intrinsics::flush_tlb_entry(virt); - } - - Ok(()) - } - - /// Returns the physical address of the root table - pub fn physical_address(&self) -> PhysicalAddress { - unsafe { self.inner.lock().as_physical_address() } + // TODO ??? + res |= MapAttributes::NON_GLOBAL; + res } } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 521b338c..3a174a56 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -10,7 +10,6 @@ use device_api::{ }; use git_version::git_version; use kernel_util::util::OneTimeInit; -use memtables::FixedTables; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; mod acpi; @@ -64,7 +63,7 @@ use self::{ mem::{ init_fixed_tables, table::{PageAttributes, PageEntry, L1, L3}, - EarlyMapping, KERNEL_TABLES, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, + EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, }, peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort}, smp::CPU_COUNT, @@ -166,7 +165,7 @@ impl Architecture for X86_64 { fn map_physical_memory + Clone>( &self, - it: I, + _it: I, _memory_start: PhysicalAddress, memory_end: PhysicalAddress, ) -> Result<(), Error> { @@ -237,8 +236,9 @@ impl Architecture for X86_64 { } impl X86_64 { - unsafe fn handle_ipi(&self, msg: CpuMessage) { - todo!() + unsafe fn handle_ipi(&self, _msg: CpuMessage) { + warnln!("Received an IPI"); + loop {} } fn set_boot_data(&self, data: BootData) { @@ -404,7 +404,7 @@ impl X86_64 { self.ioapic.init(IoApic::from_acpi(&apic_info)?); - // acpi::init_acpi(acpi).unwrap(); + acpi::init_acpi(acpi).unwrap(); if let Ok(mcfg) = acpi.find_table::() { for entry in mcfg.entries() { diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index e2687c38..970d776f 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -9,7 +9,7 @@ use crate::{ boot::__x86_64_ap_entry, intrinsics::flush_tlb_entry, mem::{ - table::{PageAttributes, L1, L2}, + table::{PageAttributes, PageEntry, PageTable, L1, L2}, KERNEL_TABLES, }, }, @@ -19,7 +19,6 @@ use crate::{ address::{AsPhysicalAddress, FromRaw, IntoRaw}, phys, pointer::PhysicalRefMut, - table::{PageEntry, PageTable}, PhysicalAddress, }, task::Cpu, @@ -35,7 +34,6 @@ static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86 const AP_STACK_PAGES: usize = 8; const AP_BOOTSTRAP_DATA: PhysicalAddress = PhysicalAddress::from_raw(0x6000usize); const AP_BOOTSTRAP_CODE: PhysicalAddress = PhysicalAddress::from_raw(0x7000usize); -const AP_ADDRESS_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(0x100000usize); #[repr(C)] #[allow(dead_code)] diff --git a/src/device/display/console.rs b/src/device/display/console.rs index ef5e57a2..9c8c3e00 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -3,7 +3,7 @@ use core::time::Duration; use abi::{error::Error, primitive_enum}; -use alloc::{boxed::Box, vec, vec::Vec}; +use alloc::{vec, vec::Vec}; use bitflags::bitflags; use kernel_util::util::StaticVector; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 563c0510..ac6107cc 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -7,7 +7,7 @@ use memfs::block::{self, BlockAllocator}; use vfs::VnodeRef; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::mem::{self, phys, PhysicalAddress}; +use crate::mem::{phys, PhysicalAddress}; pub mod devfs; diff --git a/src/mem/address.rs b/src/mem/address.rs index 626ea3e6..8bf10baa 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -1,14 +1,11 @@ use core::{ fmt, iter::Step, - marker::PhantomData, mem::align_of, ops::{Add, Deref, DerefMut, Sub}, }; -use bytemuck::{Pod, Zeroable}; - -use crate::arch::{Architecture, ArchitectureImpl, ARCHITECTURE}; +use crate::arch::{Architecture, ArchitectureImpl}; use super::{pointer::PhysicalPointer, table::EntryLevel, KERNEL_VIRT_OFFSET}; @@ -185,16 +182,16 @@ impl From for usize { // Ranges impl Step for PhysicalAddress { - fn steps_between(start: &Self, end: &Self) -> Option { - loop {} + fn steps_between(_start: &Self, _end: &Self) -> Option { + todo!() } fn forward_checked(start: Self, count: usize) -> Option { start.0.checked_add(count as u64).map(Self) } - fn backward_checked(start: Self, count: usize) -> Option { - loop {} + fn backward_checked(_start: Self, _count: usize) -> Option { + todo!() } } diff --git a/src/mem/heap.rs b/src/mem/heap.rs index cb3bd678..fca4f7ce 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -35,7 +35,7 @@ unsafe impl GlobalAlloc for KernelAllocator { match self.inner.lock().allocate_first_fit(layout) { Ok(v) => v.as_ptr(), Err(e) => { - // errorln!("Failed to allocate {:?}: {:?}", layout, e); + errorln!("Failed to allocate {:?}: {:?}", layout, e); null_mut() } } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 94fcc0b3..fbceef93 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,25 +1,9 @@ -// //! Memory management utilities and types -// // use core::{alloc::Layout, mem::size_of}; -// -// use core::{alloc::Layout, ffi::c_void, mem::size_of}; -// -// use abi::error::Error; -// -// // use abi::error::Error; -// // -// use crate::arch::{Architecture, ArchitectureImpl /*, PlatformImpl*/}; -// -// use self::table::AddressSpace; -// // -// // use self::table::AddressSpace; -// -// pub mod device; +//! Memory management utilities and types use core::{ alloc::Layout, ffi::c_void, mem::{align_of, size_of}, - ops::Add, }; use abi::error::Error; @@ -31,11 +15,12 @@ pub mod device; pub mod heap; pub mod phys; pub mod pointer; +pub mod process; pub mod table; pub use address::PhysicalAddress; -use self::{device::DeviceMemoryMapping, table::AddressSpace}; +use self::{device::DeviceMemoryMapping, process::ProcessAddressSpace}; pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; @@ -60,42 +45,7 @@ pub unsafe fn write_memory(address: PhysicalAddress, value: T) { (address as *mut T).write_unaligned(value) } } -// pub mod phys; -// -// /// Kernel's physical load address -// // pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE; -// /// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] + -// /// [KERNEL_VIRT_OFFSET]) -// pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; -// -// /// Interface for converting between address spaces. -// /// -// /// # Safety -// /// -// /// An incorrect implementation can produce invalid address. -// pub unsafe trait ConvertAddress { -// /// Convert the address into a virtual one -// /// -// /// # Panics -// /// -// /// Panics if the address is already a virtual one -// /// -// /// # Safety -// /// -// /// An incorrect implementation can produce invalid address. -// unsafe fn virtualize(self) -> Self; -// /// Convert the address into a physical one -// /// -// /// # Panics -// /// -// /// Panics if the address is already a physical one -// /// -// /// # Safety -// /// -// /// An incorrect implementation can produce invalid address. -// unsafe fn physicalize(self) -> Self; -// } -// + /// Helper trait to allow cross-address space access to pointers pub trait ForeignPointer: Sized { /// Perform a volatile pointer write without dropping the old value. @@ -111,7 +61,7 @@ pub trait ForeignPointer: Sized { /// # Safety /// /// As this function allows direct memory writes, it is inherently unsafe. - unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: Self); + unsafe fn write_foreign_volatile(self: *mut Self, space: &ProcessAddressSpace, value: Self); /// Performs pointer validation for given address space: /// @@ -125,7 +75,7 @@ pub trait ForeignPointer: Sized { /// conversion, and thus is unsafe. unsafe fn validate_user_ptr<'a>( self: *const Self, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a Self, Error>; /// [ForeignPointer::validate_user_ptr], with extra "writability" check. @@ -136,7 +86,7 @@ pub trait ForeignPointer: Sized { /// conversion, and thus is unsafe. unsafe fn validate_user_mut<'a>( self: *mut Self, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a mut Self, Error>; /// [ForeignPointer::validate_user_ptr], but for slices @@ -148,7 +98,7 @@ pub trait ForeignPointer: Sized { unsafe fn validate_user_slice<'a>( self: *const Self, len: usize, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a [Self], Error>; /// [ForeignPointer::validate_user_slice], but for mutable slices @@ -160,12 +110,12 @@ pub trait ForeignPointer: Sized { unsafe fn validate_user_slice_mut<'a>( self: *mut Self, len: usize, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a mut [Self], Error>; } impl ForeignPointer for T { - unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) { + unsafe fn write_foreign_volatile(self: *mut Self, space: &ProcessAddressSpace, value: T) { // TODO check align let addr = self as usize; let start_page = addr & !0xFFF; @@ -187,7 +137,7 @@ impl ForeignPointer for T { unsafe fn validate_user_slice_mut<'a>( self: *mut Self, len: usize, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a mut [Self], Error> { let base = self as usize; let layout = Layout::array::(len).unwrap(); @@ -201,7 +151,7 @@ impl ForeignPointer for T { unsafe fn validate_user_slice<'a>( self: *const Self, len: usize, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a [Self], Error> { let base = self as usize; let layout = Layout::array::(len).unwrap(); @@ -214,7 +164,7 @@ impl ForeignPointer for T { unsafe fn validate_user_mut<'a>( self: *mut Self, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a mut Self, Error> { let addr = self as usize; let layout = Layout::new::(); @@ -231,7 +181,7 @@ impl ForeignPointer for T { unsafe fn validate_user_ptr<'a>( self: *const Self, - space: &AddressSpace, + space: &ProcessAddressSpace, ) -> Result<&'a Self, Error> { let addr = self as usize; let layout = Layout::new::(); @@ -262,7 +212,7 @@ fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { /// Validates access to given userspace memory region with given constraints pub fn validate_user_region( - space: &AddressSpace, + space: &ProcessAddressSpace, base: usize, len: usize, _need_write: bool, @@ -276,7 +226,7 @@ pub fn validate_user_region( for page in (aligned_start..aligned_end).step_by(0x1000) { // TODO check writability - space.translate(page).ok_or(Error::InvalidArgument)?; + space.translate(page)?; } Ok(()) diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 5af2b58c..1f22832f 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -30,10 +30,8 @@ impl PhysicalMemoryManager { bitmap_phys_base: PhysicalAddress, page_count: usize, ) -> PhysicalMemoryManager { - // let bitmap_addr = bitmap_phys_base.virtualize(); let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE; let mut bitmap = PhysicalRefMut::<'static, u64>::map_slice(bitmap_phys_base, bitmap_len); - // let bitmap = core::slice::from_raw_parts_mut(bitmap_addr as *mut BitmapWord, bitmap_len); bitmap.fill(BitmapWord::MAX); @@ -85,7 +83,7 @@ impl PhysicalMemoryManager { 'l0: for i in (aligned_bit..self.page_count).step_by(512) { for j in 0..HUGE_PAGE_WORD_COUNT { - if self.bitmap[i / BITMAP_WORD_SIZE] != 0 { + if self.bitmap[i / BITMAP_WORD_SIZE + j] != 0 { continue 'l0; } } diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 70c1da90..31aa664f 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,4 +1,4 @@ -use core::{iter::StepBy, ops::Range}; +use core::ops::Range; use abi::error::Error; use kernel_util::util::OneTimeInit; @@ -10,68 +10,18 @@ use crate::{ }; use self::{ - manager::{PhysicalMemoryManager, BITMAP_PAGE_COUNT, BITMAP_WORD_SIZE, TRACKED_PAGE_LIMIT}, + manager::{PhysicalMemoryManager, BITMAP_WORD_SIZE, TRACKED_PAGE_LIMIT}, reserved::reserve_region, }; use super::{address::FromRaw, PhysicalAddress}; -// //! Physical memory management facilities -// use core::{iter::StepBy, mem::size_of, ops::Range}; -// -// use abi::error::Error; -// use kernel_util::util::OneTimeInit; -// -// use crate::{ -// debug::LogLevel, -// mem::{ -// phys::reserved::{is_reserved, reserve_region}, -// ConvertAddress, /*, KERNEL_PHYS_BASE */ -// }, -// sync::IrqSafeSpinlock, -// }; -// -// use self::manager::PhysicalMemoryManager; -// -// // Enumerating lots of pages is slow and I'm too lazy right now to write a better algorithm, so -// // capping the page count helps -// const PHYS_MEMORY_PAGE_CAP: usize = 65536; -// - // 8 * 4096 bits per page, 1 page per bit const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PAGE_LIMIT * 4096); mod manager; pub mod reserved; -// -// /// Contains information about the physical memory usage -// #[derive(Clone, Copy, Debug)] -// pub struct PhysicalMemoryStats { -// /// Number of pages available for allocation -// pub available_pages: usize, -// /// Number of pages being used -// pub used_pages: usize, -// } -// -// /// Represents the way in which the page is used (or not) -// #[derive(PartialEq, Clone, Copy, Debug)] -// #[repr(u32)] -// pub enum PageUsage { -// /// Page is not available for allocation or use -// Reserved = 0, -// /// Regular page available for allocation -// Available, -// /// Page is used by some kernel facility -// Used, -// } -// -// /// Page descriptor structure for the page management array -// #[repr(C)] -// pub struct Page { -// usage: PageUsage, -// refcount: u32, -// } -// + /// Defines an usable memory region #[derive(Clone, Copy, Debug)] pub struct PhysicalMemoryRegion { @@ -103,46 +53,7 @@ impl PhysicalMemoryRegion { } } } -// -// impl PhysicalMemoryStats { -// /// Handles "alloc" cases of the memory manager -// pub fn add_allocated_pages(&mut self, count: usize, _usage: PageUsage) { -// assert!(self.available_pages >= count); -// self.available_pages -= count; -// self.used_pages += count; -// } -// -// /// Handles "free" cases of the memory manager -// pub fn add_freed_pages(&mut self, count: usize, _usage: PageUsage) { -// assert!(self.used_pages >= count); -// self.used_pages -= count; -// self.available_pages += count; -// } -// -// /// Increases the available pages counter -// pub fn add_available_pages(&mut self, count: usize) { -// self.available_pages += count; -// } -// -// /// Prints out the statistics into specified log level -// pub fn dump(&self, level: LogLevel) { -// log_print_raw!(level, "+++ Physical memory stats +++\n"); -// log_print_raw!( -// level, -// "Available: {}K ({} pages)\n", -// self.available_pages * 4, -// self.available_pages -// ); -// log_print_raw!( -// level, -// "Used: {}K ({} pages)\n", -// self.used_pages * 4, -// self.used_pages -// ); -// log_print_raw!(level, "-----------------------------\n"); -// } -// } -// + /// Global physical memory manager pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); @@ -267,58 +178,7 @@ pub unsafe fn init_from_iter + Clone>( Ok(()) } -// -// debugln!("Initializing physical memory manager"); -// debugln!("Total tracked pages: {}", total_count); -// -// // Reserve memory regions from which allocation is forbidden -// reserve_region("kernel", kernel_physical_memory_region()); -// -// let pages_array_base = find_contiguous_region(it.clone(), (pages_array_size + 0xFFF) / 0x1000) -// .ok_or(Error::OutOfMemory)?; -// -// debugln!( -// "Placing page tracking at {:#x}", -// pages_array_base.virtualize() -// ); -// -// reserve_region( -// "pages", -// PhysicalMemoryRegion { -// base: pages_array_base, -// size: (pages_array_size + 0xFFF) & !0xFFF, -// }, -// ); -// -// let mut manager = -// PhysicalMemoryManager::new(phys_start, pages_array_base.virtualize(), pages_array_size); -// let mut page_count = 0; -// -// for region in it { -// if page_count >= PHYS_MEMORY_PAGE_CAP { -// break; -// } -// -// for page in region.pages() { -// if is_reserved(page) { -// continue; -// } -// -// manager.add_available_page(page); -// page_count += 1; -// -// if page_count >= PHYS_MEMORY_PAGE_CAP { -// break; -// } -// } -// } -// -// infoln!("{} available pages ({}KiB)", page_count, page_count * 4); -// -// PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager)); -// Ok(()) -// } -// + fn kernel_physical_memory_region() -> PhysicalMemoryRegion { extern "C" { static __kernel_phys_start: u8; diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 099a2309..58e58a29 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -13,14 +13,7 @@ static mut RESERVED_MEMORY: StaticVector = StaticVector /// # Safety /// /// Can only be called from initialization code **before** physical memory manager is initialized. -pub unsafe fn reserve_region(reason: &str, region: PhysicalMemoryRegion) { - // debugln!( - // "Reserve {:?} memory: {:#x}..{:#x}", - // reason, - // region.base, - // region.end() - // ); - +pub unsafe fn reserve_region(_reason: &str, region: PhysicalMemoryRegion) { RESERVED_MEMORY.push(region); } diff --git a/src/mem/pointer.rs b/src/mem/pointer.rs index d54b99a9..b79e302e 100644 --- a/src/mem/pointer.rs +++ b/src/mem/pointer.rs @@ -1,5 +1,4 @@ use core::{ - alloc::Layout, fmt, mem::align_of, ops::{Deref, DerefMut}, diff --git a/src/mem/process.rs b/src/mem/process.rs new file mode 100644 index 00000000..a6727de6 --- /dev/null +++ b/src/mem/process.rs @@ -0,0 +1,232 @@ +use abi::error::Error; +use cfg_if::cfg_if; +use vmalloc::VirtualMemoryAllocator; + +use crate::{arch::x86_64::mem::table::L3, mem::phys, sync::IrqSafeSpinlock}; + +use super::{address::AsPhysicalAddress, table::MapAttributes, PhysicalAddress}; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + use crate::arch::aarch64::table::AddressSpace; + } else if #[cfg(target_arch = "x86_64")] { + use crate::arch::x86_64::mem::process::ProcessAddressSpaceImpl; + } +} + +/// Interface for virtual memory address space management +pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress { + const PAGE_SIZE: usize; + const LOWER_LIMIT_PFN: usize; + const UPPER_LIMIT_PFN: usize; + + fn new() -> Result; + + unsafe fn map_page( + &mut self, + address: usize, + physical: PhysicalAddress, + flags: MapAttributes, + ) -> Result<(), Error>; + + unsafe fn unmap_page(&mut self, address: usize) -> Result; + + fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; +} + +struct Inner { + allocator: VirtualMemoryAllocator, + table: ProcessAddressSpaceImpl, +} + +pub struct ProcessAddressSpace { + inner: IrqSafeSpinlock, +} + +impl Inner { + fn try_map_pages Result>( + &mut self, + address: usize, + page_count: usize, + get_page: F, + attributes: MapAttributes, + ) -> Result<(), (usize, Error)> { + for i in 0..page_count { + let virt = address + i * ProcessAddressSpaceImpl::PAGE_SIZE; + let phys = match get_page(virt) { + Ok(page) => page, + Err(err) => { + return Err((i, err)); + } + }; + + if let Err(err) = unsafe { self.table.map_page(virt, phys, attributes) } { + return Err((i, err)); + } + } + + Ok(()) + } + + fn map_range Result>( + &mut self, + address: usize, + page_count: usize, + get_page: F, + attributes: MapAttributes, + ) -> Result<(), Error> { + // If inserting fails, the range cannot be mapped + let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; + self.allocator.insert(start_pfn, page_count)?; + + if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) { + // TODO rollback & remove the range + todo!(); + }; + + Ok(()) + } + + fn alloc_range Result>( + &mut self, + page_count: usize, + get_page: F, + attributes: MapAttributes, + ) -> Result { + let start_pfn = self.allocator.allocate(page_count)?; + let address = start_pfn * ProcessAddressSpaceImpl::PAGE_SIZE; + + if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) { + // TODO rollback + todo!(); + }; + + Ok(address) + } + + unsafe fn unmap_range( + &mut self, + start_address: usize, + page_count: usize, + free_page: F, + ) -> Result<(), Error> { + let start_pfn = start_address / ProcessAddressSpaceImpl::PAGE_SIZE; + + // Deallocate the range first + self.allocator.free(start_pfn, page_count)?; + + // Then unmap it from the table + for i in 0..page_count { + let virt = start_address + i * ProcessAddressSpaceImpl::PAGE_SIZE; + // This should not fail under normal circumstances + // TODO handle failures here? + let phys = self.table.unmap_page(virt).unwrap(); + + free_page(virt, phys); + } + + Ok(()) + } +} + +impl ProcessAddressSpace { + pub fn new() -> Result { + let table = ProcessAddressSpaceImpl::new()?; + let allocator = VirtualMemoryAllocator::new( + ProcessAddressSpaceImpl::LOWER_LIMIT_PFN, + ProcessAddressSpaceImpl::UPPER_LIMIT_PFN, + ); + Ok(Self { + inner: IrqSafeSpinlock::new(Inner { table, allocator }), + }) + } + + pub fn allocate Result>( + &self, + _hint: Option, + size: usize, + get_page: F, + attributes: MapAttributes, + ) -> Result { + assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); + + let mut lock = self.inner.lock(); + + lock.alloc_range( + size / ProcessAddressSpaceImpl::PAGE_SIZE, + get_page, + attributes, + ) + } + + pub fn map Result>( + &self, + address: usize, + size: usize, + get_page: F, + attributes: MapAttributes, + ) -> Result<(), Error> { + assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); + assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); + + let mut lock = self.inner.lock(); + + lock.map_range( + address, + size / ProcessAddressSpaceImpl::PAGE_SIZE, + get_page, + attributes, + ) + } + + pub fn map_single( + &self, + address: usize, + physical: PhysicalAddress, + attributes: MapAttributes, + ) -> Result<(), Error> { + assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); + assert!(physical.is_aligned_for::()); + + self.inner + .lock() + .map_range(address, 1, |_| Ok(physical), attributes) + } + + pub fn translate(&self, address: usize) -> Result { + // Offset is handled at impl level + self.inner.lock().table.translate(address).map(|e| e.0) + } + + pub unsafe fn unmap(&self, address: usize, size: usize) -> Result<(), Error> { + assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); + assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); + + let mut lock = self.inner.lock(); + + lock.unmap_range( + address, + size / ProcessAddressSpaceImpl::PAGE_SIZE, + |_, paddr| phys::free_page(paddr), + ) + } + + pub fn debug_dump(&self) { + let lock = self.inner.lock(); + + debugln!("Address space @ {:#x}", unsafe { + lock.table.as_physical_address() + }); + for (used, range) in lock.allocator.ranges() { + let start = range.start_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE; + let end = range.end_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE; + debugln!("{:#x?}: {}", start..end, used); + } + } +} + +impl AsPhysicalAddress for ProcessAddressSpace { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + self.inner.lock().table.as_physical_address() + } +} diff --git a/src/mem/table.rs b/src/mem/table.rs index d15bc766..f7dc9493 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -3,17 +3,6 @@ use core::ops::{Deref, DerefMut}; use abi::error::Error; use bitflags::bitflags; -use cfg_if::cfg_if; - -use super::PhysicalAddress; - -cfg_if! { - if #[cfg(target_arch = "aarch64")] { - pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable}; - } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::mem::table::{AddressSpace, PageEntry, PageTable}; - } -} bitflags! { /// Describes how a page translation mapping should behave @@ -28,40 +17,6 @@ bitflags! { } } -/// Interface for virtual memory address space management -pub trait VirtualMemoryManager { - /// Allocates a region of virtual memory inside the address space and maps it to physical - /// memory pages with given attributes - fn allocate( - &self, - hint: Option, - len: usize, - attrs: MapAttributes, - ) -> Result; - - /// Insert a single 4KiB-page translation mapping into the table - fn map_page( - &self, - virt: usize, - phys: PhysicalAddress, - attrs: MapAttributes, - ) -> Result<(), Error>; - - /// Releases the virtual memory region from the address space and the pages it refers to - fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error>; -} - -pub trait KernelAddressSpace { - type Mapping; - - fn map_page( - &self, - virt: usize, - physical: PhysicalAddress, - attrs: MapAttributes, - ) -> Result; -} - /// Interface for non-terminal tables to retrieve the next level of address translation tables pub trait NextPageTable { /// Type for the next-level page table diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 4948923a..63f14c97 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -10,9 +10,7 @@ use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::mem::{ - phys, - pointer::PhysicalRefMut, - table::{AddressSpace, MapAttributes, VirtualMemoryManager}, + phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, table::MapAttributes, }; #[derive(Clone, Copy)] @@ -59,24 +57,16 @@ fn from_parse_error(v: ParseError) -> Error { } fn load_bytes( - space: &AddressSpace, + space: &ProcessAddressSpace, addr: usize, mut src: F, len: usize, - elf_attrs: u32, ) -> Result<(), Error> where F: FnMut(usize, PhysicalRefMut<'_, [u8]>) -> Result<(), Error>, { // TODO check for crazy addresses here - let attrs = match (elf_attrs & PF_W, elf_attrs & PF_X) { - (0, 0) => MapAttributes::USER_READ, - (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, - (0, _) => MapAttributes::USER_READ, - (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, - } | MapAttributes::NON_GLOBAL; - let dst_page_off = addr & 0xFFF; let dst_page_aligned = addr & !0xFFF; let mut off = 0usize; @@ -89,22 +79,9 @@ where let virt_page = dst_page_aligned + page_idx * 0x1000; assert_eq!(virt_page & 0xFFF, 0); - if let Some(page) = space.translate(virt_page) { - // TODO Handle these cases - warnln!("Page {:#x} is already mapped to {:#x}", virt_page, page); - todo!(); - } - - let phys_page = phys::alloc_page()?; - space.map_page(virt_page, phys_page, attrs)?; - // debugln!("Map {:#x} -> {:#x}", virt_page, phys_page); + let phys_page = space.translate(virt_page)?; let dst_slice = unsafe { PhysicalRefMut::map_slice(phys_page.add(page_off), count) }; - // let dst_slice = unsafe { - // let addr = (phys_page + page_off).virtualize(); - - // core::slice::from_raw_parts_mut(addr as *mut u8, count) - // }; src(off, dst_slice)?; @@ -116,17 +93,41 @@ where } /// Loads an ELF binary from `file` into the target address space -pub fn load_elf_from_file(space: &AddressSpace, file: FileRef) -> Result { +pub fn load_elf_from_file(space: &ProcessAddressSpace, file: FileRef) -> Result { let file = FileReader { file: &file }; let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; + space.debug_dump(); + for phdr in elf.segments() { if phdr.p_type != PT_LOAD { continue; } - debugln!("LOAD {:#x}", phdr.p_vaddr); + debugln!("LOAD {:#x?}", phdr.p_vaddr..phdr.p_vaddr + phdr.p_memsz); + + let attrs = match (phdr.p_flags & PF_W, phdr.p_flags & PF_X) { + (0, 0) => MapAttributes::USER_READ, + (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + (0, _) => MapAttributes::USER_READ, + (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + } | MapAttributes::NON_GLOBAL; + + if phdr.p_memsz > 0 { + // Map the range + let aligned_start = (phdr.p_vaddr as usize) & !0xFFF; + let aligned_end = ((phdr.p_vaddr + phdr.p_memsz) as usize + 0xFFF) & !0xFFF; + + space.map( + aligned_start, + aligned_end - aligned_start, + |_| phys::alloc_page(), + attrs, + )?; + } else { + continue; + } if phdr.p_filesz > 0 { load_bytes( @@ -138,7 +139,6 @@ pub fn load_elf_from_file(space: &AddressSpace, file: FileRef) -> Result Result Result<(), Error> { +fn setup_args(space: &ProcessAddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { // arg data len let args_size: usize = args.iter().map(|x| x.len()).sum(); // 1 + arg ptr:len count @@ -30,13 +29,11 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er debugln!("arg data size = {}", args_size); let phys_page = phys::alloc_page()?; - // TODO check if this doesn't overwrite anything - space.map_page( + space.map_single( virt, phys_page, MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; - let write = phys_page.virtualize_raw(); let mut offset = args_ptr_size; @@ -57,6 +54,7 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er } // Place the argument data + // TODO rewrite using write_foreign unsafe { let arg_data_slice = core::slice::from_raw_parts_mut((write + args_ptr_size) as *mut u8, args_size); @@ -72,7 +70,7 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er fn setup_binary>( name: S, - space: AddressSpace, + space: ProcessAddressSpace, entry: usize, args: &[&str], ) -> Result, Error> { @@ -82,14 +80,12 @@ fn setup_binary>( // 0x1000 of guard page let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; - for i in 0..USER_STACK_PAGES { - let phys = phys::alloc_page()?; - space.map_page( - virt_stack_base + i * 0x1000, - phys, - MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - )?; - } + space.map( + virt_stack_base, + USER_STACK_PAGES * 0x1000, + |_| phys::alloc_page(), + MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, + )?; setup_args(&space, virt_args_base, args)?; @@ -113,7 +109,12 @@ fn setup_binary>( } } - let context = TaskContext::user(entry, virt_args_base, space.physical_address(), user_sp)?; + let context = TaskContext::user( + entry, + virt_args_base, + unsafe { space.as_physical_address() }, + user_sp, + )?; Ok(Process::new_with_context(name, Some(space), context)) } @@ -124,7 +125,7 @@ pub fn load_elf>( file: FileRef, args: &[&str], ) -> Result, Error> { - let space = AddressSpace::new_empty()?; + let space = ProcessAddressSpace::new()?; let elf_entry = proc::elf::load_elf_from_file(&space, file)?; setup_binary(name, space, elf_entry, args) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 075f08dc..40860ad6 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -16,7 +16,7 @@ use yggdrasil_abi::{ use crate::{ block, fs, - mem::table::{MapAttributes, VirtualMemoryManager}, + mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo}, sync::IrqSafeSpinlockGuard, task::{process::Process, runtime, ProcessId}, @@ -82,11 +82,14 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - space.allocate( + let res = space.allocate( None, - len / 0x1000, - MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, - ) + len, + |_| phys::alloc_page(), + MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, + ); + + res } SyscallFunction::UnmapMemory => { let addr = args[0] as usize; @@ -99,7 +102,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - space.deallocate(addr, len)?; + unsafe { + space.unmap(addr, len)?; + } Ok(0) } diff --git a/src/task/process.rs b/src/task/process.rs index e79f4cd8..a13beecd 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -18,8 +18,7 @@ use kernel_util::util::OneTimeInit; use vfs::VnodeRef; use crate::{ - arch::{Architecture, ArchitectureImpl}, - mem::{table::AddressSpace, ForeignPointer}, + mem::{process::ProcessAddressSpace, ForeignPointer}, proc::io::ProcessIo, sync::{IrqGuard, IrqSafeSpinlock}, task::context::TaskContextImpl, @@ -72,7 +71,7 @@ pub struct Process { id: OneTimeInit, state: AtomicProcessState, cpu_id: AtomicU32, - space: Option, + space: Option, inner: IrqSafeSpinlock, exit_waker: QueueWaker, @@ -92,7 +91,7 @@ impl Process { /// Has side-effect of allocating a new PID for itself. pub fn new_with_context>( name: S, - space: Option, + space: Option, normal_context: TaskContext, ) -> Arc { let this = Arc::new(Self { @@ -172,12 +171,12 @@ impl Process { } /// Returns the address space of the task - pub fn address_space(&self) -> &AddressSpace { + pub fn address_space(&self) -> &ProcessAddressSpace { self.space.as_ref().unwrap() } /// Returns the address space of the task, if one is set - pub fn get_address_space(&self) -> Option<&AddressSpace> { + pub fn get_address_space(&self) -> Option<&ProcessAddressSpace> { self.space.as_ref() } From d5859d93a9e4c4cc4c4ebcbd68e422603ab133b4 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 3 Oct 2023 10:17:13 +0300 Subject: [PATCH 077/211] x86-64/mm: implement 2MiB RAM mapping --- src/arch/x86_64/intrinsics.rs | 7 ++++ src/arch/x86_64/mem/mod.rs | 12 +++++++ src/arch/x86_64/mem/table.rs | 4 +++ src/arch/x86_64/mod.rs | 61 +++++++++++++++++++++++++++++++---- src/mem/phys/mod.rs | 6 ++-- src/util/mod.rs | 9 ++---- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index 6c11490f..8fe7991b 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -134,3 +134,10 @@ pub unsafe fn outl(port: u16, value: u32) { pub unsafe fn flush_tlb_entry(address: usize) { core::arch::asm!("invlpg ({0})", in(reg) address, options(att_syntax)); } + +#[inline] +pub fn flush_cpu_cache() { + unsafe { + core::arch::asm!("wbinvd"); + } +} diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index b52bf5ff..77ee41d2 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -242,6 +242,18 @@ pub struct EarlyMapping<'a, T: ?Sized> { } impl<'a, T: Sized> EarlyMapping<'a, T> { + pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { + let layout = Layout::new::(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(physical, page_count)?; + let value = &mut *((virt + offset) as *mut T); + + Ok(EarlyMapping { value, page_count }) + } + pub unsafe fn map_slice( physical: PhysicalAddress, len: usize, diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index a70084f7..c8e1234d 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -210,6 +210,10 @@ impl PageTable { } } + pub unsafe fn from_raw_slice_mut(data: &mut [PageEntry; 512]) -> &mut Self { + core::mem::transmute(data) + } + /// Allocates a new page table, filling it with non-preset entries pub fn new_zeroed<'a>() -> Result, Error> { let physical = phys::alloc_page()?; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 3a174a56..df67b8eb 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,5 +1,5 @@ // TODO fix all TODOs -use core::{mem::size_of, sync::atomic::Ordering}; +use core::{mem::size_of, ops::DerefMut, sync::atomic::Ordering}; use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; @@ -30,7 +30,11 @@ mod syscall; use crate::{ arch::x86_64::{ intrinsics::{IoPort, IoPortAccess}, - mem::{map_heap_block, table::L2, HEAP_MAPPING_OFFSET}, + mem::{ + map_heap_block, + table::{PageTable, L2}, + HEAP_MAPPING_OFFSET, + }, }, debug::{self, LogLevel}, device::{ @@ -165,7 +169,7 @@ impl Architecture for X86_64 { fn map_physical_memory + Clone>( &self, - _it: I, + it: I, _memory_start: PhysicalAddress, memory_end: PhysicalAddress, ) -> Result<(), Error> { @@ -195,11 +199,54 @@ impl Architecture for X86_64 { ); } } - - Ok(()) } else { - todo!(); + // Allocate the intermediate tables first + let l2_tables_start = phys::find_contiguous_region(it, end_l1i) + .expect("Could not allocate the memory for RAM mapping L2 tables"); + + unsafe { + reserve_region( + "ram-l2-tables", + PhysicalMemoryRegion { + base: l2_tables_start, + size: end_l1i * 0x1000, + }, + ); + } + + // Fill in the tables + for l1i in 0..end_l1i { + let l2_phys_addr = l2_tables_start.add(l1i * 0x1000); + + // TODO (minor) the slice is uninitialized, maybe find some way to deal with that + // case nicely + // Safety: ok, the mapping is done to the memory obtained from + // find_contiguous_region() + let mut l2_data = + unsafe { EarlyMapping::<[PageEntry; 512]>::map(l2_phys_addr)? }; + // Safety: ok, the slice comes from EarlyMapping of a page-aligned region + let l2 = unsafe { PageTable::from_raw_slice_mut(l2_data.deref_mut()) }; + + for l2i in 0..512 { + // TODO NX + l2[l2i] = PageEntry::::block( + PhysicalAddress::from_raw((l1i << 30) | (l2i << 21)), + PageAttributes::WRITABLE, + ); + } + + // Point the L1 entry to the L2 table + unsafe { + RAM_MAPPING_L1[l1i] = + PageEntry::::table(l2_phys_addr, PageAttributes::WRITABLE) + }; + + intrinsics::flush_cpu_cache(); + // The EarlyMapping is then dropped + } } + + Ok(()) } #[inline] @@ -404,7 +451,7 @@ impl X86_64 { self.ioapic.init(IoApic::from_acpi(&apic_info)?); - acpi::init_acpi(acpi).unwrap(); + // acpi::init_acpi(acpi).unwrap(); if let Ok(mcfg) = acpi.find_table::() { for entry in mcfg.entries() { diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 31aa664f..1b691cf1 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -103,7 +103,7 @@ fn physical_memory_range>( } } -fn find_contiguous_region>( +pub fn find_contiguous_region>( it: I, count: usize, ) -> Option { @@ -144,14 +144,14 @@ pub unsafe fn init_from_iter + Clone>( // Map the physical memory let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap(); + reserve_region("kernel", kernel_physical_memory_region()); + ARCHITECTURE.map_physical_memory(it.clone(), phys_start, phys_end)?; let total_count = (phys_end - phys_start) / 0x1000; let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / BITMAP_WORD_SIZE; let page_bitmap_page_count = (page_bitmap_size + 0xFFF) / 0x1000; - reserve_region("kernel", kernel_physical_memory_region()); - let page_bitmap_phys_base = find_contiguous_region(it.clone(), page_bitmap_page_count).unwrap(); reserve_region( diff --git a/src/util/mod.rs b/src/util/mod.rs index d69d94c1..20124369 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -3,7 +3,7 @@ use core::time::Duration; use yggdrasil_abi::error::Error; -use crate::arch::{Architecture, ARCHITECTURE}; +// use crate::arch::{Architecture, ARCHITECTURE}; pub mod queue; pub mod ring; @@ -26,12 +26,9 @@ impl>> ResultIterator for I { /// Performs a busy-loop sleep until the specified duration has passed pub fn polling_sleep(duration: Duration) -> Result<(), Error> { - let timer = ARCHITECTURE.monotonic_timer(); - let deadline = timer.monotonic_timestamp()? + duration; - - while timer.monotonic_timestamp()? < deadline { + // TODO no non-IRQ mode timestamp provider + for i in 0..1000000 { core::hint::spin_loop(); } - Ok(()) } From fbb804f14f00f1ebd77c2f424bea9c43f50c0e0a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 6 Nov 2023 10:29:53 +0200 Subject: [PATCH 078/211] x86-64/mm: limit physical memory, init is slow --- Cargo.toml | 6 +- src/arch/x86_64/boot/mod.rs | 9 +- src/arch/x86_64/mem/mod.rs | 13 +- src/arch/x86_64/mod.rs | 12 +- src/arch/x86_64/registers/mod.rs | 215 +++++++++++++++++++++++-------- src/main.rs | 3 +- src/mem/phys/mod.rs | 7 + tools/gentables/src/x86_64.rs | 31 +++-- 8 files changed, 208 insertions(+), 88 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4f800e69..960cd447 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,9 +43,9 @@ fdt-rs = { version = "0.4.3", default-features = false } [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } -aml = { git = "https://github.com/alnyan/acpi.git", version = "0.16.4" } -acpi_lib = { git = "https://github.com/alnyan/acpi.git", version = "4.1.1", package = "acpi" } -acpi-system = { git = "https://github.com/alnyan/acpi-system.git", version = "0.1.0" } +aml = { git = "https://github.com/alnyan/acpi.git", branch = "acpi-system" } +acpi_lib = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" } +acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } # TODO currently only supported here xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 679dafdc..7b23deea 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,21 +1,24 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; -use tock_registers::interfaces::Writeable; +use tock_registers::interfaces::{ReadWriteable, Writeable}; use yboot_proto::{ v1::{FramebufferOption, MemoryMap}, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, }; use crate::{ - arch::x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, + arch::x86_64::{ + registers::{CR0, MSR_IA32_KERNEL_GS_BASE}, + smp::CPU_COUNT, + }, fs::devfs, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, task::runtime, }; -use super::{cpuid::init_cpuid, exception, ARCHITECTURE}; +use super::{cpuid::init_cpuid, exception, registers::CR4, ARCHITECTURE}; pub enum BootData { YBoot(&'static LoadProtocolV1), diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 77ee41d2..a5168bfc 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -5,14 +5,15 @@ use core::{ use abi::error::Error; use kernel_util::util::OneTimeInit; -use memtables::FixedTables; +use memtables::{FixedTables, KERNEL_L3_COUNT}; use static_assertions::{const_assert_eq, const_assert_ne}; pub mod process; pub mod table; use crate::{ - arch::x86_64::{intrinsics, mem::table::PageAttributes}, + arch::x86_64::{intrinsics, mem::table::PageAttributes, registers::CR3}, + device::display::font::PcScreenFont, mem::{ address::{FromRaw, IntoRaw, KernelImageObject}, device::RawDeviceMemoryMapping, @@ -318,6 +319,7 @@ fn clone_kernel_tables(dst: &mut PageTable) { /// * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000 : DEVICE_MAPPING_L3S /// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ... pub unsafe fn init_fixed_tables() { + // TODO this could be built in compile-time too? let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; @@ -330,19 +332,22 @@ pub unsafe fn init_fixed_tables() { DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE); } + assert_eq!(KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I], 0); KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + assert_eq!(KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I], 0); KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I] = (heap_mapping_l2_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + assert_eq!(KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I], 0); KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + assert_eq!(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I], 0); KERNEL_TABLES.l0.data[RAM_MAPPING_L0I] = (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); // TODO ENABLE EFER.NXE - let cr3 = &KERNEL_TABLES.l0 as *const _ as usize - KERNEL_VIRT_OFFSET; - core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax)); + CR3.set_address(cr3); } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index df67b8eb..37272ea3 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -390,6 +390,12 @@ impl X86_64 { debug::init(); + infoln!( + "Yggdrasil v{} ({})", + env!("CARGO_PKG_VERSION"), + git_version!() + ); + let ps2 = Box::leak(Box::new(PS2Controller::new( IrqNumber::Isa(1), IrqNumber::Isa(12), @@ -398,12 +404,6 @@ impl X86_64 { ))); ps2.init()?; - infoln!( - "Yggdrasil v{} ({})", - env!("CARGO_PKG_VERSION"), - git_version!() - ); - if let Some(acpi) = self.acpi.try_get() { self.init_platform_from_acpi(acpi)?; } diff --git a/src/arch/x86_64/registers/mod.rs b/src/arch/x86_64/registers/mod.rs index b13cabe9..03e9e70e 100644 --- a/src/arch/x86_64/registers/mod.rs +++ b/src/arch/x86_64/registers/mod.rs @@ -1,57 +1,103 @@ //! Helper types for interfacing with x86-64 registers -macro_rules! msr_impl_read { - ($t:ident, $addr:expr, $register:ty) => { +macro_rules! impl_read { + ($t:ident, $register:ty, $body:expr) => { impl tock_registers::interfaces::Readable for $t { type T = u64; type R = $register; #[inline] fn get(&self) -> u64 { - let (high, low): (u32, u32); - unsafe { - core::arch::asm!( - "rdmsr", - in("ecx") $addr, - out("eax") low, - out("edx") high, - options(att_syntax) - ); - } - ((high as u64) << 32) | (low as u64) + $body } } }; +} + +macro_rules! impl_write { + ($t:ident, $register:ty, $value:ident, $body:expr) => { + impl tock_registers::interfaces::Writeable for $t { + type T = u64; + type R = $register; + + #[inline] + fn set(&self, $value: u64) { + $body + } + } + }; +} + +macro_rules! msr_impl_read { + ($t:ident, $addr:expr, $register:ty) => { + impl_read!($t, $register, { + let (high, low): (u32, u32); + unsafe { + core::arch::asm!( + "rdmsr", + in("ecx") $addr, + out("eax") low, + out("edx") high, + options(att_syntax) + ); + } + ((high as u64) << 32) | (low as u64) + }); + }; ($t:ident, $addr:expr) => { msr_impl_read!($t, $addr, ()); }; } macro_rules! msr_impl_write { ($t:ident, $addr:expr, $register:ty) => { - impl tock_registers::interfaces::Writeable for $t { - type T = u64; - type R = $register; - - #[inline] - fn set(&self, value: u64) { - let low = value as u32; - let high = (value >> 32) as u32; - unsafe { - core::arch::asm!( - "wrmsr", - in("ecx") $addr, - in("eax") low, - in("edx") high, - options(att_syntax) - ); - } + impl_write!($t, $register, value, { + let low = value as u32; + let high = (value >> 32) as u32; + unsafe { + core::arch::asm!( + "wrmsr", + in("ecx") $addr, + in("eax") low, + in("edx") high, + options(att_syntax) + ); } - } + }); }; ($t:ident, $addr:expr) => { msr_impl_write!($t, $addr, ()); }; } +macro_rules! cr_impl_read { + ($t:ident, $cr:ident, $register:ty) => { + impl_read!($t, $register, { + let value: u64; + unsafe { + core::arch::asm!( + concat!("mov %", stringify!($cr), ", {}"), + out(reg) value, + options(att_syntax) + ); + } + value + }); + }; +} + +macro_rules! cr_impl_write { + ($t:ident, $cr:ident, $register:ty) => { + impl_write!($t, $register, value, { + unsafe { + core::arch::asm!( + concat!("mov {}, %", stringify!($cr)), + in(reg) value, + options(att_syntax) + ); + } + }); + }; +} + mod msr_ia32_kernel_gs_base { const ADDR: u32 = 0xC0000102; pub struct Reg; @@ -182,12 +228,73 @@ mod msr_ia32_efer { pub const MSR_IA32_EFER: Reg = Reg; } -mod cr4 { +mod cr0 { + use tock_registers::register_bitfields; + + register_bitfields! { + u64, + #[allow(missing_docs)] + pub CR0 [ + PG OFFSET(31) NUMBITS(1) [], + CD OFFSET(30) NUMBITS(1) [], + NW OFFSET(29) NUMBITS(1) [], + AM OFFSET(18) NUMBITS(1) [], + WP OFFSET(16) NUMBITS(1) [], + NE OFFSET(5) NUMBITS(1) [], + ET OFFSET(4) NUMBITS(1) [], + TS OFFSET(3) NUMBITS(1) [], + EM OFFSET(2) NUMBITS(1) [], + MP OFFSET(1) NUMBITS(1) [], + PE OFFSET(0) NUMBITS(1) [], + ] + } + + pub struct Reg; + + cr_impl_read!(Reg, cr0, CR0::Register); + cr_impl_write!(Reg, cr0, CR0::Register); + + /// x86-64 control register 0 + pub const CR0: Reg = Reg; +} + +mod cr3 { use tock_registers::{ - interfaces::{Readable, Writeable}, + interfaces::{ReadWriteable, Readable}, register_bitfields, }; + register_bitfields! { + u64, + #[allow(missing_docs)] + pub CR3 [ + ADDR OFFSET(12) NUMBITS(40) [], + ] + } + + pub struct Reg; + + cr_impl_read!(Reg, cr3, CR3::Register); + cr_impl_write!(Reg, cr3, CR3::Register); + + impl Reg { + pub fn set_address(&self, address: usize) { + assert_eq!(address & 0xFFF, 0); + self.modify(CR3::ADDR.val((address as u64) >> 12)) + } + + pub fn address(&self) -> usize { + (self.read(CR3::ADDR) as usize) << 12 + } + } + + /// x86-64 control register 3 + pub const CR3: Reg = Reg; +} + +mod cr4 { + use tock_registers::register_bitfields; + register_bitfields! { u64, #[allow(missing_docs)] @@ -196,36 +303,28 @@ mod cr4 { OSXSAVE OFFSET(18) NUMBITS(1) [], /// Indicates OS support for FXSAVE and FXRSTOR instructions OSFXSR OFFSET(9) NUMBITS(1) [], + /// Performance-Monitoring Counter enable + PCE OFFSET(8) NUMBITS(1) [], /// If set, "page global" attribute is enabled - PGE OFFSET(8) NUMBITS(1) [], + PGE OFFSET(7) NUMBITS(1) [], + /// Machine Check enable + MCE OFFSET(6) NUMBITS(1) [], + /// Physical Address Extension (enabled if 64-bit mode) + PAE OFFSET(5) NUMBITS(1) [], + /// Page Size Extension (should be enabled by yboot) + PSE OFFSET(4) NUMBITS(1) [], + /// Debugging extensions + DE OFFSET(3) NUMBITS(1) [], + TSD OFFSET(2) NUMBITS(1) [], + PVI OFFSET(1) NUMBITS(1) [], + VME OFFSET(0) NUMBITS(1) [], ] } pub struct Reg; - impl Readable for Reg { - type T = u64; - type R = CR4::Register; - - fn get(&self) -> Self::T { - let value: u64; - unsafe { - core::arch::asm!("mov %cr4, {0}", out(reg) value, options(att_syntax)); - } - value - } - } - - impl Writeable for Reg { - type T = u64; - type R = CR4::Register; - - fn set(&self, value: Self::T) { - unsafe { - core::arch::asm!("mov {0}, %cr4", in(reg) value, options(att_syntax)); - } - } - } + cr_impl_read!(Reg, cr4, CR4::Register); + cr_impl_write!(Reg, cr4, CR4::Register); /// x86-64 control register 4 pub const CR4: Reg = Reg; @@ -295,6 +394,8 @@ mod xcr0 { pub const XCR0: Reg = Reg; } +pub use cr0::CR0; +pub use cr3::CR3; pub use cr4::CR4; pub use msr_ia32_apic_base::MSR_IA32_APIC_BASE; pub use msr_ia32_efer::MSR_IA32_EFER; diff --git a/src/main.rs b/src/main.rs index c7171e96..bdb659c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ asm_const, panic_info_message, optimize_attribute, + effects, const_trait_impl, maybe_uninit_slice, arbitrary_self_types, @@ -14,7 +15,7 @@ linked_list_cursors, rustc_private, allocator_api, - async_fn_in_trait, + trait_alias, strict_provenance )] #![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 1b691cf1..e1328d03 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -163,14 +163,21 @@ pub unsafe fn init_from_iter + Clone>( ); let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, total_count); + let mut collected = 0; + const MAX_MEMORY: usize = 16 * 1024; for (start, end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { for page in (start..end).step_by(0x1000) { + if collected >= MAX_MEMORY { + break; + } + if is_reserved(page) { continue; } manager.add_available_page(page); + collected += 1; } } diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs index 8ce01464..d41551a4 100644 --- a/tools/gentables/src/x86_64.rs +++ b/tools/gentables/src/x86_64.rs @@ -20,7 +20,7 @@ bitflags! { struct PageFlags: u64 { const PRESENT = 1 << 0; const WRITABLE = 1 << 1; - const NX = 1 << 63; + // const NX = 1 << 63; } } @@ -38,12 +38,12 @@ pub struct X8664Builder { impl PageFlags { fn from_elf(flags: u32) -> Self { let mut out = Self::empty(); - if flags & PF_W != 0 { - out |= Self::WRITABLE; - } - if flags & PF_X == 0 { - out |= Self::NX; - } + // if flags & PF_W != 0 { + out |= Self::WRITABLE; + // } + // if flags & PF_X == 0 { + // // out |= Self::NX; + // } out } } @@ -58,7 +58,7 @@ impl fmt::Display for PageFlags { } else { '-' }, - if self.contains(Self::NX) { '-' } else { 'x' } + 'x' // if self.contains(Self::NX) { '-' } else { 'x' } ) } } @@ -98,24 +98,27 @@ impl X8664Builder { } pub fn build(mut self) -> Result<(FixedTables, u64), GenError> { - // L0 -> L1 + assert_eq!(offset_of!(FixedTables, l0), 0); let l1_physical_address = self.data.table_physical_address + offset_of!(FixedTables, kernel_l1) as u64; + let l2_physical_address = + self.data.table_physical_address + offset_of!(FixedTables, kernel_l2) as u64; + + // L0 -> L1 self.tables.l0.data[self.l0i] = l1_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); // L1 -> L2 - let l2_physical_address = - self.data.table_physical_address + offset_of!(FixedTables, kernel_l2) as u64; self.tables.kernel_l1.data[self.l1i] = l2_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); // L2 -> L3s - for i in 0..KERNEL_L3_COUNT { + for l2i in self.start_l2i..self.end_l2i { + let l3_table_index = l2i - self.start_l2i; let l3_physical_address = self.data.table_physical_address - + (offset_of!(FixedTables, kernel_l3s) + 0x1000 * i) as u64; + + (offset_of!(FixedTables, kernel_l3s) + 0x1000 * l3_table_index) as u64; - self.tables.kernel_l2.data[i + self.start_l2i] = + self.tables.kernel_l2.data[l2i] = l3_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); } From a530a34a09c04afb4a70b253a120d2eedf98d3fb Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 6 Nov 2023 18:30:41 +0200 Subject: [PATCH 079/211] refactor: fix warnings/fix incorrect EarlyMapping::map --- src/arch/x86_64/acpi.rs | 14 +++++++++----- src/arch/x86_64/boot/mod.rs | 9 +++------ src/arch/x86_64/mem/mod.rs | 5 ++--- src/arch/x86_64/registers/mod.rs | 10 ++-------- src/main.rs | 6 +++++- src/util/mod.rs | 21 ++++++++------------- 6 files changed, 29 insertions(+), 36 deletions(-) diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 80ce12d6..9380f8f7 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -28,7 +28,6 @@ use crate::{ PhysicalAddress, }, sync::IrqSafeSpinlock, - util, }; use super::intrinsics; @@ -180,8 +179,10 @@ impl acpi_system::Handler for AcpiHandlerImpl { Ok(()) } - fn stall(duration: Duration) { - util::polling_sleep(duration).ok(); + fn stall(_duration: Duration) { + // TODO polling_sleep is not yet implemented properly + todo!() + // util::polling_sleep(duration).ok(); } } @@ -307,8 +308,9 @@ impl aml::Handler for AcpiHandlerImpl { fn write_ec_u8(&self, _address: u64, _value: u8) {} - fn sleep(&self, duration: Duration) { - util::polling_sleep(duration).unwrap(); + fn sleep(&self, _duration: Duration) { + todo!() + // util::polling_sleep(duration).unwrap(); } } @@ -335,7 +337,9 @@ impl AcpiHandler for AcpiHandlerImpl { } /// Initializes ACPI management +#[allow(unused)] pub fn init_acpi(tables: &'static AcpiTables) -> Result<(), Error> { + // TODO currently broken for real HW let mut system = AcpiSystem::new(tables, Box::new(AcpiHandlerImpl)).unwrap(); system.initialize(AcpiInterruptMethod::Apic).unwrap(); diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 7b23deea..679dafdc 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,24 +1,21 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; -use tock_registers::interfaces::{ReadWriteable, Writeable}; +use tock_registers::interfaces::Writeable; use yboot_proto::{ v1::{FramebufferOption, MemoryMap}, LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, }; use crate::{ - arch::x86_64::{ - registers::{CR0, MSR_IA32_KERNEL_GS_BASE}, - smp::CPU_COUNT, - }, + arch::x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, fs::devfs, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, task::runtime, }; -use super::{cpuid::init_cpuid, exception, registers::CR4, ARCHITECTURE}; +use super::{cpuid::init_cpuid, exception, ARCHITECTURE}; pub enum BootData { YBoot(&'static LoadProtocolV1), diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index a5168bfc..ebdc8f97 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -5,7 +5,7 @@ use core::{ use abi::error::Error; use kernel_util::util::OneTimeInit; -use memtables::{FixedTables, KERNEL_L3_COUNT}; +use memtables::FixedTables; use static_assertions::{const_assert_eq, const_assert_ne}; pub mod process; @@ -13,7 +13,6 @@ pub mod table; use crate::{ arch::x86_64::{intrinsics, mem::table::PageAttributes, registers::CR3}, - device::display::font::PcScreenFont, mem::{ address::{FromRaw, IntoRaw, KernelImageObject}, device::RawDeviceMemoryMapping, @@ -249,7 +248,7 @@ impl<'a, T: Sized> EarlyMapping<'a, T> { let offset = physical.page_offset::(); let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; - let virt = map_early_pages(physical, page_count)?; + let virt = map_early_pages(aligned, page_count)?; let value = &mut *((virt + offset) as *mut T); Ok(EarlyMapping { value, page_count }) diff --git a/src/arch/x86_64/registers/mod.rs b/src/arch/x86_64/registers/mod.rs index 03e9e70e..0d05e99a 100644 --- a/src/arch/x86_64/registers/mod.rs +++ b/src/arch/x86_64/registers/mod.rs @@ -1,4 +1,5 @@ //! Helper types for interfacing with x86-64 registers +#![allow(unused)] macro_rules! impl_read { ($t:ident, $register:ty, $body:expr) => { @@ -259,10 +260,7 @@ mod cr0 { } mod cr3 { - use tock_registers::{ - interfaces::{ReadWriteable, Readable}, - register_bitfields, - }; + use tock_registers::{interfaces::ReadWriteable, register_bitfields}; register_bitfields! { u64, @@ -282,10 +280,6 @@ mod cr3 { assert_eq!(address & 0xFFF, 0); self.modify(CR3::ADDR.val((address as u64) >> 12)) } - - pub fn address(&self) -> usize { - (self.read(CR3::ADDR) as usize) << 12 - } } /// x86-64 control register 3 diff --git a/src/main.rs b/src/main.rs index bdb659c3..f82b4acb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,11 @@ trait_alias, strict_provenance )] -#![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)] +#![allow( + clippy::new_without_default, + clippy::fn_to_numeric_cast, + async_fn_in_trait +)] // #![warn(missing_docs)] #![allow(missing_docs)] #![no_std] diff --git a/src/util/mod.rs b/src/util/mod.rs index 20124369..fa3e2598 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,9 +1,4 @@ //! Various kernel utility functions -use core::time::Duration; - -use yggdrasil_abi::error::Error; - -// use crate::arch::{Architecture, ARCHITECTURE}; pub mod queue; pub mod ring; @@ -24,11 +19,11 @@ impl>> ResultIterator for I { } } -/// Performs a busy-loop sleep until the specified duration has passed -pub fn polling_sleep(duration: Duration) -> Result<(), Error> { - // TODO no non-IRQ mode timestamp provider - for i in 0..1000000 { - core::hint::spin_loop(); - } - Ok(()) -} +// /// Performs a busy-loop sleep until the specified duration has passed +// pub fn polling_sleep(duration: Duration) -> Result<(), Error> { +// // TODO no non-IRQ mode timestamp provider +// for i in 0..1000000 { +// core::hint::spin_loop(); +// } +// Ok(()) +// } From 639f13eda4d55acd8447925e63e43d14858d1bb6 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 11 Nov 2023 10:22:30 +0200 Subject: [PATCH 080/211] dev: DeviceRequest::ConfigureTerminal for combined tty --- src/device/tty.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/device/tty.rs b/src/device/tty.rs index 158fc04d..bd892269 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -93,6 +93,7 @@ pub mod combined { self.set_signal_group(id as _); Ok(()) } + DeviceRequest::ConfigureTerminal(config) => self.context.set_config(config), _ => Err(Error::InvalidArgument), } } @@ -286,6 +287,11 @@ impl TtyContext { pub async fn getc(&self) -> u8 { self.ring.read().await } + + pub fn set_config(&self, config: &TerminalOptions) -> Result<(), Error> { + self.inner.lock().config = config.clone(); + Ok(()) + } } // impl CharRingInner { From b21dbca0f903590d4a9b99162ba3308877f4f267 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 14 Nov 2023 12:26:12 +0200 Subject: [PATCH 081/211] proc: pass env along with args to spawn() --- src/init.rs | 2 +- src/proc/exec.rs | 131 +++++++++++++++++++++++++++++++++------------ src/syscall/mod.rs | 7 ++- 3 files changed, 104 insertions(+), 36 deletions(-) diff --git a/src/init.rs b/src/init.rs index 5cb9e73b..2ebda395 100644 --- a/src/init.rs +++ b/src/init.rs @@ -60,7 +60,7 @@ pub fn kinit() { let stderr = stdout.clone(); { - let user_init = proc::exec::load_elf("init", file, &["/init", "xxx"]).unwrap(); + let user_init = proc::exec::load_elf("init", file, &["/init", "xxx"], &[]).unwrap(); let mut io = user_init.io.lock(); io.set_ioctx(ioctx); io.set_file(RawFd::STDIN, stdin).unwrap(); diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 735da9e4..653c930e 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -7,26 +7,78 @@ use vfs::FileRef; use crate::{ mem::{ - address::AsPhysicalAddress, phys, process::ProcessAddressSpace, table::MapAttributes, - ForeignPointer, + address::AsPhysicalAddress, phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, + table::MapAttributes, ForeignPointer, }, proc, task::{context::TaskContextImpl, process::Process, TaskContext}, }; -fn setup_args(space: &ProcessAddressSpace, virt: usize, args: &[&str]) -> Result<(), Error> { - // arg data len - let args_size: usize = args.iter().map(|x| x.len()).sum(); - // 1 + arg ptr:len count - let args_ptr_size = (1 + args.len() * 2) * size_of::(); +pub struct EnvWriter<'a> { + data: &'a mut [u8], + offset: usize, +} - let total_size = args_size + args_ptr_size; +impl<'a> EnvWriter<'a> { + pub fn new(data: &'a mut [u8]) -> Self { + Self { data, offset: 0 } + } + + pub fn write_usize(&mut self, value: usize) -> Result<(), Error> { + self.write_bytes(&value.to_ne_bytes()) + } + + pub fn write_bytes(&mut self, value: &[u8]) -> Result<(), Error> { + if value.len() + self.offset > self.data.len() { + return Err(Error::InvalidArgument); + } + debugln!("write_bytes {:#x} {:x?}", self.offset, value); + self.data[self.offset..self.offset + value.len()].copy_from_slice(value); + self.offset += value.len(); + Ok(()) + } +} + +fn write_2d_str_array( + space: &ProcessAddressSpace, + virt: usize, + array: &[&str], +) -> Result<(), Error> { + todo!() +} + +// TODO I hate this function, it's ugly +fn setup_program_env( + space: &ProcessAddressSpace, + virt: usize, + args: &[&str], + envs: &[&str], +) -> Result<(), Error> { + const fn str_array_size(n: usize) -> usize { + // length of the array itself (1 usize) + ptr:len pairs (2x usize * n) + (1 + n * 2) * size_of::() + } + + const HEADER_SIZE: usize = 2 * size_of::(); + + // arg data len + let args_data_size: usize = args.iter().map(|x| x.len()).sum(); + let envs_data_size: usize = envs.iter().map(|x| x.len()).sum(); + let ptrs_size = str_array_size(args.len()) + str_array_size(envs.len()); + let total_size = args_data_size + envs_data_size + ptrs_size + HEADER_SIZE; + // 1 + arg ptr:len count + // let args_ptr_size = (1 + args.len() * 2) * size_of::(); + // let envs_ptr_size = (1 + envs.len() * 2) * size_of::(); + + // Total size: offset arrays + data + args/env split position + // let total_size = args_size + args_ptr_size + envs_size + envs_ptr_size + size_of::(); if total_size > 0x1000 { todo!(); } - debugln!("arg data size = {}", args_size); + // debugln!("arg data size = {}", args_size); + // debugln!("env data size = {}", envs_size); let phys_page = phys::alloc_page()?; space.map_single( @@ -34,35 +86,44 @@ fn setup_args(space: &ProcessAddressSpace, virt: usize, args: &[&str]) -> Result phys_page, MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; - let write = phys_page.virtualize_raw(); + let mut slice = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; + let mut writer = EnvWriter::new(&mut *slice); - let mut offset = args_ptr_size; + let args_array_offset = HEADER_SIZE; + let envs_array_offset = args_array_offset + str_array_size(args.len()); + let args_data_offset = envs_array_offset + str_array_size(envs.len()); + let envs_data_offset = args_data_offset + args_data_size; - unsafe { - (write as *mut usize).write_volatile(args.len()); - } + // Header + writer.write_usize(virt + args_array_offset); + writer.write_usize(virt + envs_array_offset); - for (i, arg) in args.iter().enumerate() { - // Place the argument pointer - let ptr_place = write + (i * 2 + 1) * size_of::(); - let len_place = ptr_place + size_of::(); - unsafe { - (ptr_place as *mut usize).write_volatile(virt + offset); - (len_place as *mut usize).write_volatile(arg.len()); - } + // Args array + writer.write_usize(args.len()); + + let mut offset = args_data_offset; + for arg in args.iter() { + writer.write_usize(arg.len()); + writer.write_usize(virt + offset); offset += arg.len(); } - // Place the argument data - // TODO rewrite using write_foreign - unsafe { - let arg_data_slice = - core::slice::from_raw_parts_mut((write + args_ptr_size) as *mut u8, args_size); - let mut offset = 0; - for &s in args { - arg_data_slice[offset..offset + s.len()].copy_from_slice(s.as_bytes()); - offset += s.len(); - } + // Envs array + writer.write_usize(envs.len()); + + let mut offset = envs_data_offset; + for env in envs.iter() { + writer.write_usize(env.len()); + writer.write_usize(virt + offset); + offset += env.len(); + } + + // String data + for arg in args.iter() { + writer.write_bytes(arg.as_bytes()); + } + for env in envs.iter() { + writer.write_bytes(env.as_bytes()); } Ok(()) @@ -73,6 +134,7 @@ fn setup_binary>( space: ProcessAddressSpace, entry: usize, args: &[&str], + envs: &[&str], ) -> Result, Error> { const USER_STACK_PAGES: usize = 16; @@ -87,7 +149,7 @@ fn setup_binary>( MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; - setup_args(&space, virt_args_base, args)?; + setup_program_env(&space, virt_args_base, args, envs)?; debugln!( "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", @@ -124,9 +186,10 @@ pub fn load_elf>( name: S, file: FileRef, args: &[&str], + envs: &[&str], ) -> Result, Error> { let space = ProcessAddressSpace::new()?; let elf_entry = proc::elf::load_elf_from_file(&space, file)?; - setup_binary(name, space, elf_entry, args) + setup_binary(name, space, elf_entry, args, envs) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 40860ad6..2b2c0b4d 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -287,7 +287,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Setup a new process from the file let file = node.open(OpenOptions::READ, FileMode::empty())?; - let child = proc::exec::load_elf(options.program, file, options.arguments)?; + let child = proc::exec::load_elf( + options.program, + file, + options.arguments, + options.environment, + )?; let pid = child.id() as u32; // Inherit group and session from the creator From d0b8d99378ccfb33201e0ed62cec7031ec194f82 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 14 Nov 2023 17:31:57 +0200 Subject: [PATCH 082/211] proc: SetProcessGroup/GainTerminal SpawnOptions --- src/syscall/mod.rs | 38 ++++++++++++++++++++++++++++---------- src/task/process.rs | 5 ++++- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 2b2c0b4d..d1968b99 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -300,20 +300,38 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Inherit root from the creator let child_ioctx = IoContext::new(io.ioctx().root().clone()); + let mut child_io = child.io.lock(); + child_io.set_ioctx(child_ioctx); - { - let mut child_io = child.io.lock(); - - child_io.set_ioctx(child_ioctx); - - for opt in options.optional { - #[allow(irrefutable_let_patterns)] - if let SpawnOption::InheritFile { source, child } = opt { - let src_file = io.file(*source)?; - child_io.set_file(*child, src_file)?; + for opt in options.optional { + match opt { + &SpawnOption::InheritFile { source, child } => { + let src_file = io.file(source)?; + child_io.set_file(child, src_file)?; } + &SpawnOption::SetProcessGroup(pgroup) => { + child.set_group_id(pgroup as _); + } + _ => (), } } + + if let Some(fd) = options.optional.iter().find_map(|item| { + if let &SpawnOption::GainTerminal(fd) = item { + Some(fd) + } else { + None + } + }) { + debugln!("{} requested terminal {:?}", pid, fd); + let file = child_io.file(fd)?; + let node = file.borrow().node()?; + let mut req = DeviceRequest::SetTerminalGroup(child.group_id() as _); + + node.device_request(&mut req)?; + } + + drop(child_io); child.enqueue_somewhere(); Ok(pid as _) diff --git a/src/task/process.rs b/src/task/process.rs index a13beecd..071dd576 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -201,7 +201,10 @@ impl Process { } /// Sets the process group ID of the task - pub fn set_group_id(&self, gid: ProcessId) { + pub fn set_group_id(&self, mut gid: ProcessId) { + if gid == 0 { + gid = self.id(); + } self.inner.lock().group_id = gid; } From 75b1807e8ee82f1e3af6bb24e72b1c5f7977645c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 16 Nov 2023 00:16:38 +0200 Subject: [PATCH 083/211] aarch64: new memory arch impl --- lib/memtables/Cargo.toml | 4 + lib/memtables/src/aarch64.rs | 26 + lib/memtables/src/any.rs | 27 + lib/memtables/src/lib.rs | 37 +- lib/memtables/src/x86_64.rs | 27 + src/arch/aarch64/boot/entry.S | 13 +- src/arch/aarch64/boot/mod.rs | 217 ++++---- src/arch/aarch64/context.rs | 19 +- src/arch/aarch64/cpu.rs | 1 + src/arch/aarch64/devtree.rs | 0 src/arch/aarch64/exception.rs | 21 +- src/arch/aarch64/gic/gicc.rs | 4 +- src/arch/aarch64/gic/gicd.rs | 8 +- src/arch/aarch64/gic/mod.rs | 27 +- src/arch/aarch64/mem/mod.rs | 374 ++++++++++++++ src/arch/aarch64/mem/process.rs | 130 +++++ src/arch/aarch64/mem/table.rs | 295 +++++++++++ src/arch/aarch64/mod.rs | 465 ++++++++--------- src/arch/aarch64/plat_orange_pi3/mod.rs | 103 ---- src/arch/aarch64/plat_orange_pi3/r_wdog.rs | 90 ---- src/arch/aarch64/plat_orange_pi3/serial.rs | 160 ------ src/arch/aarch64/smp.rs | 43 +- src/arch/aarch64/table.rs | 553 --------------------- src/arch/aarch64/timer.rs | 6 +- src/arch/mod.rs | 1 - src/arch/x86_64/context.rs | 13 +- src/arch/x86_64/mem/process.rs | 13 +- src/arch/x86_64/mem/table.rs | 40 +- src/device/devtree.rs | 20 +- src/device/mod.rs | 5 +- src/device/power/mod.rs | 2 +- src/device/serial/mod.rs | 2 +- src/device/serial/pl011.rs | 30 +- src/mem/device.rs | 12 +- src/mem/phys/manager.rs | 17 +- src/mem/phys/mod.rs | 12 +- src/mem/process.rs | 30 +- src/mem/table.rs | 24 +- src/proc/elf.rs | 2 - src/proc/exec.rs | 2 +- src/task/context.rs | 7 +- tools/gentables/Cargo.toml | 2 +- tools/gentables/src/aarch64.rs | 193 +++++++ tools/gentables/src/main.rs | 33 +- tools/gentables/src/x86_64.rs | 2 +- 45 files changed, 1671 insertions(+), 1441 deletions(-) create mode 100644 lib/memtables/src/aarch64.rs create mode 100644 lib/memtables/src/any.rs create mode 100644 lib/memtables/src/x86_64.rs delete mode 100644 src/arch/aarch64/devtree.rs create mode 100644 src/arch/aarch64/mem/mod.rs create mode 100644 src/arch/aarch64/mem/process.rs create mode 100644 src/arch/aarch64/mem/table.rs delete mode 100644 src/arch/aarch64/plat_orange_pi3/mod.rs delete mode 100644 src/arch/aarch64/plat_orange_pi3/r_wdog.rs delete mode 100644 src/arch/aarch64/plat_orange_pi3/serial.rs delete mode 100644 src/arch/aarch64/table.rs create mode 100644 tools/gentables/src/aarch64.rs diff --git a/lib/memtables/Cargo.toml b/lib/memtables/Cargo.toml index 948e37e0..2697e627 100644 --- a/lib/memtables/Cargo.toml +++ b/lib/memtables/Cargo.toml @@ -7,3 +7,7 @@ edition = "2021" [dependencies] bytemuck = { version = "1.14.0", features = ["derive"] } + +[features] +default = [] +all = [] diff --git a/lib/memtables/src/aarch64.rs b/lib/memtables/src/aarch64.rs new file mode 100644 index 00000000..5adab8bc --- /dev/null +++ b/lib/memtables/src/aarch64.rs @@ -0,0 +1,26 @@ +use bytemuck::{Pod, Zeroable}; + +use crate::RawTable; + +pub const KERNEL_L3_COUNT: usize = 4; + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub struct FixedTables { + // 1GiB entries + pub l1: RawTable, + + // 2MiB entries + pub l2: RawTable, + pub l3s: [RawTable; KERNEL_L3_COUNT], +} + +impl FixedTables { + pub const fn zeroed() -> Self { + Self { + l1: RawTable::zeroed(), + l2: RawTable::zeroed(), + l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], + } + } +} diff --git a/lib/memtables/src/any.rs b/lib/memtables/src/any.rs new file mode 100644 index 00000000..1dcbcfe4 --- /dev/null +++ b/lib/memtables/src/any.rs @@ -0,0 +1,27 @@ +use crate::{aarch64, x86_64}; + +pub enum AnyTables { + X86_64(x86_64::FixedTables), + AArch64(aarch64::FixedTables), +} + +impl AnyTables { + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::X86_64(tables) => bytemuck::bytes_of(tables), + Self::AArch64(tables) => bytemuck::bytes_of(tables), + } + } +} + +impl From for AnyTables { + fn from(value: x86_64::FixedTables) -> Self { + Self::X86_64(value) + } +} + +impl From for AnyTables { + fn from(value: aarch64::FixedTables) -> Self { + Self::AArch64(value) + } +} diff --git a/lib/memtables/src/lib.rs b/lib/memtables/src/lib.rs index 79caa00c..fa3334e1 100644 --- a/lib/memtables/src/lib.rs +++ b/lib/memtables/src/lib.rs @@ -2,7 +2,20 @@ use bytemuck::{Pod, Zeroable}; -pub const KERNEL_L3_COUNT: usize = 16; +// AArch64 +#[cfg(any(feature = "all", target_arch = "aarch64"))] +pub mod aarch64; +#[cfg(all(not(feature = "all"), target_arch = "aarch64"))] +pub use aarch64::FixedTables; + +// x86-64 +#[cfg(any(feature = "all", target_arch = "x86_64"))] +pub mod x86_64; +#[cfg(all(not(feature = "all"), target_arch = "x86_64"))] +pub use x86_64::FixedTables; + +#[cfg(feature = "all")] +pub mod any; #[derive(Clone, Copy, Pod, Zeroable)] #[repr(C, align(0x1000))] @@ -10,30 +23,8 @@ pub struct RawTable { pub data: [u64; 512], } -#[derive(Clone, Copy, Pod, Zeroable)] -#[repr(C)] -pub struct FixedTables { - pub l0: RawTable, - - pub kernel_l1: RawTable, - pub kernel_l2: RawTable, - pub kernel_l3s: [RawTable; KERNEL_L3_COUNT], -} - impl RawTable { pub const fn zeroed() -> Self { Self { data: [0; 512] } } } - -impl FixedTables { - pub const fn zeroed() -> Self { - Self { - l0: RawTable::zeroed(), - - kernel_l1: RawTable::zeroed(), - kernel_l2: RawTable::zeroed(), - kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], - } - } -} diff --git a/lib/memtables/src/x86_64.rs b/lib/memtables/src/x86_64.rs new file mode 100644 index 00000000..cea2608a --- /dev/null +++ b/lib/memtables/src/x86_64.rs @@ -0,0 +1,27 @@ +use bytemuck::{Pod, Zeroable}; + +use crate::RawTable; + +pub const KERNEL_L3_COUNT: usize = 16; + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub struct FixedTables { + pub l0: RawTable, + + pub kernel_l1: RawTable, + pub kernel_l2: RawTable, + pub kernel_l3s: [RawTable; KERNEL_L3_COUNT], +} + +impl FixedTables { + pub const fn zeroed() -> Self { + Self { + l0: RawTable::zeroed(), + + kernel_l1: RawTable::zeroed(), + kernel_l2: RawTable::zeroed(), + kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], + } + } +} diff --git a/src/arch/aarch64/boot/entry.S b/src/arch/aarch64/boot/entry.S index 7cdfcc48..c41548c7 100644 --- a/src/arch/aarch64/boot/entry.S +++ b/src/arch/aarch64/boot/entry.S @@ -13,6 +13,12 @@ movk \reg, #((\value) >> 16), lsl #16 .endm +.macro MOV_ABS reg, sym + movz \reg, #:abs_g2:\sym + movk \reg, #:abs_g1_nc:\sym + movk \reg, #:abs_g0_nc:\sym +.endm + .macro LEAVE_EL2, ret_label mrs x8, CNTHCTL_EL2 orr x8, x8, #(CNTHCTL_EL2_EL1PCTEN | CNTHCTL_EL2_EL1PCEN) @@ -61,8 +67,9 @@ __aarch64_entry: isb // Zero .bss - adr x8, __bss_start_phys - adr x9, __bss_end_phys + MOV_ABS x8, __bss_start_phys + MOV_ABS x9, __bss_end_phys + // Zero .bss 1: cmp x8, x9 beq 2f @@ -84,7 +91,7 @@ __aarch64_entry: .section .text __aarch64_ap_entry: - // x0 -- stack pointer (lower address space) + // x0 -- physical sp mrs x8, CurrentEL lsr x8, x8, #2 diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index c209e74b..338f0910 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -1,135 +1,168 @@ -//! Main entry point for the AArch64 platforms -use core::{ - arch::{asm, global_asm}, - sync::atomic::Ordering, -}; +use core::{arch::global_asm, sync::atomic::Ordering}; use aarch64_cpu::{ asm::barrier, - registers::{CPACR_EL1, TTBR0_EL1}, + registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; -use tock_registers::interfaces::{ReadWriteable, Writeable}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use super::{cpu::Cpu, exception, AArch64, KernelStack, ARCHITECTURE, BOOT_STACK_SIZE}; +use super::{ + exception, mem::load_fixed_tables, smp::CPU_COUNT, AArch64, BootStack, ARCHITECTURE, + BOOT_STACK_SIZE, +}; use crate::{ - absolute_address, - arch::{aarch64::smp::CPU_COUNT, Architecture, ArchitectureImpl}, + arch::{aarch64::mem::table::L3, Architecture}, fs::devfs, kernel_main, kernel_secondary_main, - mem::{ - heap, - phys::{self, PageUsage}, - ConvertAddress, KERNEL_VIRT_OFFSET, - }, + mem::{address::IntoRaw, phys, table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET}, + task::runtime, }; -extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { - // Unmask FP operations - CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); - - unsafe { - ARCHITECTURE.init_mmu(true); +unsafe fn pre_init_mmu() { + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { + // TODO early panic + loop {} } - let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE).virtualize() }; - let elr = absolute_address!(__aarch64_bsp_upper_entry); + MAIR_EL1.write( + //// Attribute 0 -- normal memory + // Inner + MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + // Outer + MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + //// Attribute 1 -- device memory + MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, + ); - barrier::dsb(barrier::SY); - barrier::isb(barrier::SY); - enter_higher_half(sp as usize, elr, dtb_phys); + TCR_EL1.modify( + // General + TCR_EL1::IPS::Bits_48 + + // TTBR0 + TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner + + // TTBR1 + TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, + ); } -unsafe extern "C" fn el1_ap_lower_entry(sp: usize) -> ! { - ArchitectureImpl::set_interrupt_mask(true); +unsafe fn enable_mmu() { + barrier::dmb(barrier::ISH); + + SCTLR_EL1.modify( + // Enable translation + SCTLR_EL1::M::Enable + + // (TODO) Disable I + D caches + SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable, + ); + + barrier::isb(barrier::SY); +} + +unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! { + unsafe { + core::arch::asm!(r#" + mov sp, {sp} + mov x0, {x0} + mov lr, xzr + br {elr} + "#, elr = in(reg) elr, sp = in(reg) sp, x0 = in(reg) x0, options(noreturn)); + } +} + +// NOTE executes in "lower-half" address space, MMU not yet enabled +unsafe extern "C" fn __aarch64_el1_bsp_lower_entry(dtb: PhysicalAddress) -> ! { + AArch64::set_interrupt_mask(true); + + // Don't trap FP operations + CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); + + // Setup MMU to jump to "higher-half" address space + pre_init_mmu(); + load_fixed_tables(); + enable_mmu(); + + // Safety: SP points to the .bss section, so it's +offset mapped + let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE) as usize } + KERNEL_VIRT_OFFSET; + let elr = absolute_address!(__aarch64_bsp_upper_entry); + + // TODO pass dtb + enter_higher_half(sp, elr, dtb.into_raw()); +} + +unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { + // Remove the "lower-half" mapping, no longer needed + TTBR0_EL1.set(0); + + // Setup the "runtime" part of the kernel tables + ARCHITECTURE + .init_memory_management(dtb) + .expect("Could not initialize memory management"); + barrier::isb(barrier::SY); + + exception::init_exceptions(); + + // // Setup initrd + // super::setup_initrd(); + + devfs::init(); + + runtime::init_task_queue(); + + // Initialize the BSP CPU + the devices + ARCHITECTURE + .init_platform(true) + .expect("Could not initialize the platform"); + + kernel_main() +} + +unsafe extern "C" fn __aarch64_el1_ap_lower_entry(sp: PhysicalAddress) -> ! { + const AP_STACK_PAGES: usize = 8; + AArch64::set_interrupt_mask(true); // Unmask FP operations CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); - unsafe { - ARCHITECTURE.init_mmu(false); - } + pre_init_mmu(); + load_fixed_tables(); + enable_mmu(); + + let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES).unwrap(); + let stack_base = stack_pages.virtualize_raw(); + let sp = stack_base + L3::SIZE * AP_STACK_PAGES; - let sp = sp.virtualize(); let elr = absolute_address!(__aarch64_ap_upper_entry); enter_higher_half(sp, elr, 0); } -fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { - unsafe { - asm!(r#" - mov sp, {sp} - mov x0, {arg} - br {entry} - "#, entry = in(reg) elr, arg = in(reg) arg, sp = in(reg) sp, options(noreturn)); - } -} - -unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { - // NOTE it is critical that the code does not panic until the debug is set up, otherwise no - // message will be displayed - - // Unmap TTBR0 - TTBR0_EL1.set(0); +extern "C" fn __aarch64_ap_upper_entry() -> ! { + barrier::dmb(barrier::ISH); barrier::isb(barrier::SY); - ARCHITECTURE.init_device_tree(dtb_phys); - - AArch64::set_interrupt_mask(true); - - exception::init_exceptions(); - - // Setup initrd - super::setup_initrd(); - - ARCHITECTURE - .init_physical_memory(dtb_phys) - .expect("Failed to initialize the physical memory manager"); - - // Setup heap - let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) - .expect("Could not allocate a block for heap"); - - heap::init_heap(heap_base.virtualize(), 16 * 0x1000); - - devfs::init(); - - // Enumerate the device tree - ARCHITECTURE.init_platform(true); - - Cpu::init_local(); - - kernel_main() -} - -extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { - assert!(ArchitectureImpl::interrupt_mask()); - - // Signal to BSP that we're up - CPU_COUNT.fetch_add(1, Ordering::SeqCst); + let cpu_id = CPU_COUNT.fetch_add(1, Ordering::SeqCst); aarch64_cpu::asm::sev(); + infoln!("cpu{} initializing", cpu_id); + exception::init_exceptions(); - // Initialize CPU-local GIC and timer unsafe { ARCHITECTURE.init_platform(false); - - Cpu::init_local(); - - kernel_secondary_main() } + + kernel_secondary_main() } -static BSP_STACK: KernelStack = KernelStack { +#[link_section = ".bss"] +static BSP_STACK: BootStack = BootStack { data: [0; BOOT_STACK_SIZE], }; global_asm!( include_str!("entry.S"), - kernel_lower_entry = sym el1_bsp_lower_entry, - kernel_ap_lower_entry = sym el1_ap_lower_entry, + kernel_lower_entry = sym __aarch64_el1_bsp_lower_entry, + kernel_ap_lower_entry = sym __aarch64_el1_ap_lower_entry, stack_bottom = sym BSP_STACK, - stack_size = const BOOT_STACK_SIZE, - kernel_virt_offset = const KERNEL_VIRT_OFFSET + kernel_virt_offset = const KERNEL_VIRT_OFFSET, + stack_size = const BOOT_STACK_SIZE ); diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 1da9a20d..508e4dbb 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -4,10 +4,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use crate::{ - mem::{ - phys::{self, PageUsage}, - ConvertAddress, - }, + mem::{address::IntoRaw, phys, PhysicalAddress}, task::context::TaskContextImpl, }; @@ -64,8 +61,8 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, ttbr0: usize) { - self.push(ttbr0); // ttbr0_el1 + fn init_common(&mut self, entry: usize, ttbr0: u64) { + self.push(ttbr0 as _); // ttbr0_el1 self.push(0); // tpidr_el0 self.push(entry); // x30/lr @@ -89,9 +86,8 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 8; - let stack_base = unsafe { - phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() - }; + let stack_base = + unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() }; let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -112,10 +108,9 @@ impl TaskContextImpl for TaskContext { }) } - fn user(entry: usize, arg: usize, ttbr0: usize, user_stack_sp: usize) -> Result { + fn user(entry: usize, arg: usize, ttbr0: u64, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 16; - let stack_base = - unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; + let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() }; let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 8ab4463a..2ef75b42 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -122,6 +122,7 @@ impl Cpu { match msg { CpuMessage::Panic => panic::panic_secondary(), + CpuMessage::Shutdown => todo!(), } } } diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 4bd89603..e82417c5 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -16,10 +16,7 @@ use abi::{ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{ - aarch64::{cpu::Cpu, table::AddressSpace}, - Architecture, ArchitectureImpl, - }, + arch::{aarch64::cpu::Cpu, Architecture, ArchitectureImpl}, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, process::Process}, @@ -179,14 +176,14 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { _ => (), } - unsafe { - let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); - let far = FAR_EL1.get() as usize; - space.walk(far, |level, raw| { - log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); - true - }); - } + // unsafe { + // let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); + // let far = FAR_EL1.get() as usize; + // space.walk(far, |level, raw| { + // log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); + // true + // }); + // } log_print_raw!(LogLevel::Fatal, "System register dump:\n"); log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs index cd55b3a6..1f11d485 100644 --- a/src/arch/aarch64/gic/gicc.rs +++ b/src/arch/aarch64/gic/gicc.rs @@ -36,11 +36,11 @@ register_structs! { } pub(super) struct Gicc { - regs: DeviceMemoryIo, + regs: DeviceMemoryIo<'static, GiccRegs>, } impl Gicc { - pub const fn new(regs: DeviceMemoryIo) -> Self { + pub const fn new(regs: DeviceMemoryIo<'static, GiccRegs>) -> Self { Self { regs } } diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index 373839b5..60314cc8 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -65,8 +65,8 @@ register_structs! { } pub(super) struct Gicd { - shared_regs: Spinlock>, - banked_regs: DeviceMemoryIo, + shared_regs: Spinlock>, + banked_regs: DeviceMemoryIo<'static, GicdBankedRegs>, } impl GicdSharedRegs { @@ -85,8 +85,8 @@ impl GicdSharedRegs { impl Gicd { pub const fn new( - shared_regs: DeviceMemoryIo, - banked_regs: DeviceMemoryIo, + shared_regs: DeviceMemoryIo<'static, GicdSharedRegs>, + banked_regs: DeviceMemoryIo<'static, GicdBankedRegs>, ) -> Self { let shared_regs = Spinlock::new(shared_regs); Self { diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 0bb60616..4b528b5d 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -4,7 +4,7 @@ use core::sync::atomic::Ordering; use aarch64_cpu::asm::barrier; use abi::error::Error; -use alloc::boxed::Box; +use alloc::{boxed::Box, sync::Arc}; use device_api::{ interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, @@ -18,7 +18,11 @@ use crate::{ arch::{aarch64::IrqNumber, Architecture, CpuMessage}, device::devtree::{self, DevTreeIndexPropExt}, device_tree_driver, - mem::device::{DeviceMemory, DeviceMemoryIo}, + mem::{ + address::FromRaw, + device::{DeviceMemoryIo, DeviceMemoryMapping, RawDeviceMemoryMapping}, + PhysicalAddress, + }, sync::IrqSafeSpinlock, }; @@ -36,8 +40,8 @@ pub mod gicd; pub struct Gic { gicc: OneTimeInit, gicd: OneTimeInit, - gicd_base: usize, - gicc_base: usize, + gicd_base: PhysicalAddress, + gicc_base: PhysicalAddress, table: IrqSafeSpinlock>, } @@ -47,10 +51,10 @@ impl Device for Gic { } unsafe fn init(&'static self) -> Result<(), Error> { - let gicd_mmio = DeviceMemory::map("GICv2 Distributor registers", self.gicd_base, 0x1000)?; - let gicd_mmio_shared = DeviceMemoryIo::new(gicd_mmio.clone()); - let gicd_mmio_banked = DeviceMemoryIo::new(gicd_mmio); - let gicc_mmio = DeviceMemoryIo::map("GICv2 CPU registers", self.gicc_base)?; + let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map(self.gicd_base, 0x1000)?); + let gicd_mmio_shared = DeviceMemoryIo::from_raw(gicd_mmio.clone())?; + let gicd_mmio_banked = DeviceMemoryIo::from_raw(gicd_mmio)?; + let gicc_mmio = DeviceMemoryIo::map(self.gicc_base)?; let gicd = Gicd::new(gicd_mmio_shared, gicd_mmio_banked); let gicc = Gicc::new(gicc_mmio); @@ -169,7 +173,7 @@ impl Gic { /// # Safety /// /// The caller must ensure the addresses actually point to the GIC components. - pub const unsafe fn new(gicd_base: usize, gicc_base: usize) -> Self { + pub const unsafe fn new(gicd_base: PhysicalAddress, gicc_base: PhysicalAddress) -> Self { Self { gicc: OneTimeInit::new(), gicd: OneTimeInit::new(), @@ -188,6 +192,9 @@ device_tree_driver! { let (gicc_base, _) = reg.cell2_array_item(0, dt.address_cells, dt.size_cells)?; let (gicd_base, _) = reg.cell2_array_item(1, dt.address_cells, dt.size_cells)?; - Some(Box::new(unsafe { Gic::new(gicc_base as usize, gicd_base as usize) })) + Some(Box::new(unsafe { Gic::new( + PhysicalAddress::from_raw(gicc_base), + PhysicalAddress::from_raw(gicd_base), + )})) } } diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs new file mode 100644 index 00000000..8ef804f2 --- /dev/null +++ b/src/arch/aarch64/mem/mod.rs @@ -0,0 +1,374 @@ +use core::{ + alloc::Layout, + ops::{Deref, DerefMut}, +}; + +use abi::error::Error; +use cfg_if::cfg_if; +use kernel_util::util::OneTimeInit; +use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; + +use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1}; +use static_assertions::const_assert_eq; +use tock_registers::interfaces::Writeable; + +use crate::mem::{ + address::{FromRaw, IntoRaw, KernelImageObject}, + device::RawDeviceMemoryMapping, + table::EntryLevel, + PhysicalAddress, KERNEL_VIRT_OFFSET, +}; + +use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}; + +pub mod process; +pub(super) mod table; + +// TODO eliminate this requirement by using precomputed indices +const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET; +cfg_if! { + if #[cfg(feature = "aarch64_qemu")] { + const KERNEL_PHYS_BASE: usize = 0x40080000; + } else { + const KERNEL_PHYS_BASE: usize = 0x40080000; + } +} + +// Precomputed mappings +const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_END_L2_INDEX: usize = + L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE) + KERNEL_L3_COUNT; + +// Must not be zero, should be at 4MiB +const_assert_eq!(KERNEL_START_L2_INDEX, 0); +// From static mapping +const_assert_eq!(KERNEL_L1_INDEX, 1); + +// Runtime mappings +// 2MiB max +const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1; +// 1GiB max +const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; +// 1GiB max +const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; +// 16GiB max +pub(super) const RAM_MAPPING_L1_COUNT: usize = 16; +const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3; +const RAM_MAPPING_END_L1I: usize = RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT; + +const DEVICE_MAPPING_L3_COUNT: usize = 4; + +// 2MiB for early mappings +const EARLY_MAPPING_OFFSET: usize = + MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE); +static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); +// 1GiB for heap mapping +pub(super) const HEAP_MAPPING_OFFSET: usize = MAPPING_OFFSET | (HEAP_MAPPING_L1I * L1::SIZE); +pub(super) static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); +// 1GiB for device MMIO mapping +const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE); +static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); +static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = + [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; +// 16GiB for RAM mapping +pub(super) const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE); +pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); + +#[link_section = ".data.tables"] +pub static mut KERNEL_TABLES: KernelImageObject = + unsafe { KernelImageObject::new(FixedTables::zeroed()) }; + +pub struct EarlyMapping<'a, T: ?Sized> { + value: &'a mut T, + page_count: usize, +} + +impl<'a, T: Sized> EarlyMapping<'a, T> { + pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { + let layout = Layout::new::(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = &mut *((virt + offset) as *mut T); + + Ok(EarlyMapping { value, page_count }) + } + + pub unsafe fn map_slice( + physical: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); + + Ok(EarlyMapping { value, page_count }) + } +} + +impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { + fn drop(&mut self) { + let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); + + for i in 0..self.page_count { + let page = address + i * L3::SIZE; + + unsafe { + unmap_early_page(page); + } + } + } +} + +fn kernel_table_flags() -> PageAttributes { + PageAttributes::TABLE + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | PageAttributes::PRESENT +} + +fn ram_block_flags() -> PageAttributes { + // TODO UXN, PXN + PageAttributes::BLOCK + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | PageAttributes::PRESENT +} + +// Early mappings +unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { + for l3i in 0..512 { + let mut taken = false; + for i in 0..count { + if EARLY_MAPPING_L3[i + l3i].is_present() { + taken = true; + break; + } + } + + if taken { + continue; + } + + for i in 0..count { + let page = physical.add(i * L3::SIZE); + // TODO NX, NC + EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty()); + } + + return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn unmap_early_page(address: usize) { + if address < EARLY_MAPPING_OFFSET || address >= EARLY_MAPPING_OFFSET + L2::SIZE { + panic!("Tried to unmap invalid early mapping: {:#x}", address); + } + + let l3i = L3::index(address - EARLY_MAPPING_OFFSET); + + assert!(EARLY_MAPPING_L3[l3i].is_present()); + EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; + + // TODO invalidate tlb +} + +pub(super) unsafe fn map_ram_l1(index: usize) { + if index >= RAM_MAPPING_L1_COUNT { + loop {} + } + assert_eq!(KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I], 0); + + KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I] = + ((index * L1::SIZE) as u64) | ram_block_flags().bits(); +} + +pub(super) unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) { + if index >= 512 { + loop {} + } + assert!(!HEAP_MAPPING_L2[index].is_present()); + // TODO UXN, PXN + HEAP_MAPPING_L2[index] = PageEntry::normal_block(page, PageAttributes::empty()); +} + +// Device mappings +unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result { + // TODO don't map pages if already mapped + + 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { + continue 'l0; + } + } + + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + // TODO NX, NC + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE)); + } + + return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result { + 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { + for j in 0..count { + if DEVICE_MAPPING_L2[i + j].is_present() { + continue 'l0; + } + } + + for j in 0..count { + DEVICE_MAPPING_L2[i + j] = PageEntry::::device_block(base.add(j * L2::SIZE)); + } + + debugln!( + "map l2s: base={:#x}, count={} -> {:#x}", + base, + count, + DEVICE_MAPPING_OFFSET + i * L2::SIZE + ); + return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); + } + + Err(Error::OutOfMemory) +} + +pub(super) unsafe fn map_device_memory( + base: PhysicalAddress, + size: usize, +) -> Result { + // debugln!("Map {}B @ {:#x}", size, base); + let l3_aligned = base.page_align_down::(); + let l3_offset = L3::page_offset(base.into_raw()); + let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE; + + if page_count > 256 { + // Large mapping, use L2 mapping instead + let l2_aligned = base.page_align_down::(); + let l2_offset = L2::page_offset(base.into_raw()); + let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE; + + let base_address = map_device_memory_l2(l2_aligned, page_count)?; + let address = base_address + l2_offset; + + Ok(RawDeviceMemoryMapping { + address, + base_address, + page_count, + page_size: L2::SIZE, + }) + } else { + // Just map the pages directly + let base_address = map_device_memory_l3(l3_aligned, page_count)?; + let address = base_address + l3_offset; + + Ok(RawDeviceMemoryMapping { + address, + base_address, + page_count, + page_size: L3::SIZE, + }) + } +} + +pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { + // debugln!( + // "Unmap {}B @ {:#x}", + // map.page_count * map.page_size, + // map.base_address + // ); + match map.page_size { + L3::SIZE => { + for i in 0..map.page_count { + let page = map.base_address + i * L3::SIZE; + let l2i = L2::index(page); + let l3i = L3::index(page); + assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; + // TODO flush the TLB entry + loop {} + // intrinsics::flush_tlb_entry(page); + } + } + L2::SIZE => todo!(), + _ => unimplemented!(), + } +} + +/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half" +/// +/// # Safety +/// +/// Unsafe, must only be called by BSP during its early init while still in "lower-half" +pub(super) unsafe fn load_fixed_tables() { + let ttbr0 = KERNEL_TABLES.l1.data.as_ptr() as u64; + TTBR0_EL1.set(ttbr0); + TTBR1_EL1.set(ttbr0); +} + +/// Sets up additional translation tables for kernel usage +/// +/// # Safety +/// +/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half" +pub(super) unsafe fn init_fixed_tables() { + // TODO this could be built in compile-time too? + let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + + for i in 0..DEVICE_MAPPING_L3_COUNT { + let device_mapping_l3_phys = PhysicalAddress::from_raw( + &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, + ); + DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty()); + } + + assert_eq!(KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I], 0); + KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I] = + (early_mapping_l3_phys as u64) | kernel_table_flags().bits(); + + assert_eq!(KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I], 0); + KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I] = + (heap_mapping_l2_phys as u64) | kernel_table_flags().bits(); + + assert_eq!(KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I], 0); + KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I] = + (device_mapping_l2_phys as u64) | kernel_table_flags().bits(); +} diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs new file mode 100644 index 00000000..0f9370c2 --- /dev/null +++ b/src/arch/aarch64/mem/process.rs @@ -0,0 +1,130 @@ +use core::sync::atomic::{AtomicU8, Ordering}; + +use abi::error::Error; + +use crate::mem::{ + address::{AsPhysicalAddress, IntoRaw}, + phys, + pointer::PhysicalRefMut, + process::ProcessAddressSpaceManager, + table::{EntryLevel, MapAttributes, NextPageTable}, + PhysicalAddress, +}; + +use super::table::{PageEntry, PageTable, L1, L2, L3}; + +#[repr(C)] +pub struct ProcessAddressSpaceImpl { + l1: PhysicalRefMut<'static, PageTable>, + asid: u8, +} + +impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { + const PAGE_SIZE: usize = L3::SIZE; + const LOWER_LIMIT_PFN: usize = 8; + // 16GiB VM limit + const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE; + + fn new() -> Result { + static LAST_ASID: AtomicU8 = AtomicU8::new(1); + + let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); + + let mut l1 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; + + for i in 0..512 { + l1[i] = PageEntry::INVALID; + } + + Ok(Self { l1, asid }) + } + + fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> { + self.read_l3_entry(address).ok_or(Error::DoesNotExist) + } + + unsafe fn map_page( + &mut self, + address: usize, + physical: PhysicalAddress, + flags: MapAttributes, + ) -> Result<(), Error> { + self.write_l3_entry( + address, + PageEntry::normal_page(physical, flags.into()), + false, + ) + } + + unsafe fn unmap_page(&mut self, address: usize) -> Result { + self.pop_l3_entry(address) + } + + fn as_address_with_asid(&self) -> u64 { + unsafe { u64::from(self.l1.as_physical_address()) | ((self.asid as u64) << 48) } + } +} + +impl ProcessAddressSpaceImpl { + // Write a single 4KiB entry + fn write_l3_entry( + &mut self, + virt: usize, + entry: PageEntry, + overwrite: bool, + ) -> Result<(), Error> { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let mut l2 = self.l1.get_mut_or_alloc(l1i)?; + let mut l3 = l2.get_mut_or_alloc(l2i)?; + + if l3[l3i].is_present() && !overwrite { + todo!(); + } + + l3[l3i] = entry; + tlb_flush_vaae1(virt); + + Ok(()) + } + + fn pop_l3_entry(&mut self, virt: usize) -> Result { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + // TODO somehow drop tables if they're known to be empty? + let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?; + + l3[l3i] = PageEntry::INVALID; + tlb_flush_vaae1(virt); + + Ok(page) + } + + fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l2 = self.l1.get(l1i)?; + let l3 = l2.get(l2i)?; + + let page = l3[l3i].as_page()?; + + Some((page, l3[l3i].attributes().into())) + } +} + +#[inline] +fn tlb_flush_vaae1(mut page: usize) { + page >>= 12; + unsafe { + core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); + } +} diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs new file mode 100644 index 00000000..36daff94 --- /dev/null +++ b/src/arch/aarch64/mem/table.rs @@ -0,0 +1,295 @@ +use core::{ + marker::PhantomData, + ops::{Index, IndexMut}, +}; + +use abi::error::Error; +use bitflags::bitflags; + +use crate::mem::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw}, + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, + PhysicalAddress, +}; + +bitflags! { + #[derive(Clone, Copy, PartialEq, Eq)] + pub struct PageAttributes: u64 { + const PRESENT = 1 << 0; + + const TABLE = 1 << 1; + const PAGE = 1 << 1; + const BLOCK = 0 << 1; + + const ACCESS = 1 << 10; + + const AP_KERNEL_READWRITE = 0 << 6; + const AP_BOTH_READWRITE = 1 << 6; + const AP_KERNEL_READONLY = 2 << 6; + const AP_BOTH_READONLY = 3 << 6; + const AP_ACCESS_MASK = 3 << 6; + + const SH_OUTER = 2 << 8; + const SH_INNER = 3 << 8; + + const PAGE_ATTR_NORMAL = 0 << 2; + const PAGE_ATTR_DEVICE = 1 << 2; + + const NON_GLOBAL = 1 << 11; + + const PXN = 1 << 53; + const UXN = 1 << 54; + } +} + +#[derive(Clone, Copy)] +#[repr(C, align(0x1000))] +pub struct PageTable { + entries: [PageEntry; 512], +} + +#[derive(Clone, Copy)] +pub struct PageEntry(u64, PhantomData); + +#[derive(Clone, Copy)] +pub struct L1; +#[derive(Clone, Copy)] +pub struct L2; +#[derive(Clone, Copy)] +pub struct L3; + +impl const EntryLevel for L1 { + const SHIFT: usize = 30; +} + +impl NonTerminalEntryLevel for L1 { + type NextLevel = L2; +} + +impl const EntryLevel for L2 { + const SHIFT: usize = 21; +} + +impl NonTerminalEntryLevel for L2 { + type NextLevel = L3; +} + +impl const EntryLevel for L3 { + const SHIFT: usize = 12; +} + +impl PageTable { + pub const fn zeroed() -> Self { + Self { + entries: [PageEntry::INVALID; 512], + } + } + + pub fn new_zeroed<'a>() -> Result, Error> { + let physical = phys::alloc_page()?; + let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) }; + + for i in 0..512 { + table[i] = PageEntry::INVALID; + } + + Ok(table) + } +} + +impl PageEntry { + pub const INVALID: Self = Self(0, PhantomData); + + pub const fn is_present(self) -> bool { + self.0 & PageAttributes::PRESENT.bits() != 0 + } + + pub fn attributes(self) -> PageAttributes { + PageAttributes::from_bits_retain(self.0) + } +} + +impl NextPageTable for PageTable { + type NextLevel = PageTable; + type TableRef = PhysicalRef<'static, PageTable>; + type TableRefMut = PhysicalRefMut<'static, PageTable>; + + fn get(&self, index: usize) -> Option { + self[index] + .as_table() + .map(|phys| unsafe { PhysicalRef::map(phys) }) + } + + fn get_mut(&mut self, index: usize) -> Option { + self[index] + .as_table() + .map(|phys| unsafe { PhysicalRefMut::map(phys) }) + } + + fn get_mut_or_alloc(&mut self, index: usize) -> Result { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + Ok(unsafe { PhysicalRefMut::map(table) }) + } else { + let table = PageTable::new_zeroed()?; + self[index] = PageEntry::::table( + unsafe { table.as_physical_address() }, + PageAttributes::empty(), + ); + Ok(table) + } + } +} + +impl PageEntry { + pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), + PhantomData, + ) + } + + pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_block(phys: PhysicalAddress) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | PageAttributes::UXN + | PageAttributes::PXN) + .bits(), + PhantomData, + ) + } + + /// Returns the physical address of the table this entry refers to, returning None if it + /// does not + pub fn as_table(self) -> Option { + if self.0 & PageAttributes::PRESENT.bits() != 0 + && self.0 & PageAttributes::BLOCK.bits() == 0 + { + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) + } else { + None + } + } +} + +impl PageEntry { + pub fn normal_page(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_page(phys: PhysicalAddress) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | PageAttributes::UXN + | PageAttributes::PXN) + .bits(), + PhantomData, + ) + } + + pub fn as_page(&self) -> Option { + let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); + if self.0 & mask == mask { + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) + } else { + None + } + } +} + +impl Index for PageTable { + type Output = PageEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +impl From for PageAttributes { + fn from(value: MapAttributes) -> Self { + let mut out = PageAttributes::empty(); + // TODO kernel cannot write el0 readonly pages + if value.contains(MapAttributes::USER_WRITE) { + // Read/write + out |= PageAttributes::AP_BOTH_READWRITE; + } else if value.contains(MapAttributes::USER_READ) { + // Read only + out |= PageAttributes::AP_BOTH_READONLY; + } else { + // No read/write + out |= PageAttributes::AP_KERNEL_READONLY; + } + + if value.contains(MapAttributes::NON_GLOBAL) { + out |= PageAttributes::NON_GLOBAL; + } + + out + } +} + +impl From for MapAttributes { + fn from(value: PageAttributes) -> Self { + let mut out = MapAttributes::empty(); + + out |= match value.intersection(PageAttributes::AP_ACCESS_MASK) { + PageAttributes::AP_BOTH_READWRITE => { + MapAttributes::USER_WRITE | MapAttributes::USER_READ + } + PageAttributes::AP_BOTH_READONLY => MapAttributes::USER_READ, + PageAttributes::AP_KERNEL_READONLY => MapAttributes::empty(), + PageAttributes::AP_KERNEL_READWRITE => panic!("This variant cannot be constructed"), + _ => unreachable!(), + }; + + if value.contains(PageAttributes::NON_GLOBAL) { + out |= MapAttributes::NON_GLOBAL; + } + + out + } +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 407924fe..71c3d84b 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -1,49 +1,47 @@ //! AArch64 architecture and platforms implementation +pub mod mem; + use core::sync::atomic::Ordering; -use aarch64_cpu::{ - asm::barrier, - registers::{ - CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, - TTBR0_EL1, TTBR1_EL1, - }, -}; +use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF}; use abi::error::Error; use device_api::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use fdt_rs::prelude::PropReader; +use fdt_rs::base::DevTree; use git_version::git_version; use kernel_util::util::OneTimeInit; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::Architecture, + arch::aarch64::{ + cpu::Cpu, + mem::table::{L2, L3}, + }, debug, device::{ self, - devtree::{ - self, DevTreeIndexNodePropGet, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, - FdtMemoryRegionIter, - }, + devtree::{self, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}, power::arm_psci::Psci, }, fs::{Initrd, INITRD_DATA}, mem::{ + address::{FromRaw, IntoRaw}, + device::RawDeviceMemoryMapping, + heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, - ConvertAddress, + pointer::PhysicalRef, + table::EntryLevel, + PhysicalAddress, }, }; -use self::{ - smp::CPU_COUNT, - table::{init_fixed_tables, KERNEL_TABLES}, -}; +use self::mem::{table::L1, EarlyMapping}; -use super::CpuMessage; +use super::{Architecture, CpuMessage}; pub mod boot; pub mod context; @@ -51,111 +49,44 @@ pub mod cpu; pub mod exception; pub mod gic; pub mod smp; -pub mod table; pub mod timer; -const BOOT_STACK_SIZE: usize = 65536; +const BOOT_STACK_SIZE: usize = 4096 * 32; #[derive(Clone, Copy)] #[repr(C, align(0x20))] -struct KernelStack { +struct BootStack { data: [u8; BOOT_STACK_SIZE], } -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum IrqNumber { - Private(u32), - Shared(u32), -} - -/// AArch64 platform interface pub struct AArch64 { dt: OneTimeInit>, - ext_intc: OneTimeInit<&'static dyn ExternalInterruptController>, - local_intc: OneTimeInit<&'static dyn LocalInterruptController>, - mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, + + pub psci: OneTimeInit<&'static Psci>, reset: OneTimeInit<&'static dyn ResetDevice>, - // ARM-only devices - /// ARM PSCI instance on this system (there may not be one) - pub psci: OneTimeInit<&'static Psci>, + lintc: OneTimeInit<&'static dyn LocalInterruptController>, + xintc: OneTimeInit<&'static dyn ExternalInterruptController>, + + mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, + + initrd: OneTimeInit>, } -/// Global platform handle -pub static ARCHITECTURE: AArch64 = AArch64 { - dt: OneTimeInit::new(), - ext_intc: OneTimeInit::new(), - local_intc: OneTimeInit::new(), - mtimer: OneTimeInit::new(), - reset: OneTimeInit::new(), - - // ARM-only devices - psci: OneTimeInit::new(), -}; - impl Architecture for AArch64 { - type IrqNumber = IrqNumber; - const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; - unsafe fn init_mmu(&self, bsp: bool) { - if bsp { - init_fixed_tables(); - } - - let tables_phys = absolute_address!(KERNEL_TABLES).physicalize() as u64; - - if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { - todo!(); - } - - MAIR_EL1.write( - // Attribute 0 -- normal memory - MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + - MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + - // Attribute 1 -- device memory - MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, - ); - - TCR_EL1.modify( - // General - TCR_EL1::IPS::Bits_48 + - // TTBR0 - TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner + - // TTBR1 - TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, - ); - - barrier::dmb(barrier::ISH); - - TTBR0_EL1.set_baddr(tables_phys); - TTBR1_EL1.set_baddr(tables_phys); - - barrier::isb(barrier::SY); - - // Enable instruction cache, data cache and translation - SCTLR_EL1 - .modify(SCTLR_EL1::M::Enable + SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable); - - barrier::isb(barrier::SY); - } + type IrqNumber = IrqNumber; unsafe fn start_application_processors(&self) { let dt = self.dt.get(); - if let Err(e) = smp::start_ap_cores(dt) { - errorln!( - "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", - e - ); + if let Err(error) = smp::start_ap_cores(dt) { + errorln!("Could not initialize AP CPUs: {:?}", error); } } - fn map_device_pages(&self, phys: usize, count: usize) -> Result { - unsafe { KERNEL_TABLES.map_device_pages(phys, count) } - } - - fn wait_for_interrupt() { - aarch64_cpu::asm::wfi(); + fn cpu_count() -> usize { + smp::CPU_COUNT.load(Ordering::Acquire) } unsafe fn set_interrupt_mask(mask: bool) { @@ -170,26 +101,98 @@ impl Architecture for AArch64 { DAIF.read(DAIF::I) != 0 } - fn cpu_count() -> usize { - CPU_COUNT.load(Ordering::Acquire) + fn wait_for_interrupt() { + aarch64_cpu::asm::wfi(); } - fn register_external_interrupt_controller( + unsafe fn map_device_memory( &self, - intc: &'static dyn ExternalInterruptController, + base: PhysicalAddress, + size: usize, + ) -> Result { + mem::map_device_memory(base, size) + } + + unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) { + mem::unmap_device_memory(map) + } + + fn map_physical_memory + Clone>( + &self, + _it: I, + _memory_start: PhysicalAddress, + memory_end: PhysicalAddress, ) -> Result<(), Error> { - self.ext_intc.init(intc); + let end_l1i = L1::index(memory_end.page_align_up::().into_raw()); + if end_l1i > mem::RAM_MAPPING_L1_COUNT { + loop {} + } + + // Map 1GiB chunks + for index in 0..end_l1i { + unsafe { + mem::map_ram_l1(index); + } + } + + mem::MEMORY_LIMIT.init(memory_end.into_raw()); + Ok(()) } + fn virtualize(address: PhysicalAddress) -> Result { + let raw: usize = address.into_raw(); + if raw < *mem::MEMORY_LIMIT.get() { + Ok(raw + mem::RAM_MAPPING_OFFSET) + } else { + errorln!("Invalid physical address: {:#x}", address); + Err(Error::InvalidMemoryOperation) + } + } + + fn physicalize(address: usize) -> Result { + if address < mem::RAM_MAPPING_OFFSET + || address - mem::RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() + { + errorln!("Not a virtualized physical address: {:#x}", address); + return Err(Error::InvalidMemoryOperation); + } + + Ok(PhysicalAddress::from_raw(address - mem::RAM_MAPPING_OFFSET)) + } + + fn local_interrupt_controller( + &'static self, + ) -> &'static dyn LocalInterruptController { + *self.lintc.get() + } + + fn external_interrupt_controller( + &'static self, + ) -> &'static dyn ExternalInterruptController { + *self.xintc.get() + } + fn register_local_interrupt_controller( &self, intc: &'static dyn LocalInterruptController, ) -> Result<(), Error> { - self.local_intc.init(intc); + self.lintc.init(intc); Ok(()) } + fn register_external_interrupt_controller( + &self, + intc: &'static dyn ExternalInterruptController, + ) -> Result<(), Error> { + self.xintc.init(intc); + Ok(()) + } + + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { + *self.mtimer.get() + } + fn register_monotonic_timer( &self, timer: &'static dyn MonotonicTimestampProviderDevice, @@ -198,107 +201,143 @@ impl Architecture for AArch64 { Ok(()) } - fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { - self.reset.init(reset); - Ok(()) - } - - fn external_interrupt_controller( - &self, - ) -> &'static dyn ExternalInterruptController { - *self.ext_intc.get() - } - - fn local_interrupt_controller( - &self, - ) -> &'static dyn LocalInterruptController { - *self.local_intc.get() - } - - fn monotonic_timer(&self) -> &'static dyn MonotonicTimestampProviderDevice { - *self.mtimer.get() - } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { - if let Some(local_intc) = self.local_intc.try_get() { + if let Some(local_intc) = self.lintc.try_get() { local_intc.send_ipi(target, msg) } else { Ok(()) } } + fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { + self.reset.init(reset); + Ok(()) + } + unsafe fn reset(&self) -> ! { if let Some(reset) = self.reset.try_get() { - reset.reset(); + reset.reset() } else { let psci = self.psci.get(); - psci.reset(); + psci.reset() } } } impl AArch64 { - /// Initializes the architecture's device tree - /// - /// # Safety - /// - /// Only makes sense to call during the early initialization, once. - pub unsafe fn init_device_tree(&self, dtb_phys: usize) { - let dt = DeviceTree::from_addr(dtb_phys.virtualize()); - self.dt.init(dt); + fn extract_initrd_from_dt( + &self, + dt: &DeviceTree, + ) -> Option<(PhysicalAddress, PhysicalAddress)> { + let chosen = dt.node_by_path("/chosen")?; + let initrd_start = devtree::find_prop(&chosen, "linux,initrd-start")?; + let initrd_end = devtree::find_prop(&chosen, "linux,initrd-end")?; + + let address_cells = dt.address_cells(); + + let initrd_start = initrd_start.cell1_array_item(0, address_cells)?; + let initrd_end = initrd_end.cell1_array_item(0, address_cells)?; + + let initrd_start = PhysicalAddress::from_raw(initrd_start); + let initrd_end = PhysicalAddress::from_raw(initrd_end); + + Some((initrd_start, initrd_end)) } - /// Returns the device tree - /// - /// # Panics - /// - /// Will panic if the device tree has not yet been initialized - pub fn device_tree(&self) -> &DeviceTree { - self.dt.get() - } + unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> { + // 16x2MiB + const HEAP_PAGES: usize = 16; - unsafe fn init_physical_memory(&self, dtb_phys: usize) -> Result<(), Error> { - let dt = self.device_tree(); + // Initialize the runtime mappings + mem::init_fixed_tables(); - if let Some(initrd) = INITRD_DATA.try_get() { - reserve_region( - "initrd", - PhysicalMemoryRegion { - base: initrd.phys_page_start, - size: initrd.phys_page_len, - }, - ); - } + // Extract the size of the device tree + let dtb_size = { + let dtb_header = EarlyMapping::::map_slice(dtb, DevTree::MIN_HEADER_SIZE)?; + DevTree::read_totalsize(dtb_header.as_ref()).unwrap() + }; reserve_region( "dtb", PhysicalMemoryRegion { - base: dtb_phys, - size: (dt.size() + 0xFFF) & !0xFFF, + base: dtb, + size: (dtb_size + 0xFFF) & !0xFFF, }, ); - let regions = FdtMemoryRegionIter::new(dt); + let dtb_slice = EarlyMapping::::map_slice(dtb, dtb_size)?; - phys::init_from_iter(regions) + let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize); + + // Setup initrd from the dt + let initrd = self.extract_initrd_from_dt(&dt); + + if let Some((start, end)) = initrd { + let aligned_start = start.page_align_down::(); + let aligned_end = end.page_align_up::(); + + let size = aligned_end - aligned_start; + reserve_region( + "initrd", + PhysicalMemoryRegion { + base: aligned_start, + size, + }, + ); + } + + // Initialize the physical memory + let regions = FdtMemoryRegionIter::new(&dt); + + phys::init_from_iter(regions)?; + + // Setup the heap + for i in 0..HEAP_PAGES { + let l2_page = phys::alloc_2m_page()?; + mem::map_heap_l2(i, l2_page); + } + + heap::init_heap(mem::HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE); + + // EarlyMapping for DTB no longer needed, it lives in physical memory and can be obtained + // through PhysicalRef + let dtb_slice: PhysicalRef<'static, [u8]> = PhysicalRef::map_slice(dtb, dtb_size); + let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize); + + self.dt.init(dt); + + // Setup initrd + if let Some((initrd_start, initrd_end)) = initrd { + let aligned_start = initrd_start.page_align_down::(); + let aligned_end = initrd_end.page_align_up::(); + let len = initrd_end - initrd_start; + + let data = unsafe { PhysicalRef::map_slice(initrd_start, len) }; + self.initrd.init(data); + + INITRD_DATA.init(Initrd { + phys_page_start: aligned_start, + phys_page_len: aligned_end - aligned_start, + data: self.initrd.get().as_ref(), + }); + } + + Ok(()) } - fn chosen_stdout_path<'a>(dt: &'a DeviceTree) -> Option<&'a str> { - let chosen = dt.node_by_path("/chosen")?; - chosen.prop("stdout-path") - } + unsafe fn init_platform(&self, is_bsp: bool) -> Result<(), Error> { + Cpu::init_local(); - fn init_platform(&self, bsp: bool) { - if bsp { - let dt = self.device_tree(); + if is_bsp { + let dt = self.dt.get(); let address_cells = dt.address_cells(); let size_cells = dt.size_cells(); - let chosen_stdout_path = Self::chosen_stdout_path(dt); + // Setup /chosen.stdout-path to get early debug printing + let chosen_stdout_path = dt.chosen_stdout_path(); let chosen_stdout = chosen_stdout_path.and_then(|path| dt.node_by_path(path)); - // Probe and initialize the /chosen.stdout-path device first if let Some(node) = chosen_stdout.clone() { let probe = DevTreeNodeInfo { address_cells, @@ -307,15 +346,12 @@ impl AArch64 { }; if let Some((device, _)) = devtree::probe_dt_node(&probe) { - unsafe { - device.init().unwrap(); - } + device.init()?; } - }; + } - debug::reset(); + debug::init(); - // Print some stuff now that the output is initialized infoln!( "Yggdrasil v{} ({})", env!("CARGO_PKG_VERSION"), @@ -323,22 +359,19 @@ impl AArch64 { ); infoln!("Initializing aarch64 platform"); - // Probe and initialize the rest of devices let nodes = dt.root().children(); if let Err(error) = devtree::enumerate_dt( address_cells, size_cells, nodes, |_, probe| { - // Ignore /chosen/stdout-path node + // Skip chosen-stdout, already initialized if let Some(ref chosen_stdout) = chosen_stdout && chosen_stdout.name() == probe.node.name() { return Ok(()); } if let Some((device, _)) = devtree::probe_dt_node(&probe) { - unsafe { - device.init()?; - } + device.init()?; } Ok(()) @@ -353,15 +386,14 @@ impl AArch64 { // Initialize IRQs for the devices device::manager_lock().devices().for_each(|dev| unsafe { if let Err(error) = dev.init_irq() { - errorln!( - "Could not init interrupts for {:?}: {:?}", + warnln!( + "Could not init IRQs for {:?}: {:?}", dev.display_name(), error ); } }); - // Print the device list infoln!("Enumerated devices:"); device::manager_lock().devices().for_each(|dev| { infoln!("* {:?}", dev.display_name()); @@ -369,7 +401,7 @@ impl AArch64 { } else { // BSP already initialized everything needed // Setup timer and local interrupt controller - let intc = self.local_intc.get(); + let intc = self.lintc.get(); unsafe { intc.init_ap().unwrap(); @@ -378,57 +410,28 @@ impl AArch64 { // TODO device-tree initialization for this CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); CNTP_TVAL_EL0.set(10000000); - self.ext_intc - .get() - .enable_irq(IrqNumber::Private(14)) - .unwrap(); + self.xintc.get().enable_irq(IrqNumber::Private(14)).unwrap(); } + + Ok(()) } } -fn setup_initrd() { - let dt = ARCHITECTURE.device_tree(); +pub static ARCHITECTURE: AArch64 = AArch64 { + dt: OneTimeInit::new(), + initrd: OneTimeInit::new(), - let Some(chosen) = dt.node_by_path("/chosen") else { - return; - }; + psci: OneTimeInit::new(), + reset: OneTimeInit::new(), - let Some(initrd_start) = devtree::find_prop(&chosen, "linux,initrd-start") else { - return; - }; - let Some(initrd_end) = devtree::find_prop(&chosen, "linux,initrd-end") else { - return; - }; + lintc: OneTimeInit::new(), + xintc: OneTimeInit::new(), - let address_cells = dt.address_cells(); + mtimer: OneTimeInit::new(), +}; - let Some(initrd_start) = initrd_start.cell1_array_item(0, address_cells) else { - infoln!("No initrd specified"); - return; - }; - let Some(initrd_end) = initrd_end.cell1_array_item(0, address_cells) else { - infoln!("No initrd specified"); - return; - }; - - let initrd_start = initrd_start as usize; - let initrd_end = initrd_end as usize; - - let start_aligned = initrd_start & !0xFFF; - let end_aligned = initrd_end & !0xFFF; - - let data = unsafe { - core::slice::from_raw_parts( - initrd_start.virtualize() as *const _, - initrd_end - initrd_start, - ) - }; - - let initrd = Initrd { - phys_page_start: start_aligned, - phys_page_len: end_aligned - start_aligned, - data, - }; - - INITRD_DATA.init(initrd); +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum IrqNumber { + Private(u32), + Shared(u32), } diff --git a/src/arch/aarch64/plat_orange_pi3/mod.rs b/src/arch/aarch64/plat_orange_pi3/mod.rs deleted file mode 100644 index 8b650c3d..00000000 --- a/src/arch/aarch64/plat_orange_pi3/mod.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Orange Pi 3 (Allwinner H6/sun50i-h6 SoC) -//! -//! # Booting using u-boot -//! -//! > fatload mmc 0:1 0x40000000 uRamdisk -//! > fatload mmc 0:1 0x4d000000 sun50i-h6-orangepi-3.dtb -//! > loady 0x44000000 -//! ... -//! > bootm 0x44000000 0x40000000 0x4d000000 -//! -use abi::error::Error; - -use crate::{ - arch::CpuMessage, - debug::{self, LogLevel}, - device::{ - interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, - platform::Platform, - timer::TimestampSource, - InitializableDevice, - }, - fs::devfs::{self, CharDeviceType}, -}; - -use self::{r_wdog::RWatchdog, serial::Serial}; - -use super::{ - gic::{Gic, IrqNumber}, - timer::ArmTimer, -}; - -pub mod r_wdog; -pub mod serial; - -/// Orange Pi 3 implementation -pub struct PlatformImpl { - uart0: Serial, - r_wdog: RWatchdog, - /// ... - pub gic: Gic, - timer: ArmTimer, -} - -impl Platform for PlatformImpl { - type IrqNumber = IrqNumber; - - const KERNEL_PHYS_BASE: usize = 0x50000000; - - unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { - if is_bsp { - self.gic.init(())?; - - self.timer.init(())?; - self.r_wdog.init(())?; - - self.timer.init_irq()?; - self.uart0.init_irq()?; - - devfs::add_char_device(&self.uart0, CharDeviceType::TtySerial)?; - } else { - todo!(); - } - - Ok(()) - } - - unsafe fn init_debug(&'static self) { - self.uart0.init(()).unwrap(); - debug::add_sink(&self.uart0, LogLevel::Debug); - } - - fn interrupt_controller( - &self, - ) -> &dyn ExternalInterruptController { - &self.gic - } - - fn timestamp_source(&self) -> &dyn TimestampSource { - &self.timer - } - - unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { - todo!(); - } - - fn name(&self) -> &'static str { - "Orange Pi 3" - } - - unsafe fn reset(&self) -> ! { - self.r_wdog.reset_board(); - } -} - -/// Orange Pi 3 platform implementation -pub static PLATFORM: PlatformImpl = unsafe { - PlatformImpl { - uart0: Serial::new(0x05000000, IrqNumber::new(32)), - r_wdog: RWatchdog::new(0x07020400), - gic: Gic::new(0x03021000, 0x03022000), - timer: ArmTimer::new(IrqNumber::new(30)), - } -}; diff --git a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs b/src/arch/aarch64/plat_orange_pi3/r_wdog.rs deleted file mode 100644 index 5deac6a9..00000000 --- a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Allwinner H6 R Watchdog driver -use abi::error::Error; -use tock_registers::{ - interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, -}; - -use crate::{ - device::InitializableDevice, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, - util::OneTimeInit, -}; - -register_bitfields! { - u32, - CTRL [ - KEY OFFSET(1) NUMBITS(12) [ - Value = 0xA57 - ], - RESTART OFFSET(0) NUMBITS(1) [] - ], - CFG [ - CONFIG OFFSET(0) NUMBITS(2) [ - System = 1, - ] - ], - MODE [ - EN OFFSET(0) NUMBITS(1) [], - ] -} - -register_structs! { - #[allow(non_snake_case)] - Regs { - (0x00 => IRQ_EN: ReadWrite), - (0x04 => IRQ_STA: ReadWrite), - (0x08 => _0), - (0x10 => CTRL: ReadWrite), - (0x14 => CFG: ReadWrite), - (0x18 => MODE: ReadWrite), - (0x1C => @END), - } -} - -/// Allwinner H6 R Watchdog -pub struct RWatchdog { - inner: OneTimeInit>>, - base: usize, -} - -impl InitializableDevice for RWatchdog { - type Options = (); - - unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { - let regs = DeviceMemoryIo::map("r_wdog", self.base)?; - - self.inner.init(IrqSafeSpinlock::new(regs)); - - Ok(()) - } -} - -impl RWatchdog { - /// Performs a reset through watchdog. - /// - /// # Safety - /// - /// Only meant to be called by platform reset code. - pub unsafe fn reset_board(&self) -> ! { - let regs = self.inner.get().lock(); - - regs.CFG.write(CFG::CONFIG::System); - regs.MODE.write(MODE::EN::SET); - regs.CTRL.write(CTRL::KEY::Value + CTRL::RESTART::SET); - - loop { - core::arch::asm!("wfe"); - } - } - - /// Constructs an instance of the device at `base`. - /// - /// # Safety - /// - /// The caller must ensure the address is valid. - pub const unsafe fn new(base: usize) -> Self { - Self { - inner: OneTimeInit::new(), - base, - } - } -} diff --git a/src/arch/aarch64/plat_orange_pi3/serial.rs b/src/arch/aarch64/plat_orange_pi3/serial.rs deleted file mode 100644 index 864189a0..00000000 --- a/src/arch/aarch64/plat_orange_pi3/serial.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Allwinner H6 UART driver -use abi::{error::Error, io::DeviceRequest}; -use tock_registers::{ - interfaces::{Readable, Writeable}, - register_bitfields, register_structs, - registers::{ReadOnly, ReadWrite}, -}; -use vfs::CharDevice; - -use crate::{ - arch::aarch64::gic::IrqNumber, - arch::PLATFORM, - debug::DebugSink, - device::{ - interrupt::InterruptSource, - platform::Platform, - serial::SerialDevice, - tty::{CharRing, TtyDevice}, - Device, InitializableDevice, - }, - mem::device::DeviceMemoryIo, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; - -register_bitfields! { - u32, - USR [ - TFE OFFSET(2) NUMBITS(1) [], - TFNF OFFSET(1) NUMBITS(1) [] - ] -} - -register_structs! { - #[allow(non_snake_case)] - Regs { - (0x00 => DLL: ReadWrite), - (0x04 => _0), - (0x7C => USR: ReadOnly), - (0x80 => @END), - } -} - -struct Inner { - regs: DeviceMemoryIo, -} - -/// Allwinner H6 UART -pub struct Serial { - inner: OneTimeInit>, - ring: CharRing<16>, - base: usize, - irq: IrqNumber, -} - -impl CharDevice for Serial { - fn read(&'static self, _blocking: bool, data: &mut [u8]) -> Result { - self.line_read(data) - } - - fn write(&self, _blocking: bool, data: &[u8]) -> Result { - self.line_write(data) - } - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - match req { - &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(id as _); - Ok(()) - } - _ => Err(Error::InvalidArgument), - } - } -} - -impl TtyDevice<16> for Serial { - fn ring(&self) -> &CharRing<16> { - &self.ring - } -} - -impl InterruptSource for Serial { - unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.interrupt_controller(); - - intc.register_handler(self.irq, self)?; - intc.enable_irq(self.irq)?; - - Ok(()) - } - - fn handle_irq(&self) -> Result { - let byte = self.inner.get().lock().regs.DLL.get(); - - self.recv_byte(byte as u8); - - Ok(true) - } -} - -impl DebugSink for Serial { - fn putc(&self, c: u8) -> Result<(), Error> { - self.send(c).ok(); - Ok(()) - } -} - -impl SerialDevice for Serial { - fn send(&self, byte: u8) -> Result<(), Error> { - let inner = self.inner.get().lock(); - if byte == b'\n' { - while inner.regs.USR.matches_all(USR::TFE::CLEAR) { - core::hint::spin_loop(); - } - inner.regs.DLL.set(b'\r' as u32); - } - while inner.regs.USR.matches_all(USR::TFE::CLEAR) { - core::hint::spin_loop(); - } - inner.regs.DLL.set(byte as u32); - Ok(()) - } - - fn receive(&self, _blocking: bool) -> Result { - todo!() - } -} - -impl InitializableDevice for Serial { - type Options = (); - - unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { - let regs = DeviceMemoryIo::::map("h6-uart", self.base)?; - self.inner.init(IrqSafeSpinlock::new(Inner { regs })); - Ok(()) - } -} - -impl Device for Serial { - fn name(&self) -> &'static str { - "Allwinner H6 UART" - } -} - -impl Serial { - /// Constructs an instance of the device at `base`. - /// - /// # Safety - /// - /// The caller must ensure the address is valid. - pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self { - Self { - inner: OneTimeInit::new(), - ring: CharRing::new(), - - base, - irq, - } - } -} diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index 424a8cf7..ee01418f 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -5,17 +5,18 @@ use abi::error::Error; use device_api::CpuBringupDevice; use fdt_rs::prelude::PropReader; +use crate::arch::Architecture; +use crate::mem::address::IntoRaw; +use crate::mem::KERNEL_VIRT_OFFSET; use crate::{ - absolute_address, - arch::ARCHITECTURE, - mem::{ - phys::{self, PageUsage}, - ConvertAddress, - }, + arch::{ArchitectureImpl, ARCHITECTURE}, + mem::phys, }; use crate::device::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; +use super::{BootStack, BOOT_STACK_SIZE}; + #[derive(Debug)] enum CpuEnableMethod { Psci, @@ -74,6 +75,12 @@ impl CpuEnableMethod { /// Number of online CPUs, initially set to 1 (BSP processor is up) pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); +// TODO can be made smaller +#[link_section = ".bss"] +static AP_TRAMPOLINE_STACK: BootStack = BootStack { + data: [0; BOOT_STACK_SIZE], +}; + /// Starts application processors using the method specified in the device tree. /// /// TODO: currently does not handle systems where APs are already started before entry. @@ -87,6 +94,9 @@ pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> { fn __aarch64_ap_entry(); } + // Safety: safe, the stack is inside the kernel + let sp = AP_TRAMPOLINE_STACK.data.as_ptr() as usize - KERNEL_VIRT_OFFSET + BOOT_STACK_SIZE; + for cpu in enumerate_cpus(dt).filter(|cpu| cpu.id != 0) { debugln!( "cpu{}: enable-method={:?}, compatible={:?}", @@ -95,19 +105,20 @@ pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> { cpu.compatible ); - const AP_STACK_PAGES: usize = 4; - let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used)?; - debugln!( - "cpu{} stack: {:#x}..{:#x}", - cpu.id, - stack_pages, - stack_pages + AP_STACK_PAGES * 0x1000 - ); + // const AP_STACK_PAGES: usize = 4; + // let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES)?; + // debugln!( + // "cpu{} stack: {:#x}..{:#x}", + // cpu.id, + // stack_pages, + // stack_pages.add(AP_STACK_PAGES * 0x1000) + // ); // Wait for the CPU to come up let old_count = CPU_COUNT.load(Ordering::Acquire); - let ip = absolute_address!(__aarch64_ap_entry).physicalize(); - let sp = stack_pages + AP_STACK_PAGES * 0x1000; + // Safety: safe, the function is inside the kernel + let ip = __aarch64_ap_entry as usize - KERNEL_VIRT_OFFSET; + // let sp = stack_pages.add(AP_STACK_PAGES * 0x1000); if let Err(error) = cpu.enable_method.start_cpu(cpu.id as usize, ip, sp) { errorln!("Couldn't start cpu{} up: {:?}", cpu.id, error); continue; diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs deleted file mode 100644 index 1690ea38..00000000 --- a/src/arch/aarch64/table.rs +++ /dev/null @@ -1,553 +0,0 @@ -//! AArch64 virtual memory management facilities -use core::{ - marker::PhantomData, - ops::{Index, IndexMut}, - sync::atomic::{AtomicU8, Ordering}, -}; - -use aarch64_cpu::registers::DAIF; -use abi::error::Error; -use bitflags::bitflags; -use tock_registers::interfaces::Readable; - -use crate::mem::{ - phys::{self, PageUsage}, - table::{ - EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, - }, - ConvertAddress, KERNEL_VIRT_OFFSET, -}; - -/// TODO -#[derive(Clone)] -#[repr(C)] -pub struct AddressSpace { - l1: *mut PageTable, - asid: u8, -} - -/// Page table representing a single level of address translation -#[derive(Clone)] -#[repr(C, align(0x1000))] -pub struct PageTable { - data: [PageEntry; 512], -} - -/// Translation level 1: Entry is 1GiB page/table -#[derive(Clone, Copy)] -pub struct L1; -/// Translation level 2: Entry is 2MiB page/table -#[derive(Clone, Copy)] -pub struct L2; -/// Translation level 3: Entry is 4KiB page -#[derive(Clone, Copy)] -pub struct L3; - -impl NonTerminalEntryLevel for L1 { - type NextLevel = L2; -} -impl NonTerminalEntryLevel for L2 { - type NextLevel = L3; -} - -bitflags! { - /// Describes how each page is mapped: access, presence, type of the mapping. - #[derive(Clone, Copy)] - pub struct PageAttributes: u64 { - /// When set, the mapping is considered valid and assumed to point to a page/table - const PRESENT = 1 << 0; - - /// For L1/L2 mappings, indicates that the mapping points to the next-level translation - /// table - const TABLE = 1 << 1; - /// (Must be set) For L3 mappings, indicates that the mapping points to a page - const PAGE = 1 << 1; - /// For L1/L2 mappings, indicates that the mapping points to a page of given level's size - const BLOCK = 0 << 1; - - /// (Must be set) For page/block mappings, indicates to the hardware that the page is - /// accessed - const ACCESS = 1 << 10; - - /// For page/block mappings, allows both user and kernel code to read/write to the page - const AP_BOTH_READWRITE = 1 << 6; - /// For page/block mappings, only allows read access for EL0/EL1 - const AP_BOTH_READONLY = 3 << 6; - - /// Indicates outer shareability domain - const SH_OUTER = 2 << 8; - /// Indicates inner shareability domain - const SH_INNER = 3 << 8; - - /// For page/block mappings, indicates regular physical memory - const PAGE_ATTR_NORMAL = 0 << 2; - /// For page/block mappings, indicates device memory - const PAGE_ATTR_DEVICE = 1 << 2; - - /// Indicates the mapping is unique to a specific ASID (important for proper TLB - /// maintenance) - const NON_GLOBAL = 1 << 11; - } -} - -impl const EntryLevel for L1 { - fn index(addr: usize) -> usize { - (addr >> 30) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x3FFFFFFF - } -} -impl const EntryLevel for L2 { - fn index(addr: usize) -> usize { - (addr >> 21) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x1FFFFF - } -} -impl const EntryLevel for L3 { - fn index(addr: usize) -> usize { - (addr >> 12) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0xFFF - } -} - -/// Represents a single entry in a translation table -#[derive(Clone, Copy)] -#[repr(transparent)] -pub struct PageEntry(u64, PhantomData); - -/// Fixed-layout kernel-space address mapping tables -pub struct FixedTables { - l1: PageTable, - device_l2: PageTable, - device_l3: PageTable, - - device_l3i: usize, -} - -impl PageEntry { - /// Creates a 4KiB normal memory page mapping - pub fn normal_page(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) - | (PageAttributes::PAGE - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | attrs) - .bits(), - PhantomData, - ) - } - - /// Creates a 4KiB device memory page mapping - pub fn device_page(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) - | (PageAttributes::PAGE - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_OUTER - | PageAttributes::PAGE_ATTR_DEVICE - | attrs) - .bits(), - PhantomData, - ) - } - - /// Returns the physical address of the page this entry refers to, returning None if it does - /// not - pub fn as_page(self) -> Option { - let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); - - if self.0 & mask == mask { - Some((self.0 & !0xFFF) as usize) - } else { - None - } - } -} - -impl PageEntry { - /// Creates a 2MiB page mapping - pub fn normal_block(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) - | (PageAttributes::BLOCK - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | attrs) - .bits(), - PhantomData, - ) - } - - /// Creates a mapping pointing to the next-level translation table - pub fn table(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), - PhantomData, - ) - } - - /// Returns the physical address of the table this entry refers to, returning None if it - /// does not - pub fn as_table(self) -> Option { - if self.0 & (PageAttributes::TABLE | PageAttributes::PRESENT).bits() - == (PageAttributes::TABLE | PageAttributes::PRESENT).bits() - { - Some((self.0 & !0xFFF) as usize) - } else { - None - } - } -} - -impl PageEntry { - /// Represents an absent/invalid mapping in the table - pub const INVALID: Self = Self(0, PhantomData); - - /// Converts a raw mapping value into this wrapper type - /// - /// # Safety - /// - /// The caller is responsible for making sure that `raw` is a valid mapping value for the - /// current translation level. - pub unsafe fn from_raw(raw: u64) -> Self { - Self(raw, PhantomData) - } - - /// Returns `true` if the entry refers to some table/block/page - pub fn is_present(&self) -> bool { - self.0 & PageAttributes::PRESENT.bits() != 0 - } -} - -impl NextPageTable for PageTable { - type NextLevel = PageTable; - - fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> { - let entry = self[index]; - - entry - .as_table() - .map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) }) - } - - fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> { - let entry = self[index]; - - if let Some(table) = entry.as_table() { - Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) }) - } else { - let table = PageTable::new_zeroed()?; - self[index] = PageEntry::::table(table.physical_address(), PageAttributes::empty()); - Ok(table) - } - } -} - -impl PageTable { - /// Constructs a page table with all entries marked as invalid - pub const fn zeroed() -> Self { - Self { - data: [PageEntry::INVALID; 512], - } - } - - /// Allocates a new page table, filling it with non-preset entries - pub fn new_zeroed() -> Result<&'static mut Self, Error> { - let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() }; - let table = unsafe { &mut *(page as *mut Self) }; - for i in 0..512 { - table[i] = PageEntry::INVALID; - } - Ok(table) - } - - /// Returns a physical address pointing to this page table - pub fn physical_address(&self) -> usize { - // &self may already by a physical address - let addr = self.data.as_ptr() as usize; - if addr < KERNEL_VIRT_OFFSET { - addr - } else { - unsafe { addr.physicalize() } - } - } -} - -impl Index for PageTable { - type Output = PageEntry; - - fn index(&self, index: usize) -> &Self::Output { - &self.data[index] - } -} - -impl IndexMut for PageTable { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.data[index] - } -} - -impl FixedTables { - /// Constructs an empty table group - pub const fn zeroed() -> Self { - Self { - l1: PageTable::zeroed(), - device_l2: PageTable::zeroed(), - device_l3: PageTable::zeroed(), - - device_l3i: 0, - } - } - - /// Maps a physical memory region as device memory and returns its allocated base address - pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result { - if count > 512 * 512 { - panic!("Unsupported device memory mapping size"); - } else if count > 512 { - // 2MiB mappings - todo!(); - } else { - // 4KiB mappings - if self.device_l3i + count > 512 { - return Err(Error::OutOfMemory); - } - - let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12); - for i in 0..count { - self.device_l3[self.device_l3i + i] = - PageEntry::device_page(phys + i * 0x1000, PageAttributes::empty()); - } - self.device_l3i += count; - - Ok(virt) - } - } -} - -impl From for PageAttributes { - fn from(value: MapAttributes) -> Self { - let mut res = Self::empty(); - if value.contains(MapAttributes::USER_WRITE) { - res |= PageAttributes::AP_BOTH_READWRITE; - } else { - res |= PageAttributes::AP_BOTH_READONLY; - } - if value.contains(MapAttributes::NON_GLOBAL) { - res |= PageAttributes::NON_GLOBAL; - } - res - } -} - -impl VirtualMemoryManager for AddressSpace { - fn allocate( - &self, - hint: Option, - len: usize, - attrs: MapAttributes, - ) -> Result { - assert_eq!(DAIF.read(DAIF::I), 1); - - if hint.is_some() { - todo!(); - } - - const TRY_ALLOC_START: usize = 0x100000000; - const TRY_ALLOC_END: usize = 0xF00000000; - - 'l0: for base in (TRY_ALLOC_START..TRY_ALLOC_END - len * 0x1000).step_by(0x1000) { - for i in 0..len { - if self.translate(base + i * 0x1000).is_some() { - continue 'l0; - } - } - - for i in 0..len { - let page = phys::alloc_page(PageUsage::Used)?; - self.map_page(base + i * 0x1000, page, attrs)?; - } - - return Ok(base); - } - - Err(Error::OutOfMemory) - } - - fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> { - assert_eq!(DAIF.read(DAIF::I), 1); - - for page in (addr..addr + len).step_by(0x1000) { - let Some(phys) = self.translate(page) else { - todo!( - "Tried to deallocate address not present in the table: {:#x}", - addr - ); - }; - - self.write_entry(page, PageEntry::INVALID, true)?; - unsafe { - phys::free_page(phys); - } - } - - Ok(()) - } - - fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> { - self.write_entry(virt, PageEntry::normal_page(phys, attrs.into()), true) - } -} - -impl AddressSpace { - /// Allocates an empty address space with all entries marked as non-present - pub fn new_empty() -> Result { - static LAST_ASID: AtomicU8 = AtomicU8::new(1); - - let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); - - let l1 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable }; - - for i in 0..512 { - unsafe { - (*l1)[i] = PageEntry::INVALID; - } - } - - Ok(Self { l1, asid }) - } - - /// Interprets a physical address as an address space structure pointer. - /// - /// # Safety - /// - /// Unsafe: accepts arbitrary addresses and ignores ASIDs. - pub unsafe fn from_phys_raw(value: usize) -> Self { - let value = value.virtualize(); - Self { - l1: value as *mut PageTable, - asid: 0, - } - } - - /// Iterates through all the translation levels for a given address, invoking the callback - /// function on each of them - pub fn walk bool>(&self, vaddr: usize, f: F) { - let l1i = L1::index(vaddr); - let l2i = L2::index(vaddr); - let l3i = L3::index(vaddr); - - let l1 = unsafe { self.as_mut() }; - let l1e = l1[l1i]; - - let Some(l2) = l1.get_mut(l1i) else { - f(1, 0); - return; - }; - if !f(1, l1e.0 as _) { - return; - } - let l2e = l2[l2i]; - - let Some(l3) = l2.get_mut(l2i) else { - f(2, 0); - return; - }; - if !f(2, l2e.0 as _) { - return; - } - - let l3e = l3[l3i]; - if !f(3, l3e.0 as _) { - return; - } - } - - unsafe fn as_mut(&self) -> &'static mut PageTable { - self.l1.as_mut().unwrap() - } - - // TODO return page size and attributes - /// Returns the physical address to which the `virt` address is mapped - pub fn translate(&self, virt: usize) -> Option { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); - - let l2 = unsafe { self.as_mut().get_mut(l1i) }?; - let l3 = l2.get_mut(l2i)?; - - l3[l3i].as_page() - } - - // Write a single 4KiB entry - fn write_entry(&self, virt: usize, entry: PageEntry, overwrite: bool) -> Result<(), Error> { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); - - let l2 = unsafe { self.as_mut().get_mut_or_alloc(l1i) }?; - let l3 = l2.get_mut_or_alloc(l2i)?; - - if l3[l3i].is_present() && !overwrite { - todo!() - } - l3[l3i] = entry; - - tlb_flush_vaae1(virt); - - Ok(()) - } - - /// Returns the physical address of the address space (to be used in a TTBRn_ELx) - pub fn physical_address(&self) -> usize { - unsafe { (self.l1 as usize).physicalize() | ((self.asid as usize) << 48) } - } -} - -/// Flush a virtual address from EL1/EL0 TLB for all ASIDs -pub fn tlb_flush_vaae1(mut page: usize) { - page >>= 12; - unsafe { - core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); - } -} - -/// Initializes mappings for the kernel and device memory tables. -/// -/// # Safety -/// -/// Only allowed to be called once during lower-half part of the initialization process. -pub unsafe fn init_fixed_tables() { - // Map first 256GiB - for i in 0..256 { - KERNEL_TABLES.l1[i] = PageEntry::::normal_block(i << 30, PageAttributes::empty()); - } - - KERNEL_TABLES.l1[256] = PageEntry::::table( - KERNEL_TABLES.device_l2.physical_address(), - PageAttributes::empty(), - ); - KERNEL_TABLES.device_l2[0] = PageEntry::::table( - KERNEL_TABLES.device_l3.physical_address(), - PageAttributes::empty(), - ); -} - -/// Offset applied to device virtual memory mappings -pub const DEVICE_VIRT_OFFSET: usize = KERNEL_VIRT_OFFSET + (256 << 30); -/// Global kernel address space translation tables -pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed(); diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index e4cc7c8f..9a7fe43e 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -11,8 +11,7 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, device_tree_driver, - proc::wait, - task::tasklet, + task::runtime, }; use super::cpu::Cpu; @@ -30,8 +29,7 @@ impl InterruptHandler for ArmTimer { CNTP_TVAL_EL0.set(TICK_INTERVAL); let now = self.monotonic_timestamp().unwrap(); - wait::tick(now); - tasklet::tick(now); + runtime::tick(now); unsafe { Cpu::local().queue().yield_cpu(); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 956f15c4..47c20ead 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -20,7 +20,6 @@ macro_rules! absolute_address { } use cfg_if::cfg_if; -// pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; use device_api::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index e26c13f6..2c4458ea 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -65,11 +65,11 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, cr3: PhysicalAddress) { + fn init_common(&mut self, entry: usize, cr3: u64) { self.push(entry); // address for ret // End of common context - self.push(cr3.into_raw()); // %cr3 + self.push(cr3 as _); // %cr3 self.push(0); // %rbp self.push(0); // %fs (TODO) @@ -99,7 +99,7 @@ impl TaskContextImpl for TaskContext { stack.push(arg); stack.init_common(__x86_64_task_enter_kernel as _, unsafe { - KERNEL_TABLES.as_physical_address() + KERNEL_TABLES.as_physical_address().into_raw() }); let sp = stack.build(); @@ -114,12 +114,7 @@ impl TaskContextImpl for TaskContext { }) } - fn user( - entry: usize, - arg: usize, - cr3: PhysicalAddress, - user_stack_sp: usize, - ) -> Result { + fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 8; let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index cde7e096..48963678 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -3,7 +3,7 @@ use yggdrasil_abi::error::Error; use crate::{ arch::x86_64::intrinsics, mem::{ - address::AsPhysicalAddress, + address::{AsPhysicalAddress, IntoRaw}, phys, pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, @@ -61,6 +61,11 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { self.read_l3_entry(address) .ok_or(Error::InvalidMemoryOperation) } + + fn as_address_with_asid(&self) -> u64 { + // TODO x86-64 PCID/ASID? + unsafe { self.l0.as_physical_address().into_raw() } + } } impl ProcessAddressSpaceImpl { @@ -128,9 +133,3 @@ impl ProcessAddressSpaceImpl { Some((page, l3[l3i].attributes().into())) } } - -impl AsPhysicalAddress for ProcessAddressSpaceImpl { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - self.l0.as_physical_address() - } -} diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index c8e1234d..c4adfa93 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -67,51 +67,19 @@ impl NonTerminalEntryLevel for L2 { } impl const EntryLevel for L0 { - const SIZE: usize = 1 << 39; - - fn index(addr: usize) -> usize { - (addr >> 39) & 0x1FF - } - - fn page_offset(_addr: usize) -> usize { - todo!() - } + const SHIFT: usize = 39; } impl const EntryLevel for L1 { - const SIZE: usize = 1 << 30; - - fn index(addr: usize) -> usize { - (addr >> 30) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x3FFFFFFF - } + const SHIFT: usize = 30; } impl const EntryLevel for L2 { - const SIZE: usize = 1 << 21; - - fn index(addr: usize) -> usize { - (addr >> 21) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x1FFFFF - } + const SHIFT: usize = 21; } impl const EntryLevel for L3 { - const SIZE: usize = 1 << 12; - - fn index(addr: usize) -> usize { - (addr >> 12) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0xFFF - } + const SHIFT: usize = 12; } impl PageEntry { diff --git a/src/device/devtree.rs b/src/device/devtree.rs index 8882fb1e..d85e7d3a 100644 --- a/src/device/devtree.rs +++ b/src/device/devtree.rs @@ -10,7 +10,10 @@ use fdt_rs::{ prelude::PropReader, }; -use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; +use crate::{ + debug::LogLevel, + mem::{address::FromRaw, phys::PhysicalMemoryRegion, PhysicalAddress}, +}; use super::register_device; @@ -162,6 +165,7 @@ impl<'a> DeviceTree<'a> { /// /// The caller must ensure the validity of the address. pub unsafe fn from_addr(virt: usize) -> Self { + FDT_INDEX_BUFFER.0.fill(0); let tree = DevTree::from_raw_pointer(virt as _).unwrap(); let index = DevTreeIndex::new(tree, &mut FDT_INDEX_BUFFER.0).unwrap(); Self { tree, index } @@ -198,6 +202,12 @@ impl<'a> DeviceTree<'a> { pub fn root(&self) -> DevTreeIndexNode { self.index.root() } + + // Commonly used functions for convenience + pub fn chosen_stdout_path(&self) -> Option<&str> { + let chosen = self.node_by_path("/chosen")?; + chosen.prop("stdout-path") + } } impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { @@ -314,10 +324,10 @@ impl Iterator for FdtMemoryRegionIter<'_> { .cell2_array_item(0, self.address_cells, self.size_cells) .unwrap(); - break Some(PhysicalMemoryRegion { - base: base as usize, - size: size as usize, - }); + let base = PhysicalAddress::from_raw(base); + let size = size as usize; + + break Some(PhysicalMemoryRegion { base, size }); } } } diff --git a/src/device/mod.rs b/src/device/mod.rs index 1dbd0a6f..032006a3 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -7,9 +7,12 @@ use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; #[cfg(target_arch = "aarch64")] pub mod devtree; +// TODO bus device support on aarch64 +#[cfg(not(target_arch = "aarch64"))] pub mod bus; + pub mod display; -// pub mod power; +pub mod power; pub mod serial; pub mod timer; pub mod tty; diff --git a/src/device/power/mod.rs b/src/device/power/mod.rs index 7cf0ce21..8b996e54 100644 --- a/src/device/power/mod.rs +++ b/src/device/power/mod.rs @@ -5,6 +5,6 @@ use cfg_if::cfg_if; cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod arm_psci; - pub mod sunxi_rwdog; + // pub mod sunxi_rwdog; } } diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index 7f672fe6..4a364aa0 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -5,6 +5,6 @@ use cfg_if::cfg_if; cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod pl011; - pub mod sunxi_uart; + // pub mod sunxi_uart; } } diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 40ed12e6..e90bbc31 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -12,14 +12,15 @@ use vfs::CharDevice; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, + block, debug::{self, DebugSink, LogLevel}, device::{ devtree::{self, DevTreeIndexPropExt}, - tty::{CharRing, TtyDevice}, + tty::{TtyContext, TtyDevice}, }, device_tree_driver, fs::devfs::{self, CharDeviceType}, - mem::device::DeviceMemoryIo, + mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, sync::IrqSafeSpinlock, }; @@ -62,15 +63,15 @@ register_structs! { } struct Pl011Inner { - regs: DeviceMemoryIo, + regs: DeviceMemoryIo<'static, Regs>, } /// PL011 device instance pub struct Pl011 { inner: OneTimeInit>, - base: usize, + base: PhysicalAddress, irq: IrqNumber, - ring: CharRing<16>, + context: TtyContext, } impl Pl011Inner { @@ -101,9 +102,9 @@ impl DebugSink for Pl011 { } } -impl TtyDevice<16> for Pl011 { - fn ring(&self) -> &CharRing<16> { - &self.ring +impl TtyDevice for Pl011 { + fn context(&self) -> &TtyContext { + &self.context } } @@ -115,7 +116,12 @@ impl CharDevice for Pl011 { fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { assert!(blocking); - self.line_read(data) + match block! { + self.line_read(data).await + } { + Ok(res) => res, + Err(err) => Err(err), + } } fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { @@ -174,7 +180,7 @@ impl Device for Pl011 { unsafe fn init(&'static self) -> Result<(), Error> { let mut inner = Pl011Inner { - regs: DeviceMemoryIo::map("pl011 UART", self.base)?, + regs: DeviceMemoryIo::map(self.base)?, }; inner.init(); @@ -207,8 +213,8 @@ device_tree_driver! { inner: OneTimeInit::new(), // TODO obtain IRQ from dt irq: IrqNumber::Shared(1), - ring: CharRing::new(), - base: base as usize + context: TtyContext::new(), + base: PhysicalAddress::from_raw(base) })) } } diff --git a/src/mem/device.rs b/src/mem/device.rs index df7f1b9f..dcba8ea0 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -67,9 +67,15 @@ impl DeviceMemoryMapping { } impl<'a, T: Sized> DeviceMemoryIo<'a, T> { - pub unsafe fn from_raw(raw: DeviceMemoryMapping) -> DeviceMemoryIo<'a, T> { - // TODO - loop {} + pub unsafe fn from_raw( + inner: Arc, + ) -> Result, Error> { + if size_of::() > inner.page_size * inner.page_count { + todo!(); + } + // TODO check align + let value = &*(inner.address as *const T); + Ok(DeviceMemoryIo { inner, value }) } pub unsafe fn map(base: PhysicalAddress) -> Result, Error> { diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 1f22832f..f98cbc7f 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -22,12 +22,14 @@ pub(super) const TRACKED_PAGE_LIMIT: usize = (BITMAP_PAGE_COUNT * 4096) * 8; pub struct PhysicalMemoryManager { bitmap: PhysicalRefMut<'static, [u64]>, last_free_bit: usize, + offset: usize, page_count: usize, } impl PhysicalMemoryManager { pub unsafe fn new( bitmap_phys_base: PhysicalAddress, + offset: usize, page_count: usize, ) -> PhysicalMemoryManager { let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE; @@ -38,6 +40,7 @@ impl PhysicalMemoryManager { Self { bitmap, page_count, + offset, last_free_bit: 0, } } @@ -67,7 +70,7 @@ impl PhysicalMemoryManager { self.last_free_bit = i + 1; self.mark_alloc(i); - return Ok(PhysicalAddress::from_raw(i * 0x1000)); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } if self.last_free_bit != 0 { @@ -93,7 +96,7 @@ impl PhysicalMemoryManager { } self.last_free_bit = i + 512; - return Ok(PhysicalAddress::from_raw(i * 0x1000)); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } if self.last_free_bit != 0 { @@ -118,7 +121,7 @@ impl PhysicalMemoryManager { } self.last_free_bit = i + count; - return Ok(PhysicalAddress::from_raw(i * 0x1000)); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } if self.last_free_bit != 0 { @@ -135,7 +138,9 @@ impl PhysicalMemoryManager { /// /// `addr` must be a page-aligned physical address previously allocated by this implementation. pub unsafe fn free_page(&mut self, page: PhysicalAddress) { - let index = IntoRaw::::into_raw(page) / 0x1000; + let page: usize = page.into_raw(); + assert!(page >= self.offset); + let index = (page - self.offset) / 0x1000; assert!(self.is_alloc(index)); self.mark_free(index); @@ -147,7 +152,9 @@ impl PhysicalMemoryManager { /// /// Will panic if the address does not point to a valid, reserved (and unallocated) page. pub fn add_available_page(&mut self, page: PhysicalAddress) { - let index = IntoRaw::::into_raw(page) / 0x1000; + let page: usize = page.into_raw(); + assert!(page >= self.offset); + let index = (page - self.offset) / 0x1000; assert!(self.is_alloc(index)); self.mark_free(index); diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index e1328d03..59476452 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -5,7 +5,10 @@ use kernel_util::util::OneTimeInit; use crate::{ arch::{Architecture, ARCHITECTURE}, - mem::phys::reserved::is_reserved, + mem::{ + address::IntoRaw, + phys::{self, reserved::is_reserved}, + }, sync::IrqSafeSpinlock, }; @@ -162,7 +165,12 @@ pub unsafe fn init_from_iter + Clone>( }, ); - let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, total_count); + if IntoRaw::::into_raw(phys_start) & 0x1FFFFFF != 0 { + loop {} + } + + let mut manager = + PhysicalMemoryManager::new(page_bitmap_phys_base, phys_start.into_raw(), total_count); let mut collected = 0; const MAX_MEMORY: usize = 16 * 1024; diff --git a/src/mem/process.rs b/src/mem/process.rs index a6727de6..f4855656 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -2,20 +2,20 @@ use abi::error::Error; use cfg_if::cfg_if; use vmalloc::VirtualMemoryAllocator; -use crate::{arch::x86_64::mem::table::L3, mem::phys, sync::IrqSafeSpinlock}; +use crate::{mem::phys, sync::IrqSafeSpinlock}; use super::{address::AsPhysicalAddress, table::MapAttributes, PhysicalAddress}; cfg_if! { if #[cfg(target_arch = "aarch64")] { - use crate::arch::aarch64::table::AddressSpace; + use crate::arch::aarch64::mem::process::ProcessAddressSpaceImpl; } else if #[cfg(target_arch = "x86_64")] { use crate::arch::x86_64::mem::process::ProcessAddressSpaceImpl; } } /// Interface for virtual memory address space management -pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress { +pub trait ProcessAddressSpaceManager: Sized { const PAGE_SIZE: usize; const LOWER_LIMIT_PFN: usize; const UPPER_LIMIT_PFN: usize; @@ -32,6 +32,8 @@ pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress { unsafe fn unmap_page(&mut self, address: usize) -> Result; fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; + + fn as_address_with_asid(&self) -> u64; } struct Inner { @@ -186,7 +188,8 @@ impl ProcessAddressSpace { attributes: MapAttributes, ) -> Result<(), Error> { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - assert!(physical.is_aligned_for::()); + // XXX + // assert!(physical.is_aligned_for::()); self.inner .lock() @@ -211,22 +214,7 @@ impl ProcessAddressSpace { ) } - pub fn debug_dump(&self) { - let lock = self.inner.lock(); - - debugln!("Address space @ {:#x}", unsafe { - lock.table.as_physical_address() - }); - for (used, range) in lock.allocator.ranges() { - let start = range.start_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE; - let end = range.end_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE; - debugln!("{:#x?}: {}", start..end, used); - } - } -} - -impl AsPhysicalAddress for ProcessAddressSpace { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - self.inner.lock().table.as_physical_address() + pub fn as_address_with_asid(&self) -> u64 { + self.inner.lock().table.as_address_with_asid() } } diff --git a/src/mem/table.rs b/src/mem/table.rs index f7dc9493..057946ad 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -4,6 +4,7 @@ use core::ops::{Deref, DerefMut}; use abi::error::Error; use bitflags::bitflags; +// TODO EXECUTABLE bitflags! { /// Describes how a page translation mapping should behave #[derive(Clone, Copy)] @@ -36,12 +37,29 @@ pub trait NextPageTable { /// Interface for a single level of address translation #[const_trait] pub trait EntryLevel: Copy { - const SIZE: usize; + const SHIFT: usize; + const SIZE: usize = 1 << Self::SHIFT; /// Returns the index into a page table for a given address - fn index(addr: usize) -> usize; + #[inline] + fn index(addr: usize) -> usize { + (addr >> Self::SHIFT) & 0x1FF + } /// Returns the offset of an address from the page start at current level - fn page_offset(addr: usize) -> usize; + #[inline] + fn page_offset(addr: usize) -> usize { + addr & (Self::SIZE - 1) + } + + #[inline] + fn align_up(addr: usize) -> usize { + (addr + Self::SIZE - 1) & !(Self::SIZE - 1) + } + + #[inline] + fn page_count(addr: usize) -> usize { + (addr + Self::SIZE - 1) / Self::SIZE + } } /// Tag trait to mark that the page table level may point to a next-level table diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 63f14c97..06afd6aa 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -98,8 +98,6 @@ pub fn load_elf_from_file(space: &ProcessAddressSpace, file: FileRef) -> Result< let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; - space.debug_dump(); - for phdr in elf.segments() { if phdr.p_type != PT_LOAD { continue; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 653c930e..e40a2b96 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -174,7 +174,7 @@ fn setup_binary>( let context = TaskContext::user( entry, virt_args_base, - unsafe { space.as_physical_address() }, + unsafe { space.as_address_with_asid() }, user_sp, )?; diff --git a/src/task/context.rs b/src/task/context.rs index dadfd1c1..a2f1bb07 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -56,12 +56,7 @@ pub trait TaskContextImpl: Sized { /// Constructs a user thread context. The caller is responsible for allocating the userspace /// stack and setting up a valid address space for the context. - fn user( - entry: usize, - arg: usize, - cr3: PhysicalAddress, - user_stack_sp: usize, - ) -> Result; + fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result; /// Performs an entry into a context. /// diff --git a/tools/gentables/Cargo.toml b/tools/gentables/Cargo.toml index ae7f7609..2e0a20e3 100644 --- a/tools/gentables/Cargo.toml +++ b/tools/gentables/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -memtables = { path = "../../lib/memtables" } +memtables = { path = "../../lib/memtables", features = ["all"] } bytemuck = "1.14.0" elf = "0.7.2" diff --git a/tools/gentables/src/aarch64.rs b/tools/gentables/src/aarch64.rs new file mode 100644 index 00000000..624b61e8 --- /dev/null +++ b/tools/gentables/src/aarch64.rs @@ -0,0 +1,193 @@ +use core::fmt; +use std::{ + io::{Read, Seek}, + mem::offset_of, +}; + +use bitflags::bitflags; +use elf::{ + abi::{PF_W, PF_X, PT_LOAD}, + endian::AnyEndian, + ElfStream, +}; +use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; + +use crate::{GenData, GenError}; + +bitflags! { + #[derive(Clone, Copy)] + struct PageFlags: u64 { + const PRESENT = 1 << 0; + const ACCESS = 1 << 10; + const SH_INNER = 3 << 8; + const PAGE_ATTR_NORMAL = 0 << 2; + const AP_BOTH_READONLY = 3 << 6; + const TABLE = 1 << 1; + const PAGE = 1 << 1; + const UXN = 1 << 54; + const PXN = 1 << 53; + } +} + +impl PageFlags { + pub fn kernel_table() -> Self { + Self::PRESENT | Self::ACCESS | Self::SH_INNER | Self::PAGE_ATTR_NORMAL | Self::TABLE + } + + pub fn kernel_page() -> Self { + Self::PRESENT | Self::ACCESS | Self::SH_INNER | Self::PAGE_ATTR_NORMAL | Self::PAGE + } +} + +pub struct AArch64Builder { + elf: ElfStream, + data: GenData, + tables: FixedTables, + + l1i: usize, + l2i_start: usize, + l2i_end: usize, +} + +impl PageFlags { + fn from_elf(flags: u32) -> Self { + let mut out = Self::UXN | Self::PXN; + if flags & PF_X != 0 { + out.remove(Self::PXN); + } + if flags & PF_W == 0 { + out |= Self::AP_BOTH_READONLY; + } + out + } +} + +impl fmt::Display for PageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let x = if self.contains(Self::PXN) { '-' } else { 'x' }; + let w = if self.contains(Self::AP_BOTH_READONLY) { + '-' + } else { + 'w' + }; + write!(f, "r{}{}", w, x) + } +} + +const L1_SHIFT: u64 = 30; +const L2_SHIFT: u64 = 21; +const L3_SHIFT: u64 = 12; +const L2_ENTRY_SIZE: u64 = 1 << L2_SHIFT; +const L3_ENTRY_SIZE: u64 = 1 << L3_SHIFT; + +// TODO proper granularity +impl AArch64Builder { + pub fn new(elf: ElfStream, data: GenData) -> Result { + let l1i = (data.kernel_start >> L1_SHIFT) as usize & 0x1FF; + let l2i_start = (data.kernel_start >> L2_SHIFT) as usize & 0x1FF; + let l2i_end = ((data.kernel_end + L2_ENTRY_SIZE - 1) >> L2_SHIFT) as usize & 0x1FF; + + if l2i_end - l2i_start > KERNEL_L3_COUNT { + todo!() + } + + Ok(Self { + elf, + data, + + tables: FixedTables::zeroed(), + + l1i, + l2i_start, + l2i_end, + }) + } + + // TODO the build function is almost identical to x86-64 one, but with slight changes, so might + // wanna unify this later + pub fn build(mut self) -> Result<(FixedTables, u64), GenError> { + assert_eq!(offset_of!(FixedTables, l1), 0); + + let l2_physical_address = + self.data.table_physical_address + offset_of!(FixedTables, l2) as u64; + + // L1 -> L2 + self.tables.l1.data[self.l1i] = l2_physical_address | PageFlags::kernel_table().bits(); + + // L2 -> L3s + for l2i in self.l2i_start..self.l2i_end { + let l3_table_index = l2i - self.l2i_start; + let l3_physical_address = self.data.table_physical_address + + (offset_of!(FixedTables, l3s) + 0x1000 * l3_table_index) as u64; + + self.tables.l2.data[l2i] = l3_physical_address | PageFlags::kernel_table().bits(); + } + + for (i, segment) in self.elf.segments().into_iter().enumerate() { + if segment.p_type != PT_LOAD + || segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset + { + continue; + } + + let aligned_virt_start = segment.p_vaddr & !(L3_ENTRY_SIZE - 1); + let aligned_virt_end = + (segment.p_vaddr + segment.p_memsz + L3_ENTRY_SIZE - 1) & !(L3_ENTRY_SIZE - 1); + let aligned_phys_start = segment.p_paddr & !(L3_ENTRY_SIZE - 1); + let count = (aligned_virt_end - aligned_virt_start) / 0x1000; + + let flags = PageFlags::from_elf(segment.p_flags); + + println!( + "{}: {:#x?} -> {:#x} {}", + i, + aligned_virt_start..aligned_virt_end, + aligned_phys_start, + flags + ); + Self::map_segment( + self.l2i_start, + &mut self.tables, + aligned_virt_start, + aligned_phys_start, + count as usize, + flags, + )?; + } + + Ok((self.tables, self.data.table_offset)) + } + + fn map_segment( + start_l2i: usize, + tables: &mut FixedTables, + vaddr_start: u64, + paddr_start: u64, + count: usize, + flags: PageFlags, + ) -> Result<(), GenError> { + for index in 0..count { + let vaddr = vaddr_start + index as u64 * L3_ENTRY_SIZE; + let paddr = paddr_start + index as u64 * L3_ENTRY_SIZE; + + let entry = paddr | (PageFlags::kernel_page() | flags).bits(); + + let l2i = (vaddr >> L2_SHIFT) as usize & 0x1FF - start_l2i; + let l3i = (vaddr >> L3_SHIFT) as usize & 0x1FF; + + let l3 = &mut tables.l3s[l2i]; + + if l3.data[l3i] != 0 { + if l3.data[l3i] != entry { + todo!(); + } else { + continue; + } + } + + l3.data[l3i] = entry; + } + + Ok(()) + } +} diff --git a/tools/gentables/src/main.rs b/tools/gentables/src/main.rs index 0ec86b02..7edfcd9c 100644 --- a/tools/gentables/src/main.rs +++ b/tools/gentables/src/main.rs @@ -10,15 +10,16 @@ use std::{ use clap::Parser; use elf::{ - abi::{EM_X86_64, PT_LOAD}, + abi::{EM_AARCH64, EM_X86_64, PT_LOAD}, endian::AnyEndian, ElfStream, }; -use memtables::FixedTables; +use memtables::any::AnyTables; use thiserror::Error; -use crate::x86_64::X8664Builder; +use crate::{aarch64::AArch64Builder, x86_64::X8664Builder}; +mod aarch64; mod x86_64; #[derive(Error, Debug)] @@ -120,7 +121,11 @@ fn find_tables(elf: &mut ElfStream) -> Result<(u64 Err(GenError::MissingSection(".data.tables")) } -fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> { +fn into_any, U>((l, r): (T, U)) -> (AnyTables, U) { + (l.into(), r) +} + +fn build_tables(file: F) -> Result<(AnyTables, u64), GenError> { let mut elf = ElfStream::::open_stream(file)?; let kernel_virt_offset = kernel_virt_offset(&mut elf)?; @@ -144,7 +149,20 @@ fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> table_physical_address, }, )? - .build(), + .build() + .map(into_any), + EM_AARCH64 => AArch64Builder::new( + elf, + GenData { + kernel_virt_offset, + kernel_start, + kernel_end, + table_offset, + table_physical_address, + }, + )? + .build() + .map(into_any), _ => todo!(), } } @@ -152,11 +170,10 @@ fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> fn write_tables( mut file: F, offset: u64, - tables: FixedTables, + tables: AnyTables, ) -> Result<(), GenError> { - let bytes = bytemuck::bytes_of(&tables); file.seek(SeekFrom::Start(offset))?; - file.write_all(bytes)?; + file.write_all(tables.as_bytes())?; Ok(()) } diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs index d41551a4..725636e3 100644 --- a/tools/gentables/src/x86_64.rs +++ b/tools/gentables/src/x86_64.rs @@ -11,7 +11,7 @@ use elf::{ endian::AnyEndian, ElfStream, }; -use memtables::{FixedTables, KERNEL_L3_COUNT}; +use memtables::x86_64::{FixedTables, KERNEL_L3_COUNT}; use crate::{GenData, GenError}; From ff04db65dc7ce121c4a464fef060eae6d4f03d1f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 16 Nov 2023 11:11:10 +0200 Subject: [PATCH 084/211] refactor: clean up kernel warnings --- src/arch/aarch64/boot/mod.rs | 8 +-- src/arch/aarch64/context.rs | 10 ++-- src/arch/aarch64/gic/mod.rs | 2 +- src/arch/aarch64/mem/mod.rs | 41 ++++++++-------- src/arch/aarch64/mem/process.rs | 2 +- src/arch/aarch64/mod.rs | 2 +- src/arch/aarch64/smp.rs | 7 +-- src/arch/mod.rs | 17 ++++++- src/device/display/fb_console.rs | 44 +++++++++-------- src/device/display/font.rs | 3 +- src/main.rs | 1 + src/mem/address.rs | 37 +++++++------- src/mem/device.rs | 24 +++++++++ src/mem/mod.rs | 12 +++++ src/mem/phys/manager.rs | 4 +- src/mem/phys/mod.rs | 7 +-- src/mem/pointer.rs | 83 +++++++++++--------------------- src/mem/process.rs | 21 +++++++- src/proc/exec.rs | 47 ++++++------------ src/syscall/mod.rs | 6 +-- src/task/context.rs | 2 +- src/task/runtime/mod.rs | 1 + src/task/runtime/waker.rs | 2 +- src/task/sched.rs | 1 + src/util/mod.rs | 5 +- 25 files changed, 198 insertions(+), 191 deletions(-) diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 338f0910..63f6fab3 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -21,7 +21,7 @@ use crate::{ unsafe fn pre_init_mmu() { if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { // TODO early panic - loop {} + todo!(); } MAIR_EL1.write( @@ -115,7 +115,7 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { kernel_main() } -unsafe extern "C" fn __aarch64_el1_ap_lower_entry(sp: PhysicalAddress) -> ! { +unsafe extern "C" fn __aarch64_el1_ap_lower_entry() -> ! { const AP_STACK_PAGES: usize = 8; AArch64::set_interrupt_mask(true); @@ -147,7 +147,9 @@ extern "C" fn __aarch64_ap_upper_entry() -> ! { exception::init_exceptions(); unsafe { - ARCHITECTURE.init_platform(false); + ARCHITECTURE + .init_platform(false) + .expect("Could not initialize the AP"); } kernel_secondary_main() diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 508e4dbb..9e1cd482 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -3,10 +3,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; -use crate::{ - mem::{address::IntoRaw, phys, PhysicalAddress}, - task::context::TaskContextImpl, -}; +use crate::{mem::phys, task::context::TaskContextImpl}; struct StackBuilder { base: usize, @@ -86,8 +83,7 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 8; - let stack_base = - unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() }; + let stack_base = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -110,7 +106,7 @@ impl TaskContextImpl for TaskContext { fn user(entry: usize, arg: usize, ttbr0: u64, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 16; - let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() }; + let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 4b528b5d..511e8792 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -20,7 +20,7 @@ use crate::{ device_tree_driver, mem::{ address::FromRaw, - device::{DeviceMemoryIo, DeviceMemoryMapping, RawDeviceMemoryMapping}, + device::{DeviceMemoryIo, RawDeviceMemoryMapping}, PhysicalAddress, }, sync::IrqSafeSpinlock, diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index 8ef804f2..38af3515 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -37,8 +37,7 @@ cfg_if! { // Precomputed mappings const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); -const KERNEL_END_L2_INDEX: usize = - L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE) + KERNEL_L3_COUNT; +const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT; // Must not be zero, should be at 4MiB const_assert_eq!(KERNEL_START_L2_INDEX, 0); @@ -52,12 +51,10 @@ const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1; const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; // 1GiB max const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; -// 16GiB max -pub(super) const RAM_MAPPING_L1_COUNT: usize = 16; -const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3; -const RAM_MAPPING_END_L1I: usize = RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT; - const DEVICE_MAPPING_L3_COUNT: usize = 4; +// 16GiB max +const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3; +pub(super) const RAM_MAPPING_L1_COUNT: usize = 16; // 2MiB for early mappings const EARLY_MAPPING_OFFSET: usize = @@ -85,19 +82,19 @@ pub struct EarlyMapping<'a, T: ?Sized> { } impl<'a, T: Sized> EarlyMapping<'a, T> { - pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { - let layout = Layout::new::(); - let aligned = physical.page_align_down::(); - let offset = physical.page_offset::(); - let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + // pub(super) unsafe fn map(physical: PhysicalAddress) -> Result, Error> { + // let layout = Layout::new::(); + // let aligned = physical.page_align_down::(); + // let offset = physical.page_offset::(); + // let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; - let virt = map_early_pages(aligned, page_count)?; - let value = &mut *((virt + offset) as *mut T); + // let virt = map_early_pages(aligned, page_count)?; + // let value = &mut *((virt + offset) as *mut T); - Ok(EarlyMapping { value, page_count }) - } + // Ok(EarlyMapping { value, page_count }) + // } - pub unsafe fn map_slice( + pub(super) unsafe fn map_slice( physical: PhysicalAddress, len: usize, ) -> Result, Error> { @@ -186,7 +183,7 @@ unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result= EARLY_MAPPING_OFFSET + L2::SIZE { + if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { panic!("Tried to unmap invalid early mapping: {:#x}", address); } @@ -200,7 +197,7 @@ unsafe fn unmap_early_page(address: usize) { pub(super) unsafe fn map_ram_l1(index: usize) { if index >= RAM_MAPPING_L1_COUNT { - loop {} + todo!() } assert_eq!(KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I], 0); @@ -210,7 +207,7 @@ pub(super) unsafe fn map_ram_l1(index: usize) { pub(super) unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) { if index >= 512 { - loop {} + todo!() } assert!(!HEAP_MAPPING_L2[index].is_present()); // TODO UXN, PXN @@ -321,8 +318,8 @@ pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { let l3i = L3::index(page); assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; - // TODO flush the TLB entry - loop {} + + todo!(); // intrinsics::flush_tlb_entry(page); } } diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index 0f9370c2..a5ad6ab0 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -3,7 +3,7 @@ use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; use crate::mem::{ - address::{AsPhysicalAddress, IntoRaw}, + address::AsPhysicalAddress, phys, pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 71c3d84b..1163da49 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -125,7 +125,7 @@ impl Architecture for AArch64 { ) -> Result<(), Error> { let end_l1i = L1::index(memory_end.page_align_up::().into_raw()); if end_l1i > mem::RAM_MAPPING_L1_COUNT { - loop {} + todo!() } // Map 1GiB chunks diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index ee01418f..9196062f 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -5,13 +5,8 @@ use abi::error::Error; use device_api::CpuBringupDevice; use fdt_rs::prelude::PropReader; -use crate::arch::Architecture; -use crate::mem::address::IntoRaw; +use crate::arch::ARCHITECTURE; use crate::mem::KERNEL_VIRT_OFFSET; -use crate::{ - arch::{ArchitectureImpl, ARCHITECTURE}, - mem::phys, -}; use crate::device::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 47c20ead..651ee988 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -69,12 +69,22 @@ pub trait Architecture { /// Only safe to call once during system init. unsafe fn start_application_processors(&self) {} - /// Allocates a virtual mapping for the specified physical memory region + /// Allocates a virtual mapping for the specified physical memory region. + /// + /// # Safety + /// + /// The caller must ensure the validity of the provided region. unsafe fn map_device_memory( &self, base: PhysicalAddress, size: usize, ) -> Result; + + /// Removes the provided mapping from the kernel's translation tables. + /// + /// # Safety + /// + /// The caller must ensure the mapping is and will no longer be used. unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping); fn map_physical_memory + Clone>( @@ -177,6 +187,9 @@ pub trait Architecture { /// The caller must ensure it is actually safe to reset, i.e. no critical processes will be /// aborted and no data will be lost. unsafe fn reset(&self) -> ! { - loop {} + Self::set_interrupt_mask(true); + loop { + Self::wait_for_interrupt(); + } } } diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 6890b168..81933eb7 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -20,6 +20,15 @@ struct Inner { height: u32, } +struct DrawGlyph { + sx: u32, + sy: u32, + c: u8, + fg: u32, + bg: u32, + bytes_per_line: usize, +} + /// Framebuffer console device wrapper pub struct FramebufferConsole { inner: IrqSafeSpinlock, @@ -66,12 +75,14 @@ impl DisplayConsole for FramebufferConsole { inner.draw_glyph( font, - (col_idx as u32) * cw, - row_idx * ch, - glyph, - fg, - bg, - bytes_per_line, + DrawGlyph { + sx: (col_idx as u32) * cw, + sy: row_idx * ch, + c: glyph, + fg, + bg, + bytes_per_line, + }, ); } } @@ -96,7 +107,7 @@ impl FramebufferConsole { framebuffer: &'static LinearFramebuffer, font: Option>, ) -> Result { - let font = font.unwrap_or(PcScreenFont::default()); + let font = font.unwrap_or_default(); let char_width = font.width(); let char_height = font.height(); let dim = framebuffer.dimensions(); @@ -120,19 +131,10 @@ impl FramebufferConsole { impl Inner { #[optimize(speed)] - fn draw_glyph( - &mut self, - font: PcScreenFont<'static>, - sx: u32, - sy: u32, - c: u8, - fg: u32, - bg: u32, - bytes_per_line: usize, - ) { + fn draw_glyph(&mut self, font: PcScreenFont<'static>, g: DrawGlyph) { let mut fb = unsafe { self.framebuffer.lock() }; - let mut c = c as u32; + let mut c = g.c as u32; if c >= font.len() { c = b'?' as u32; } @@ -146,13 +148,13 @@ impl Inner { let mut x = 0; while x < font.width() { - let v = if glyph[0] & mask != 0 { fg } else { bg }; - fb[sy + y][(sx + x) as usize] = v; + let v = if glyph[0] & mask != 0 { g.fg } else { g.bg }; + fb[g.sy + y][(g.sx + x) as usize] = v; mask >>= 1; x += 1; } - glyph = &glyph[bytes_per_line..]; + glyph = &glyph[g.bytes_per_line..]; y += 1; } } diff --git a/src/device/display/font.rs b/src/device/display/font.rs index cbebdcfc..1ac9412c 100644 --- a/src/device/display/font.rs +++ b/src/device/display/font.rs @@ -5,7 +5,7 @@ use bytemuck::{Pod, Zeroable}; use kernel_util::AlignedTo; // static CONSOLE_FONT: &[u8] = include_bytes!("font.psfu"); -static CONSOLE_FONT: &'static AlignedTo = &AlignedTo { +static CONSOLE_FONT: &AlignedTo = &AlignedTo { align: [], bytes: *include_bytes!("font.psfu"), }; @@ -53,6 +53,7 @@ impl<'a> PcScreenFont<'a> { self.header.height } + #[allow(clippy::len_without_is_empty)] #[inline] pub const fn len(&self) -> u32 { self.header.num_glyph diff --git a/src/main.rs b/src/main.rs index f82b4acb..a783262e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ #![allow( clippy::new_without_default, clippy::fn_to_numeric_cast, + clippy::match_ref_pats, async_fn_in_trait )] // #![warn(missing_docs)] diff --git a/src/mem/address.rs b/src/mem/address.rs index 8bf10baa..c03b834a 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -7,7 +7,7 @@ use core::{ use crate::arch::{Architecture, ArchitectureImpl}; -use super::{pointer::PhysicalPointer, table::EntryLevel, KERNEL_VIRT_OFFSET}; +use super::{table::EntryLevel, KERNEL_VIRT_OFFSET}; #[repr(transparent)] pub struct KernelImageObject { @@ -29,12 +29,22 @@ pub trait IntoRaw { } pub trait AsPhysicalAddress { + /// Returns the value's physical address. + /// + /// # Safety + /// + /// The caller must ensure the value has been constructed and obtained through proper means. unsafe fn as_physical_address(&self) -> PhysicalAddress; } // KernelImageObject wrapper for objects inside the kernel impl KernelImageObject { + /// Wraps a value in the [KernelImageObject], allowing its physical address calculation. + /// + /// # Safety + /// + /// The caller must ensure the T is a `static (mut)` binding inside the kernel. pub const unsafe fn new(inner: T) -> Self { Self { inner } } @@ -98,6 +108,13 @@ impl PhysicalAddress { self.0 as usize % align_of::() == 0 } + /// Converts a previously virtualized physical address back into its physical form. + /// + /// # Safety + /// + /// The caller must ensure the function only receives addresses obtained through + /// [PhysicalAddress::virtualize_raw] or + /// [super::pointer::PhysicalRef]/[super::pointer::PhysicalRefMut] facilities. pub unsafe fn from_virtualized(address: usize) -> Self { ArchitectureImpl::physicalize(address).unwrap() } @@ -105,24 +122,6 @@ impl PhysicalAddress { pub fn virtualize_raw(self) -> usize { ArchitectureImpl::virtualize(self).unwrap() } - - pub unsafe fn virtualize(self) -> PhysicalPointer { - if !self.is_aligned_for::() { - todo!(); - } - - let base = self.virtualize_raw(); - PhysicalPointer::from_raw(base as *mut T) - } - - pub unsafe fn virtualize_slice(self, len: usize) -> PhysicalPointer<[T]> { - if !self.is_aligned_for::() { - todo!(); - } - - let base = self.virtualize_raw(); - PhysicalPointer::from_raw_parts(base as *mut T, len) - } } impl Add for PhysicalAddress { diff --git a/src/mem/device.rs b/src/mem/device.rs index dcba8ea0..a0e2f46b 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -31,6 +31,11 @@ pub struct DeviceMemoryIo<'a, T: ?Sized> { } impl RawDeviceMemoryMapping { + /// Maps a region of physical memory as device memory of given size. + /// + /// # Safety + /// + /// The caller must ensure proper access synchronization, as well as the address' origin. #[inline] pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { ARCHITECTURE.map_device_memory(base, size) @@ -52,6 +57,13 @@ impl Drop for RawDeviceMemoryMapping { } impl DeviceMemoryMapping { + /// Maps a region of physical memory as device memory of given size. + /// + /// See [RawDeviceMemoryMapping::map]. + /// + /// # Safety + /// + /// The caller must ensure proper access synchronization, as well as the address' origin. pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { let inner = RawDeviceMemoryMapping::map(base, size)?; let address = inner.address; @@ -67,6 +79,12 @@ impl DeviceMemoryMapping { } impl<'a, T: Sized> DeviceMemoryIo<'a, T> { + /// Interprets a raw device memory mapping as pointing to a value of `T`. + /// + /// # Safety + /// + /// The caller must ensure the mapping actually contains the value of `T`, as well as proper + /// access synchronization. pub unsafe fn from_raw( inner: Arc, ) -> Result, Error> { @@ -78,6 +96,12 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { Ok(DeviceMemoryIo { inner, value }) } + /// Maps a physical address as device memory of type `T`. + /// + /// # Safety + /// + /// The caller must ensure the address actually points to a value of type `T`, as well as + /// proper access synchronization. pub unsafe fn map(base: PhysicalAddress) -> Result, Error> { let inner = RawDeviceMemoryMapping::map(base, size_of::())?; let value = &*(inner.address as *const T); diff --git a/src/mem/mod.rs b/src/mem/mod.rs index fbceef93..28950dc2 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -24,6 +24,12 @@ use self::{device::DeviceMemoryMapping, process::ProcessAddressSpace}; pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; +/// Reads a value from an arbitrary physical address. +/// +/// # Safety +/// +/// The caller must ensure the correct origin of the address, its alignment and that the access is +/// properly synchronized. pub unsafe fn read_memory(address: PhysicalAddress) -> T { let io = DeviceMemoryMapping::map(address, size_of::()).unwrap(); let address = io.address(); @@ -35,6 +41,12 @@ pub unsafe fn read_memory(address: PhysicalAddress) -> T { } } +/// Writes a value to an arbitrary physical address. +/// +/// # Safety +/// +/// The caller must ensure the correct origin of the address, its alignment and that the access is +/// properly synchronized. pub unsafe fn write_memory(address: PhysicalAddress, value: T) { let io = DeviceMemoryMapping::map(address, size_of::()).unwrap(); let address = io.address(); diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index f98cbc7f..17a943bd 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -1,6 +1,4 @@ //! Physical memory manager implementation -use core::mem::size_of; - use abi::error::Error; use crate::mem::{ @@ -11,7 +9,7 @@ use crate::mem::{ pub type BitmapWord = u64; -pub(super) const BITMAP_WORD_SIZE: usize = size_of::() * 8; +pub(super) const BITMAP_WORD_SIZE: usize = BitmapWord::BITS as usize; pub(super) const BITMAP_PAGE_COUNT: usize = 256; const HUGE_PAGE_WORD_COUNT: usize = 512 / BITMAP_WORD_SIZE; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 59476452..0c0a19b6 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -5,10 +5,7 @@ use kernel_util::util::OneTimeInit; use crate::{ arch::{Architecture, ARCHITECTURE}, - mem::{ - address::IntoRaw, - phys::{self, reserved::is_reserved}, - }, + mem::{address::IntoRaw, phys::reserved::is_reserved}, sync::IrqSafeSpinlock, }; @@ -166,7 +163,7 @@ pub unsafe fn init_from_iter + Clone>( ); if IntoRaw::::into_raw(phys_start) & 0x1FFFFFF != 0 { - loop {} + todo!(); } let mut manager = diff --git a/src/mem/pointer.rs b/src/mem/pointer.rs index b79e302e..c328d2db 100644 --- a/src/mem/pointer.rs +++ b/src/mem/pointer.rs @@ -1,17 +1,10 @@ use core::{ fmt, - mem::align_of, ops::{Deref, DerefMut}, }; use super::{address::AsPhysicalAddress, PhysicalAddress}; -#[derive(Clone, Copy, PartialEq, PartialOrd, Debug, Hash)] -#[repr(transparent)] -pub struct PhysicalPointer { - pointer: *mut T, -} - #[repr(transparent)] pub struct PhysicalRef<'a, T: ?Sized> { value: &'a T, @@ -22,63 +15,28 @@ pub struct PhysicalRefMut<'a, T: ?Sized> { value: &'a mut T, } -// PhysicalPointer wrapper for direct access to any memory location - -impl PhysicalPointer { - pub fn into_address(self) -> usize { - self.pointer.addr() - } -} - -impl PhysicalPointer { - #[inline(always)] - pub fn is_aligned(&self) -> bool { - self.pointer.addr() % align_of::() == 0 - } - - #[inline(always)] - pub const unsafe fn from_raw(pointer: *mut T) -> PhysicalPointer { - PhysicalPointer { pointer } - } - - #[inline(always)] - pub unsafe fn from_raw_parts(base: *mut T, len: usize) -> PhysicalPointer<[T]> { - PhysicalPointer { - pointer: core::ptr::slice_from_raw_parts_mut(base, len), - } - } - - pub unsafe fn write_unaligned(self, value: T) { - self.pointer.write_unaligned(value) - } - - pub unsafe fn write_volatile(self, value: T) { - self.pointer.write_volatile(value) - } - - pub unsafe fn read_unaligned(self) -> T { - self.pointer.read_unaligned() - } - - pub unsafe fn read_volatile(self) -> T { - self.pointer.read_volatile() - } -} - -impl AsPhysicalAddress for PhysicalPointer { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - todo!() - } -} - // PhysicalRefMut wrapper for safe mutable access to physical addresses impl<'a, T: Sized> PhysicalRefMut<'a, T> { + /// Maps a physical address into the kernel space as &mut T, allowing mmutable access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains T. The caller must also take care of access synchronization and make sure no + /// aliasing occurs. pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRefMut<'a, T> { let value = virtualize_raw(physical); PhysicalRefMut { value } } + /// Maps a physical address into the kernel space as &mut [T], allowing mmutable access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains [T; len]. The caller must also take care of access synchronization and make + /// sure no aliasing occurs. pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRefMut<'a, [T]> { let value = virtualize_slice_raw(physical, len); PhysicalRefMut { value } @@ -121,11 +79,24 @@ impl fmt::Pointer for PhysicalRefMut<'_, T> { // PhysicalRef: same as PhysicalRefMut, except immutable impl<'a, T: Sized> PhysicalRef<'a, T> { + /// Maps a physical address into the kernel space as &T, allowing immutable access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains T. pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRef<'a, T> { let value = virtualize_raw(physical); PhysicalRef { value } } + /// Maps a physical address into the kernel space as &[T] of given len, allowing immutable + /// access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains [T; len]. pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRef<'a, [T]> { let value = virtualize_slice_raw(physical, len); PhysicalRef { value } diff --git a/src/mem/process.rs b/src/mem/process.rs index f4855656..cac37a1c 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -4,7 +4,7 @@ use vmalloc::VirtualMemoryAllocator; use crate::{mem::phys, sync::IrqSafeSpinlock}; -use super::{address::AsPhysicalAddress, table::MapAttributes, PhysicalAddress}; +use super::{table::MapAttributes, PhysicalAddress}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -22,6 +22,11 @@ pub trait ProcessAddressSpaceManager: Sized { fn new() -> Result; + /// Places a single PAGE_SIZE mapping into the address space. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address being mapped. unsafe fn map_page( &mut self, address: usize, @@ -29,6 +34,12 @@ pub trait ProcessAddressSpaceManager: Sized { flags: MapAttributes, ) -> Result<(), Error>; + /// Removes a single PAGE_SIZE mapping from the address space. + /// + /// # Safety + /// + /// The caller must ensure the process to which this address space belongs does not and + /// will not access this page. unsafe fn unmap_page(&mut self, address: usize) -> Result; fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; @@ -201,6 +212,14 @@ impl ProcessAddressSpace { self.inner.lock().table.translate(address).map(|e| e.0) } + /// Removes a single PAGE_SIZE mapping from the address space. + /// + /// See [ProcessAddressSpaceManager::unmap]. + /// + /// # Safety + /// + /// The caller must ensure the process to which this address space belongs does not and + /// will not access this page. pub unsafe fn unmap(&self, address: usize, size: usize) -> Result<(), Error> { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); diff --git a/src/proc/exec.rs b/src/proc/exec.rs index e40a2b96..793b2813 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -7,8 +7,8 @@ use vfs::FileRef; use crate::{ mem::{ - address::AsPhysicalAddress, phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, - table::MapAttributes, ForeignPointer, + phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, table::MapAttributes, + ForeignPointer, }, proc, task::{context::TaskContextImpl, process::Process, TaskContext}, @@ -39,14 +39,6 @@ impl<'a> EnvWriter<'a> { } } -fn write_2d_str_array( - space: &ProcessAddressSpace, - virt: usize, - array: &[&str], -) -> Result<(), Error> { - todo!() -} - // TODO I hate this function, it's ugly fn setup_program_env( space: &ProcessAddressSpace, @@ -66,12 +58,6 @@ fn setup_program_env( let envs_data_size: usize = envs.iter().map(|x| x.len()).sum(); let ptrs_size = str_array_size(args.len()) + str_array_size(envs.len()); let total_size = args_data_size + envs_data_size + ptrs_size + HEADER_SIZE; - // 1 + arg ptr:len count - // let args_ptr_size = (1 + args.len() * 2) * size_of::(); - // let envs_ptr_size = (1 + envs.len() * 2) * size_of::(); - - // Total size: offset arrays + data + args/env split position - // let total_size = args_size + args_ptr_size + envs_size + envs_ptr_size + size_of::(); if total_size > 0x1000 { todo!(); @@ -87,7 +73,7 @@ fn setup_program_env( MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; let mut slice = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; - let mut writer = EnvWriter::new(&mut *slice); + let mut writer = EnvWriter::new(&mut slice); let args_array_offset = HEADER_SIZE; let envs_array_offset = args_array_offset + str_array_size(args.len()); @@ -95,35 +81,35 @@ fn setup_program_env( let envs_data_offset = args_data_offset + args_data_size; // Header - writer.write_usize(virt + args_array_offset); - writer.write_usize(virt + envs_array_offset); + writer.write_usize(virt + args_array_offset)?; + writer.write_usize(virt + envs_array_offset)?; // Args array - writer.write_usize(args.len()); + writer.write_usize(args.len())?; let mut offset = args_data_offset; for arg in args.iter() { - writer.write_usize(arg.len()); - writer.write_usize(virt + offset); + writer.write_usize(arg.len())?; + writer.write_usize(virt + offset)?; offset += arg.len(); } // Envs array - writer.write_usize(envs.len()); + writer.write_usize(envs.len())?; let mut offset = envs_data_offset; for env in envs.iter() { - writer.write_usize(env.len()); - writer.write_usize(virt + offset); + writer.write_usize(env.len())?; + writer.write_usize(virt + offset)?; offset += env.len(); } // String data for arg in args.iter() { - writer.write_bytes(arg.as_bytes()); + writer.write_bytes(arg.as_bytes())?; } for env in envs.iter() { - writer.write_bytes(env.as_bytes()); + writer.write_bytes(env.as_bytes())?; } Ok(()) @@ -171,12 +157,7 @@ fn setup_binary>( } } - let context = TaskContext::user( - entry, - virt_args_base, - unsafe { space.as_address_with_asid() }, - user_sp, - )?; + let context = TaskContext::user(entry, virt_args_base, space.as_address_with_asid(), user_sp)?; Ok(Process::new_with_context(name, Some(space), context)) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index d1968b99..053aab3c 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -82,14 +82,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - let res = space.allocate( + space.allocate( None, len, |_| phys::alloc_page(), MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - ); - - res + ) } SyscallFunction::UnmapMemory => { let addr = args[0] as usize; diff --git a/src/task/context.rs b/src/task/context.rs index a2f1bb07..81ebaafa 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -4,7 +4,7 @@ use abi::{arch::SavedFrame, error::Error, process::ExitCode}; use alloc::boxed::Box; use cfg_if::cfg_if; -use crate::{mem::PhysicalAddress, task::process::Process}; +use crate::task::process::Process; cfg_if! { if #[cfg(target_arch = "aarch64")] { diff --git a/src/task/runtime/mod.rs b/src/task/runtime/mod.rs index 93fccd9e..4b6e9533 100644 --- a/src/task/runtime/mod.rs +++ b/src/task/runtime/mod.rs @@ -29,6 +29,7 @@ where type Output = T; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + #[allow(clippy::needless_borrow)] (unsafe { &mut self.get_unchecked_mut().f })(cx) } } diff --git a/src/task/runtime/waker.rs b/src/task/runtime/waker.rs index b6368b19..2ca9b9f4 100644 --- a/src/task/runtime/waker.rs +++ b/src/task/runtime/waker.rs @@ -18,7 +18,7 @@ impl QueueWaker { pub fn register(&self, waker: &Waker) { let mut queue = self.queue.lock(); - if queue.iter().find(|other| other.will_wake(waker)).is_some() { + if queue.iter().any(|other| other.will_wake(waker)) { return; } diff --git a/src/task/sched.rs b/src/task/sched.rs index 9aaf760c..a3f97548 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -81,6 +81,7 @@ impl CpuQueueInner { /// None if the queue is empty or no valid task was found, in which case the scheduler should /// go idle. pub fn next_ready_task(&mut self) -> Option> { + #[allow(clippy::never_loop)] while !self.queue.is_empty() { let task = self.queue.pop_front().unwrap(); diff --git a/src/util/mod.rs b/src/util/mod.rs index fa3e2598..e7a12c05 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -10,9 +10,8 @@ pub trait ResultIterator { impl>> ResultIterator for I { fn collect_error(self) -> Option { for item in self { - match item { - Err(e) => return Some(e), - _ => (), + if let Err(e) = item { + return Some(e); } } None From aae4874f3571d59895917548a7bb19a82c7e9a61 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 16 Nov 2023 11:54:16 +0200 Subject: [PATCH 085/211] aarch64: fix release build ld error --- src/arch/aarch64/vectors.S | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/arch/aarch64/vectors.S b/src/arch/aarch64/vectors.S index 586d76dc..32b865e6 100644 --- a/src/arch/aarch64/vectors.S +++ b/src/arch/aarch64/vectors.S @@ -82,7 +82,8 @@ __aa\bits\()_el\el\ht\()_\kind: add sp, sp, #PT_REGS_SIZE .endm -.section .text +.section .text.vectors +.global __aarch64_el1_vectors .p2align 12 __aarch64_el1_vectors: EXC_VECTOR 1, t, 64, sync @@ -105,7 +106,7 @@ __aarch64_el1_vectors: EXC_VECTOR 0, t, 32, fiq EXC_VECTOR 0, t, 32, serror - +.section .text .p2align 7 EXC_HANDLER 1, t, 64, sync EXC_HANDLER 1, t, 64, irq From de63a0456b43d8eb5ed065c2b8b7e38b05a144fe Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 16 Nov 2023 16:12:32 +0200 Subject: [PATCH 086/211] proc: reimplement argument passing --- src/main.rs | 3 +- src/proc/exec.rs | 146 +++++++++++++++++++-------------------------- src/syscall/mod.rs | 1 - 3 files changed, 64 insertions(+), 86 deletions(-) diff --git a/src/main.rs b/src/main.rs index a783262e..bd3f75e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,8 @@ rustc_private, allocator_api, trait_alias, - strict_provenance + strict_provenance, + slice_ptr_get )] #![allow( clippy::new_without_default, diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 793b2813..94d7afc8 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -1,7 +1,11 @@ //! Binary execution functions -use core::mem::size_of; +use core::alloc::Layout; -use abi::error::Error; +use abi::{ + error::Error, + pass::{Place, Placer}, + process::ProgramArgumentInner, +}; use alloc::{string::String, sync::Arc}; use vfs::FileRef; @@ -14,105 +18,79 @@ use crate::{ task::{context::TaskContextImpl, process::Process, TaskContext}, }; -pub struct EnvWriter<'a> { - data: &'a mut [u8], +pub struct BufferPlacer<'a> { + buffer: &'a mut [u8], + virtual_offset: usize, offset: usize, } -impl<'a> EnvWriter<'a> { - pub fn new(data: &'a mut [u8]) -> Self { - Self { data, offset: 0 } - } - - pub fn write_usize(&mut self, value: usize) -> Result<(), Error> { - self.write_bytes(&value.to_ne_bytes()) - } - - pub fn write_bytes(&mut self, value: &[u8]) -> Result<(), Error> { - if value.len() + self.offset > self.data.len() { - return Err(Error::InvalidArgument); +impl<'a> BufferPlacer<'a> { + pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self { + Self { + buffer, + virtual_offset, + offset: 0, + } + } + + unsafe fn alloc_layout(&mut self, layout: Layout) -> Result<(*mut u8, usize), Error> { + // TODO checks + let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1); + self.offset = aligned + layout.size(); + Ok(( + self.buffer.as_mut_ptr().add(aligned), + self.virtual_offset + aligned, + )) + } +} + +impl<'a> Placer for BufferPlacer<'a> { + fn place_ref(&mut self, r: &T) -> Result<*const T::Output, Error> { + let layout = Layout::new::(); + unsafe { + let (ptr, addr) = self.alloc_layout(layout)?; + let ptr = ptr as *mut T::Output; + ptr.write(r.place(self)?); + Ok(&*(addr as *const T::Output)) + } + } + + fn place_slice(&mut self, r: &[T]) -> Result<*const [T::Output], Error> { + let layout = Layout::array::(r.len()).unwrap(); + unsafe { + let (ptr, addr) = self.alloc_layout(layout)?; + let ptr_slice = core::ptr::slice_from_raw_parts_mut(ptr as *mut T::Output, r.len()); + for (i, elem) in r.iter().enumerate() { + ptr_slice.get_unchecked_mut(i).write(elem.place(self)?); + } + Ok(core::slice::from_raw_parts( + addr as *const T::Output, + r.len(), + )) } - debugln!("write_bytes {:#x} {:x?}", self.offset, value); - self.data[self.offset..self.offset + value.len()].copy_from_slice(value); - self.offset += value.len(); - Ok(()) } } -// TODO I hate this function, it's ugly fn setup_program_env( space: &ProcessAddressSpace, virt: usize, args: &[&str], - envs: &[&str], -) -> Result<(), Error> { - const fn str_array_size(n: usize) -> usize { - // length of the array itself (1 usize) + ptr:len pairs (2x usize * n) - (1 + n * 2) * size_of::() - } - - const HEADER_SIZE: usize = 2 * size_of::(); - - // arg data len - let args_data_size: usize = args.iter().map(|x| x.len()).sum(); - let envs_data_size: usize = envs.iter().map(|x| x.len()).sum(); - let ptrs_size = str_array_size(args.len()) + str_array_size(envs.len()); - let total_size = args_data_size + envs_data_size + ptrs_size + HEADER_SIZE; - - if total_size > 0x1000 { - todo!(); - } - - // debugln!("arg data size = {}", args_size); - // debugln!("env data size = {}", envs_size); - + env: &[&str], +) -> Result { + // TODO growing buffer let phys_page = phys::alloc_page()?; space.map_single( virt, phys_page, MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; - let mut slice = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; - let mut writer = EnvWriter::new(&mut slice); + let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; + let mut placer = BufferPlacer::new(virt, &mut buffer); - let args_array_offset = HEADER_SIZE; - let envs_array_offset = args_array_offset + str_array_size(args.len()); - let args_data_offset = envs_array_offset + str_array_size(envs.len()); - let envs_data_offset = args_data_offset + args_data_size; + let in_kernel = ProgramArgumentInner { args, env }; + let in_user = in_kernel.place_ref(&mut placer)?; - // Header - writer.write_usize(virt + args_array_offset)?; - writer.write_usize(virt + envs_array_offset)?; - - // Args array - writer.write_usize(args.len())?; - - let mut offset = args_data_offset; - for arg in args.iter() { - writer.write_usize(arg.len())?; - writer.write_usize(virt + offset)?; - offset += arg.len(); - } - - // Envs array - writer.write_usize(envs.len())?; - - let mut offset = envs_data_offset; - for env in envs.iter() { - writer.write_usize(env.len())?; - writer.write_usize(virt + offset)?; - offset += env.len(); - } - - // String data - for arg in args.iter() { - writer.write_bytes(arg.as_bytes())?; - } - for env in envs.iter() { - writer.write_bytes(env.as_bytes())?; - } - - Ok(()) + Ok(in_user as *const _ as usize) } fn setup_binary>( @@ -135,7 +113,7 @@ fn setup_binary>( MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; - setup_program_env(&space, virt_args_base, args, envs)?; + let arg = setup_program_env(&space, virt_args_base, args, envs)?; debugln!( "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", @@ -157,7 +135,7 @@ fn setup_binary>( } } - let context = TaskContext::user(entry, virt_args_base, space.as_address_with_asid(), user_sp)?; + let context = TaskContext::user(entry, arg, space.as_address_with_asid(), user_sp)?; Ok(Process::new_with_context(name, Some(space), context)) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 053aab3c..1817665b 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -277,7 +277,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::Spawn => { let options = arg_user_ref::(args[0] as usize)?; - debugln!("Spawn {:#?}", options); let proc = Process::current(); run_with_io(|mut io| { From 4e1560f38b738f8b2231afe12e62d5bc7d7606b1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 16 Nov 2023 16:45:15 +0200 Subject: [PATCH 087/211] proc: use NonNull in Place/Placer --- src/proc/exec.rs | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 94d7afc8..f81dca62 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -1,5 +1,5 @@ //! Binary execution functions -use core::alloc::Layout; +use core::{alloc::Layout, ptr::NonNull}; use abi::{ error::Error, @@ -33,40 +33,43 @@ impl<'a> BufferPlacer<'a> { } } - unsafe fn alloc_layout(&mut self, layout: Layout) -> Result<(*mut u8, usize), Error> { + unsafe fn alloc_layout( + &mut self, + layout: Layout, + ) -> Result<(NonNull, NonNull), Error> { // TODO checks let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1); self.offset = aligned + layout.size(); Ok(( - self.buffer.as_mut_ptr().add(aligned), - self.virtual_offset + aligned, + NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T), + NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T), )) } } -impl<'a> Placer for BufferPlacer<'a> { - fn place_ref(&mut self, r: &T) -> Result<*const T::Output, Error> { +unsafe impl<'a> Placer for BufferPlacer<'a> { + fn place_ref(&mut self, r: &T) -> Result, Error> { let layout = Layout::new::(); unsafe { - let (ptr, addr) = self.alloc_layout(layout)?; - let ptr = ptr as *mut T::Output; - ptr.write(r.place(self)?); - Ok(&*(addr as *const T::Output)) + let (kernel, user) = self.alloc_layout::(layout)?; + kernel.as_ptr().write(r.place(self)?); + Ok(user) } } - fn place_slice(&mut self, r: &[T]) -> Result<*const [T::Output], Error> { + fn place_slice(&mut self, r: &[T]) -> Result, Error> { let layout = Layout::array::(r.len()).unwrap(); unsafe { - let (ptr, addr) = self.alloc_layout(layout)?; - let ptr_slice = core::ptr::slice_from_raw_parts_mut(ptr as *mut T::Output, r.len()); + let (kernel, user) = self.alloc_layout::(layout)?; + let kernel = NonNull::slice_from_raw_parts(kernel, r.len()); + let user = NonNull::slice_from_raw_parts(user, r.len()); for (i, elem) in r.iter().enumerate() { - ptr_slice.get_unchecked_mut(i).write(elem.place(self)?); + kernel + .get_unchecked_mut(i) + .as_ptr() + .write(elem.place(self)?); } - Ok(core::slice::from_raw_parts( - addr as *const T::Output, - r.len(), - )) + Ok(user) } } } From da62b5b53353ab3cc66e68e9d49d705ee8ef4d47 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 21 Nov 2023 14:16:18 +0200 Subject: [PATCH 088/211] proc,dev: split processes into threads, small fb fixes --- lib/memfs/src/dir.rs | 34 +- lib/memfs/src/file.rs | 7 +- lib/memfs/src/lib.rs | 130 +++-- lib/memfs/src/tar.rs | 6 +- lib/vfs/src/char.rs | 2 +- lib/vfs/src/ioctx.rs | 4 +- lib/vfs/src/lib.rs | 2 +- lib/vfs/src/node.rs | 26 +- src/arch/aarch64/exception.rs | 42 +- src/arch/x86_64/apic/mod.rs | 10 +- src/arch/x86_64/exception.rs | 12 +- src/arch/x86_64/syscall.rs | 6 +- src/debug.rs | 39 +- src/device/display/console.rs | 60 ++- src/device/display/fb_console.rs | 32 +- src/device/display/linear_fb.rs | 1 + src/device/serial/pl011.rs | 44 +- src/device/tty.rs | 26 +- src/fs/devfs.rs | 5 +- src/init.rs | 12 +- src/panic.rs | 118 ++--- src/proc/exec.rs | 9 +- src/syscall/arg.rs | 14 +- src/syscall/mod.rs | 315 ++++++------ src/task/context.rs | 7 +- src/task/mod.rs | 87 ++-- src/task/process.rs | 812 +++++++++++++++---------------- src/task/runtime/executor.rs | 8 +- src/task/runtime/task_queue.rs | 10 +- src/task/sched.rs | 207 ++++---- src/task/thread.rs | 430 ++++++++++++++++ 31 files changed, 1576 insertions(+), 941 deletions(-) create mode 100644 src/task/thread.rs diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs index c9280b88..024662f2 100644 --- a/lib/memfs/src/dir.rs +++ b/lib/memfs/src/dir.rs @@ -2,7 +2,7 @@ use core::{cell::RefCell, marker::PhantomData}; use alloc::boxed::Box; -use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; +use vfs::{CreateInfo, Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; use yggdrasil_abi::{ error::Error, io::{FileAttr, FileMode, FileType, OpenOptions}, @@ -11,16 +11,27 @@ use yggdrasil_abi::{ use crate::{block::BlockAllocator, bvec::BVec, file::FileNode}; pub(crate) struct DirectoryNode { + uid: u32, + gid: u32, + mode: FileMode, _pd: PhantomData, } impl VnodeImpl for DirectoryNode { - fn create(&self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { - let child = Vnode::new(name, kind); - match kind { - VnodeKind::Directory => child.set_data(Box::new(Self { _pd: PhantomData })), + fn create(&self, at: &VnodeRef, info: &CreateInfo) -> Result { + let child = Vnode::new(info.name, info.kind); + match info.kind { + VnodeKind::Directory => child.set_data(Box::new(Self { + uid: info.uid, + gid: info.gid, + mode: info.mode, + _pd: PhantomData, + })), VnodeKind::Regular => child.set_data(Box::new(FileNode { data: RefCell::new(BVec::::new()), + uid: info.uid, + gid: info.gid, + mode: info.mode, })), _ => todo!(), } @@ -28,7 +39,7 @@ impl VnodeImpl for DirectoryNode { Ok(child) } - fn open(&self, _node: &VnodeRef, _opts: OpenOptions, _mode: FileMode) -> Result { + fn open(&self, _node: &VnodeRef, _opts: OpenOptions) -> Result { Ok(DIR_POSITION_FROM_CACHE) } @@ -43,14 +54,19 @@ impl VnodeImpl for DirectoryNode { fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { size: 0, - mode: FileMode::default_dir(), + mode: self.mode, ty: FileType::Directory, }) } } impl DirectoryNode { - pub fn new() -> Self { - Self { _pd: PhantomData } + pub fn new(uid: u32, gid: u32, mode: FileMode) -> Self { + Self { + uid, + gid, + mode, + _pd: PhantomData, + } } } diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs index bc02276b..c8f7c436 100644 --- a/lib/memfs/src/file.rs +++ b/lib/memfs/src/file.rs @@ -10,10 +10,13 @@ use crate::{block::BlockAllocator, bvec::BVec}; pub(crate) struct FileNode { pub(crate) data: RefCell>, + pub(crate) uid: u32, + pub(crate) gid: u32, + pub(crate) mode: FileMode, } impl VnodeImpl for FileNode { - fn open(&self, _node: &VnodeRef, opts: OpenOptions, _mode: FileMode) -> Result { + fn open(&self, _node: &VnodeRef, opts: OpenOptions) -> Result { if opts.contains(OpenOptions::APPEND) { Ok(self.data.borrow().size() as u64) } else { @@ -40,7 +43,7 @@ impl VnodeImpl for FileNode { fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { size: self.data.borrow().size() as u64, - mode: FileMode::default_file(), + mode: self.mode, ty: FileType::File, }) } diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index 94d2b5d9..22344364 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -17,8 +17,8 @@ use core::{ use alloc::{boxed::Box, rc::Rc}; use block::BlockAllocator; -use vfs::{BlockDevice, Filesystem, Vnode, VnodeKind, VnodeRef}; -use yggdrasil_abi::{error::Error, path}; +use vfs::{BlockDevice, CreateInfo, Filesystem, Vnode, VnodeKind, VnodeRef}; +use yggdrasil_abi::{error::Error, io::FileMode, path}; use crate::{bvec::BVec, dir::DirectoryNode, file::FileNode, tar::TarIterator}; @@ -58,6 +58,9 @@ mod dir; mod file; mod tar; +const DEFAULT_FILE_MODE: FileMode = FileMode::new(0o644); +const DEFAULT_DIR_MODE: FileMode = FileMode::new(0o755); + /// In-memory read/write filesystem pub struct MemoryFilesystem { root: RefCell>, @@ -101,6 +104,10 @@ impl MemoryFilesystem { } let node = self.create_node_initial(element, kind); + // TODO require .tar's to have all the directories present to extract their metadata? + if kind == VnodeKind::Directory { + node.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); + } at.add_child(node.clone()); node @@ -122,11 +129,13 @@ impl MemoryFilesystem { let node = Vnode::new(name, kind); node.set_fs(self.clone()); - match kind { - VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new())), - VnodeKind::Regular => {} - _ => todo!(), - } + // match kind { + // VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new( + // info.uid, info.gid, info.mode, + // ))), + // VnodeKind::Regular => {} + // _ => todo!(), + // } node } @@ -134,7 +143,7 @@ impl MemoryFilesystem { fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { let root = Vnode::new("", VnodeKind::Directory); root.set_fs(self.clone()); - root.set_data(Box::new(DirectoryNode::::new())); + root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); // 1. Create paths in tar for item in TarIterator::new(tar_data) { @@ -155,14 +164,23 @@ impl MemoryFilesystem { let Ok((hdr, data)) = item else { panic!("Unreachable"); }; - if hdr.node_kind() == VnodeKind::Regular { - let data = data.unwrap(); - let path = hdr.name.as_str()?.trim_matches('/'); - let node = self.make_path(&root, path, VnodeKind::Directory, false)?; + let path = hdr.name.as_str()?.trim_matches('/'); + let node = self.make_path(&root, path, VnodeKind::Directory, false)?; + assert_eq!(node.kind(), hdr.node_kind()); + + if hdr.node_kind() == VnodeKind::Regular { + let uid = usize::from(&hdr.uid).try_into().unwrap(); + let gid = usize::from(&hdr.gid).try_into().unwrap(); + let mode = convert_mode(usize::from(&hdr.mode))?; + + let data = data.unwrap(); let bvec = BVec::::try_from(data)?; assert_eq!(bvec.size(), data.len()); node.set_data(Box::new(FileNode { + uid, + gid, + mode, data: RefCell::new(bvec), })); } @@ -190,13 +208,17 @@ impl MemoryFilesystem { _pd: PhantomData, }); let root = Vnode::new("", VnodeKind::Directory); - root.set_data(Box::new(DirectoryNode::::new())); + root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); root.set_fs(fs.clone()); fs.root.replace(Some(root)); fs } } +fn convert_mode(mode: usize) -> Result { + Ok(FileMode::new(mode as u32 & 0o777)) +} + #[cfg(test)] mod tests { use core::sync::atomic::Ordering; @@ -225,7 +247,7 @@ mod tests { assert_eq!(node.kind(), VnodeKind::Regular); assert_eq!(node.size().unwrap(), expected_data.len() as u64); - let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let file = node.open(OpenOptions::READ).unwrap(); let mut buf = [0; 512]; assert_eq!( @@ -257,7 +279,7 @@ mod tests { // Write to the file { let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); - let file = node.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); + let file = node.open(OpenOptions::WRITE).unwrap(); assert_eq!(file.borrow_mut().write(b"Hello").unwrap(), 5); } @@ -267,7 +289,7 @@ mod tests { // Read back { let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); - let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let file = node.open(OpenOptions::READ).unwrap(); let mut buf = [0; 512]; assert_eq!(file.borrow_mut().read(&mut buf).unwrap(), old_data.len()); @@ -277,50 +299,50 @@ mod tests { } } - #[test] - fn test_memfs_create_and_write() { - test_allocator_with_counter!(A_COUNTER, A); + // #[test] + // fn test_memfs_create_and_write() { + // test_allocator_with_counter!(A_COUNTER, A); - let fs = MemoryFilesystem::::empty(); - let root = fs.root().unwrap(); + // let fs = MemoryFilesystem::::empty(); + // let root = fs.root().unwrap(); - let ioctx = IoContext::new(root.clone()); + // let ioctx = IoContext::new(root.clone()); - // Create, write, seek and read file - { - // TODO CREATE option handling - root.create("test1.txt", VnodeKind::Regular).unwrap(); + // // Create, write, seek and read file + // { + // // TODO CREATE option handling + // root.create("test1.txt", VnodeKind::Regular).unwrap(); - let file = ioctx - .open( - None, - "/test1.txt", - OpenOptions::WRITE | OpenOptions::READ, - FileMode::empty(), - ) - .unwrap(); + // let file = ioctx + // .open( + // None, + // "/test1.txt", + // OpenOptions::WRITE | OpenOptions::READ, + // FileMode::empty(), + // ) + // .unwrap(); - let write_data = [1, 2, 3, 4]; - let mut read_data = [0; 512]; + // let write_data = [1, 2, 3, 4]; + // let mut read_data = [0; 512]; - let mut file = file.borrow_mut(); - assert_eq!(file.write(&write_data).unwrap(), write_data.len()); - assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); - assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); - assert_eq!(&read_data[..write_data.len()], &write_data[..]); - } + // let mut file = file.borrow_mut(); + // assert_eq!(file.write(&write_data).unwrap(), write_data.len()); + // assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); + // assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); + // assert_eq!(&read_data[..write_data.len()], &write_data[..]); + // } - // Create a directory - { - // TODO read directory - root.create("dir1", VnodeKind::Directory).unwrap(); + // // Create a directory + // { + // // TODO read directory + // root.create("dir1", VnodeKind::Directory).unwrap(); - let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); - let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); - assert!(Rc::ptr_eq( - &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), - &node - )); - } - } + // let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); + // let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); + // assert!(Rc::ptr_eq( + // &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), + // &node + // )); + // } + // } } diff --git a/lib/memfs/src/tar.rs b/lib/memfs/src/tar.rs index a6e7afe0..6b7ae64a 100644 --- a/lib/memfs/src/tar.rs +++ b/lib/memfs/src/tar.rs @@ -20,9 +20,9 @@ pub(crate) struct TarIterator<'a> { #[repr(packed)] pub(crate) struct TarEntry { pub name: TarString<100>, - _mode: OctalField<8>, - _uid: OctalField<8>, - _gid: OctalField<8>, + pub mode: OctalField<8>, + pub uid: OctalField<8>, + pub gid: OctalField<8>, pub size: OctalField<12>, _mtime: OctalField<12>, _checksum: OctalField<8>, diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index bb44a524..2ca3853f 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -23,7 +23,7 @@ impl CharDeviceWrapper { } impl VnodeImpl for CharDeviceWrapper { - fn open(&self, _node: &VnodeRef, _opts: OpenOptions, _mode: FileMode) -> Result { + fn open(&self, _node: &VnodeRef, _opts: OpenOptions) -> Result { Ok(0) } diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index db774a91..06ad5ac4 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -112,7 +112,7 @@ impl IoContext { at: Option, path: &str, opts: OpenOptions, - mode: FileMode, + _mode: FileMode, ) -> Result { let node = match self.find(at.clone(), path, true, true) { Err(Error::DoesNotExist) => { @@ -122,7 +122,7 @@ impl IoContext { o => o, }?; - node.open(opts, mode) + node.open(opts) } pub fn root(&self) -> &VnodeRef { diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index cb486bdc..75855bd4 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -22,7 +22,7 @@ pub use self::char::{CharDevice, CharDeviceWrapper}; pub use file::{File, FileFlags, FileRef}; pub use fs::Filesystem; pub use ioctx::IoContext; -pub use node::{Vnode, VnodeDump, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; +pub use node::{CreateInfo, Vnode, VnodeDump, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; pub const DIR_POSITION_FROM_CACHE: u64 = u64::MAX; diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index d32735c4..78bc6ac5 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -50,9 +50,17 @@ pub struct Vnode { target: RefCell>, } +pub struct CreateInfo<'a> { + pub name: &'a str, + pub kind: VnodeKind, + pub mode: FileMode, + pub uid: u32, + pub gid: u32, +} + #[allow(unused_variables)] pub trait VnodeImpl { - fn create(&self, at: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + fn create(&self, at: &VnodeRef, info: &CreateInfo) -> Result { Err(Error::NotImplemented) } @@ -60,7 +68,7 @@ pub trait VnodeImpl { Err(Error::NotImplemented) } - fn open(&self, node: &VnodeRef, opts: OpenOptions, mode: FileMode) -> Result { + fn open(&self, node: &VnodeRef, opts: OpenOptions) -> Result { Err(Error::NotImplemented) } @@ -265,7 +273,7 @@ impl Vnode { } // Node operations - pub fn open(self: &VnodeRef, flags: OpenOptions, mode: FileMode) -> Result { + pub fn open(self: &VnodeRef, flags: OpenOptions) -> Result { let mut open_flags = FileFlags::empty(); if flags.contains(OpenOptions::READ) { @@ -280,7 +288,7 @@ impl Vnode { } if let Some(data) = self.data() { - let pos = data.open(self, flags, mode)?; + let pos = data.open(self, flags)?; Ok(File::normal(self.clone(), pos, open_flags)) } else { todo!() @@ -293,7 +301,7 @@ impl Vnode { } if let Some(data) = self.data() { - let pos = data.open(self, OpenOptions::READ, FileMode::empty())?; + let pos = data.open(self, OpenOptions::READ)?; Ok(File::directory(self.clone(), pos)) } else { // TODO: some options here? @@ -309,22 +317,22 @@ impl Vnode { } } - pub fn create(self: &VnodeRef, name: &str, kind: VnodeKind) -> Result { + pub fn create(self: &VnodeRef, info: &CreateInfo) -> Result { if self.kind != VnodeKind::Directory { todo!(); } - if name.contains('/') { + if info.name.contains('/') { return Err(Error::InvalidArgument); } - match self.lookup_or_load(name) { + match self.lookup_or_load(info.name) { Err(Error::DoesNotExist) => {} Ok(_) => return Err(Error::AlreadyExists), e => return e, }; if let Some(data) = self.data() { - let vnode = data.create(self, name, kind)?; + let vnode = data.create(self, info)?; if let Some(fs) = self.fs() { vnode.set_fs(fs); } diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index e82417c5..03c2dd36 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -19,7 +19,7 @@ use crate::{ arch::{aarch64::cpu::Cpu, Architecture, ArchitectureImpl}, debug::LogLevel, syscall::raw_syscall_handler, - task::{context::TaskFrame, process::Process}, + task::{context::TaskFrame, thread::Thread}, }; use super::ARCHITECTURE; @@ -128,13 +128,14 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { log_print_raw!(LogLevel::Fatal, "Register dump:\n"); log_print_raw!(LogLevel::Fatal, "{:?}\n", frame); - if let Some(cpu) = cpu { - let current = cpu.queue().current_process(); + // XXX + // if let Some(cpu) = cpu { + // let current = cpu.queue().current_process(); - if let Some(current) = current { - log_print_raw!(LogLevel::Fatal, "In process {}\n", current.id()); - } - } + // if let Some(current) = current { + // log_print_raw!(LogLevel::Fatal, "In process {}\n", current.id()); + // } + // } match ec { // Data abort from lower level @@ -218,8 +219,8 @@ extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) { // ); unsafe { - let process = Process::current(); - process.handle_signal(frame); + let thread = Thread::current(); + thread.handle_pending_signals(frame); } } @@ -253,8 +254,8 @@ extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) { // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); unsafe { - let process = Process::current(); - process.handle_signal(frame); + let thread = Thread::current(); + thread.handle_pending_signals(frame); } } @@ -321,23 +322,28 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { } // BRK in AArch64 0b111100 => { - let proc = Process::current(); - warnln!("Process #{} hit a breakpoint", proc.id()); - proc.raise_signal(Signal::Aborted); + let thread = Thread::current(); + warnln!( + "Thread {} {:?} hit a breakpoint", + thread.id(), + thread.name() + ); + thread.raise_signal(Signal::Aborted); } _ => { let iss = esr_el1 & 0x1FFFFFF; if ec == 0b100100 { // Data abort from lower level - let proc = Process::current(); + let thread = Thread::current(); warnln!( - "Data abort in #{} at {:#x} with address {:#x}", - proc.id(), + "Data abort in {} {:?} at {:#x} with address {:#x}", + thread.id(), + thread.name(), ELR_EL1.get(), FAR_EL1.get() ); - proc.raise_signal(Signal::MemoryAccessViolation); + thread.raise_signal(Signal::MemoryAccessViolation); return; } diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index a363b572..29a03f15 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -4,7 +4,7 @@ use core::arch::global_asm; use crate::{ arch::{x86_64::cpu::Cpu, Architecture}, - task::process::Process, + task::thread::Thread, }; use super::{ @@ -57,8 +57,8 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { .handle_specific_irq(vector); cpu.local_apic().clear_interrupt(); - if let Some(process) = Process::get_current() { - process.handle_signal(frame); + if let Some(thread) = Thread::get_current() { + thread.handle_pending_signals(frame); } } @@ -69,8 +69,8 @@ unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { cpu.local_apic().clear_interrupt(); cpu.queue().yield_cpu(); - if let Some(process) = Process::get_current() { - process.handle_signal(frame); + if let Some(thread) = Thread::get_current() { + thread.handle_pending_signals(frame); } } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 0db3f6ae..465de90e 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -5,7 +5,7 @@ use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ arch::x86_64::apic, - task::{context::TaskFrame, process::Process, Cpu}, + task::{context::TaskFrame, process::Process, thread::Thread, Cpu}, }; use super::ARCHITECTURE; @@ -265,9 +265,9 @@ impl Entry { static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { - let process = Process::current(); + let thread = Thread::current(); - warnln!("{:?} in #{} {:?}", kind, process.id(), process.name()); + warnln!("{:?} in {} {:?}", kind, thread.id(), thread.name()); warnln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); warnln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); @@ -280,10 +280,10 @@ fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { warnln!("CR2 = {:#x}", cr2); - process.raise_signal(Signal::MemoryAccessViolation); + thread.raise_signal(Signal::MemoryAccessViolation); } ExceptionKind::InvalidOpcode => { - process.raise_signal(Signal::Aborted); + thread.raise_signal(Signal::Aborted); } _ => todo!("No handler for exception: {:?}", kind), } @@ -316,7 +316,7 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { user_exception_inner(kind, frame); unsafe { - Process::current().handle_signal(frame); + Thread::current().handle_pending_signals(frame); } } else { if kind == ExceptionKind::NonMaskableInterrupt { diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 71aebdc9..6f46d7ab 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -8,7 +8,7 @@ use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ arch::x86_64::registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, syscall::raw_syscall_handler, - task::{context::TaskFrame, process::Process}, + task::{context::TaskFrame, process::Process, thread::Thread}, }; /// Set of registers saved when taking a syscall instruction @@ -125,10 +125,10 @@ fn syscall_inner(frame: &mut SyscallFrame) { extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { let frame = unsafe { &mut *frame }; - let process = Process::current(); + let thread = Thread::current(); syscall_inner(frame); unsafe { - process.handle_signal(frame); + thread.handle_pending_signals(frame); } } diff --git a/src/debug.rs b/src/debug.rs index 8f297b1b..9ba51389 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -4,7 +4,7 @@ use core::fmt::{self, Arguments}; use abi::error::Error; use kernel_util::util::StaticVector; -use crate::sync::IrqSafeSpinlock; +use crate::{sync::IrqSafeSpinlock, task::process::Process}; const MAX_DEBUG_SINKS: usize = 4; @@ -13,6 +13,8 @@ struct SimpleLogger; /// Defines the severity of the message #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum LogLevel { + /// Very verbose low-level debugging information + Trace, /// Debugging and verbose information Debug, /// General information about transitions in the system state @@ -93,11 +95,25 @@ impl log::Log for SimpleLogger { } fn log(&self, record: &log::Record) { + let file = record.file().unwrap_or(""); + let line = record.line().unwrap_or(0); + match record.level() { - log::Level::Warn => warnln!("{}", record.args()), - log::Level::Info => infoln!("{}", record.args()), - log::Level::Trace | log::Level::Debug => debugln!("{}", record.args()), - log::Level::Error => errorln!("{}", record.args()), + log::Level::Error => { + log_print_raw!(LogLevel::Error, "{}:{}: {}\n", file, line, record.args()) + } + log::Level::Warn => { + log_print_raw!(LogLevel::Warning, "{}:{}: {}\n", file, line, record.args()) + } // warnln!("{}", record.args()), + log::Level::Info => { + log_print_raw!(LogLevel::Info, "{}:{}: {}\n", file, line, record.args()) + } + log::Level::Debug => { + log_print_raw!(LogLevel::Debug, "{}:{}: {}\n", file, line, record.args()) + } + log::Level::Trace => { + log_print_raw!(LogLevel::Trace, "{}:{}: {}\n", file, line, record.args()) + } } } @@ -107,6 +123,7 @@ impl log::Log for SimpleLogger { impl LogLevel { fn log_prefix(self) -> &'static str { match self { + LogLevel::Trace => "", LogLevel::Debug => "", LogLevel::Info => "\x1b[36m\x1b[1m", LogLevel::Warning => "\x1b[33m\x1b[1m", @@ -117,6 +134,7 @@ impl LogLevel { fn log_suffix(self) -> &'static str { match self { + LogLevel::Trace => "", LogLevel::Debug => "", LogLevel::Info => "\x1b[0m", LogLevel::Warning => "\x1b[0m", @@ -166,6 +184,17 @@ pub fn add_sink(sink: &'static dyn DebugSink, level: LogLevel) { .push(DebugSinkWrapper { inner: sink, level }); } +/// Print a trace message coming from a process +pub fn program_trace(process: &Process, message: &str) { + log_print_raw!( + LogLevel::Trace, + "[trace {} {:?}] {}\n", + process.id(), + process.name(), + message + ); +} + /// Resets the debugging terminal by clearing it pub fn init() { log::set_logger(&LOGGER) diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 9c8c3e00..31505c13 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -137,6 +137,12 @@ pub trait DisplayConsole { self.flush(&mut state); } } + + /// Returns the dimensions of the console in chars: (rows, columns) + fn text_dimensions(&self) -> (usize, usize) { + let state = self.state().lock(); + (state.buffer.height as _, CONSOLE_ROW_LEN as _) + } } impl ConsoleChar { @@ -275,6 +281,11 @@ impl ConsoleBuffer { self.rows[row as usize].chars[col as usize] = c; } + #[inline(never)] + fn set_dirty(&mut self, row: u32) { + self.rows[row as usize].dirty = 1; + } + /// Returns an iterator over dirty rows, while clearing dirty flag for them pub fn flush_rows(&mut self) -> RowIter { RowIter { @@ -289,6 +300,21 @@ impl ConsoleBuffer { } } + fn clear_row(&mut self, row: u32, bg: ColorAttribute) { + self.rows[row as usize].dirty = 1; + self.rows[row as usize].clear(bg); + } + + fn erase_in_row(&mut self, row: u32, start: usize, bg: ColorAttribute) { + self.rows[row as usize].dirty = 1; + self.rows[row as usize].chars[start..].fill(ConsoleChar::from_parts( + b' ', + DEFAULT_FG_COLOR, + bg, + Attributes::empty(), + )); + } + fn scroll_once(&mut self, bg: ColorAttribute) { self.rows.copy_within(1.., 0); self.rows[(self.height - 1) as usize].clear(bg); @@ -395,9 +421,22 @@ impl ConsoleState { self.attributes |= Attributes::BOLD; } // Foreground colors - 31..=37 => { + 30..=39 => { let vt_color = self.esc_args[0] % 10; - self.fg_color = ColorAttribute::from_vt100(vt_color as u8); + if vt_color == 9 { + self.fg_color = DEFAULT_FG_COLOR; + } else { + self.fg_color = ColorAttribute::from_vt100(vt_color as u8); + } + } + // Background colors + 40..=49 => { + let vt_color = self.esc_args[0] % 10; + if vt_color == 9 { + self.bg_color = DEFAULT_BG_COLOR; + } else { + self.bg_color = ColorAttribute::from_vt100(vt_color as u8); + } } _ => (), } @@ -405,9 +444,11 @@ impl ConsoleState { } // Move cursor to position b'f' => { - let row = self.esc_args[1].clamp(1, self.buffer.height) - 1; + let row = self.esc_args[0].clamp(1, self.buffer.height) - 1; let col = self.esc_args[1].clamp(1, CONSOLE_ROW_LEN as u32) - 1; + self.buffer.set_dirty(row); + self.cursor_row = row; self.cursor_col = col; } @@ -423,6 +464,19 @@ impl ConsoleState { } _ => (), }, + // Erase in Line + b'K' => match self.esc_args[0] { + // Erase to Right + 0 => { + self.buffer + .erase_in_row(self.cursor_row, self.cursor_col as _, self.bg_color); + } + // Erase All + 2 => { + self.buffer.clear_row(self.cursor_row, self.bg_color); + } + _ => (), + }, _ => (), } diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 81933eb7..53c4cb6b 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -18,6 +18,8 @@ struct Inner { char_height: u32, width: u32, height: u32, + cursor_row: u32, + cursor_col: u32, } struct DrawGlyph { @@ -61,6 +63,20 @@ impl DisplayConsole for FramebufferConsole { let mut iter = state.buffer.flush_rows(); + let old_cursor_col = inner.cursor_col; + let old_cursor_row = inner.cursor_row; + // New cursor + let cursor_col = state.cursor_col; + let cursor_row = state.cursor_row; + + inner.fill_rect( + old_cursor_col * cw, + old_cursor_row * ch, + cw, + ch, + state.bg_color.as_rgba(false), + ); + while let Some((row_idx, row)) = iter.next_dirty() { if row_idx >= inner.height { break; @@ -70,9 +86,12 @@ impl DisplayConsole for FramebufferConsole { let glyph = chr.character(); let (fg, bg, attr) = chr.attributes(); - let fg = fg.as_rgba(attr.contains(Attributes::BOLD)); - let bg = bg.as_rgba(false); + let mut fg = fg.as_rgba(attr.contains(Attributes::BOLD)); + let mut bg = bg.as_rgba(false); + if row_idx == cursor_row && col_idx == cursor_col as usize { + core::mem::swap(&mut fg, &mut bg); + } inner.draw_glyph( font, DrawGlyph { @@ -88,9 +107,6 @@ impl DisplayConsole for FramebufferConsole { } // Place cursor - let cursor_row = state.cursor_row; - let cursor_col = state.cursor_col; - inner.fill_rect( cursor_col * cw, cursor_row * ch, @@ -98,6 +114,9 @@ impl DisplayConsole for FramebufferConsole { ch, state.fg_color.as_rgba(false), ); + + inner.cursor_col = cursor_col; + inner.cursor_row = cursor_row; } } @@ -120,6 +139,8 @@ impl FramebufferConsole { height: dim.height / char_height, char_width, char_height, + cursor_row: 0, + cursor_col: 0, }; Ok(Self { @@ -149,6 +170,7 @@ impl Inner { while x < font.width() { let v = if glyph[0] & mask != 0 { g.fg } else { g.bg }; + let v = v | 0xFF000000; fb[g.sy + y][(g.sx + x) as usize] = v; mask >>= 1; x += 1; diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index cd3dd189..0d15b89b 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -92,6 +92,7 @@ impl DisplayDevice for LinearFramebuffer { impl FramebufferAccess { /// Copies `count` rows starting from `src_row` to `dst_row` + #[optimize(speed)] pub fn copy_rows(&mut self, src_row: u32, dst_row: u32, count: u32) { use core::ffi::c_void; extern "C" { diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index e90bbc31..22136890 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -22,6 +22,7 @@ use crate::{ fs::devfs::{self, CharDeviceType}, mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, sync::IrqSafeSpinlock, + task::process::ProcessId, }; register_bitfields! { @@ -127,7 +128,12 @@ impl CharDevice for Pl011 { fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { match req { &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(id as _); + self.set_signal_group(ProcessId::from(id)); + Ok(()) + } + DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), + DeviceRequest::GetTerminalOptions(config) => { + config.write(self.context.config()); Ok(()) } _ => Err(Error::InvalidArgument), @@ -149,25 +155,25 @@ impl InterruptHandler for Pl011 { let byte = inner.regs.DR.get(); drop(inner); - if byte == b'\x1b' as u32 { - use crate::task::sched::CpuQueue; + // if byte == b'\x1b' as u32 { + // use crate::task::sched::CpuQueue; - for (i, queue) in CpuQueue::all().enumerate() { - log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); - let lock = unsafe { queue.grab() }; - for item in lock.iter() { - log_print_raw!( - LogLevel::Fatal, - "* {} {:?} {:?}\n", - item.id(), - item.name(), - item.state() - ); - } - } - } else { - self.recv_byte(byte as u8); - } + // for (i, queue) in CpuQueue::all().enumerate() { + // log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); + // let lock = unsafe { queue.grab() }; + // for item in lock.iter() { + // log_print_raw!( + // LogLevel::Fatal, + // "* {} {:?} {:?}\n", + // item.id(), + // item.name(), + // item.state() + // ); + // } + // } + // } else { + self.recv_byte(byte as u8); + // } true } diff --git a/src/device/tty.rs b/src/device/tty.rs index bd892269..6f75a59d 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -8,15 +8,18 @@ use device_api::serial::SerialDevice; use crate::{ sync::IrqSafeSpinlock, - task::{process::Process, ProcessId}, + task::process::{Process, ProcessId}, util::ring::AsyncRing, }; #[cfg(feature = "fb_console")] pub mod combined { //! Combined console + keyboard terminal device - use crate::block; - use abi::{error::Error, io::DeviceRequest}; + use crate::{block, task::process::ProcessId}; + use abi::{ + error::Error, + io::{DeviceRequest, TerminalSize}, + }; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; use vfs::CharDevice; @@ -90,10 +93,19 @@ pub mod combined { fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { match req { &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(id as _); + self.set_signal_group(ProcessId::from(id)); + Ok(()) + } + DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), + DeviceRequest::GetTerminalOptions(config) => { + config.write(self.context.config()); + Ok(()) + } + DeviceRequest::GetTerminalSize(out) => { + let (rows, columns) = self.output.text_dimensions(); + out.write(TerminalSize { rows, columns }); Ok(()) } - DeviceRequest::ConfigureTerminal(config) => self.context.set_config(config), _ => Err(Error::InvalidArgument), } } @@ -292,6 +304,10 @@ impl TtyContext { self.inner.lock().config = config.clone(); Ok(()) } + + pub fn config(&self) -> TerminalOptions { + self.inner.lock().config + } } // impl CharRingInner { diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index 30441c39..0dba0878 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -23,14 +23,15 @@ pub enum CharDeviceType { struct DevfsDirectory; impl VnodeImpl for DevfsDirectory { - fn open(&self, _node: &VnodeRef, _opts: OpenOptions, _mode: FileMode) -> Result { + fn open(&self, _node: &VnodeRef, _opts: OpenOptions) -> Result { Ok(DIR_POSITION_FROM_CACHE) } fn metadata(&self, _node: &VnodeRef) -> Result { Ok(FileAttr { size: 0, - mode: FileMode::default_dir(), + // TODO mutable directory mode + mode: FileMode::from(0o755), ty: FileType::Directory, }) } diff --git a/src/init.rs b/src/init.rs index 2ebda395..dfdcdbdc 100644 --- a/src/init.rs +++ b/src/init.rs @@ -46,7 +46,7 @@ pub fn kinit() { let ioctx = IoContext::new(root); let node = ioctx.find(None, "/init", true, true).unwrap(); - let file = node.open(OpenOptions::READ, FileMode::empty()).unwrap(); + let file = node.open(OpenOptions::READ).unwrap(); let devfs = devfs::root(); #[cfg(target_arch = "x86_64")] @@ -55,12 +55,14 @@ pub fn kinit() { let console = ioctx .find(Some(devfs.clone()), "ttyS0", true, true) .unwrap(); - let stdin = console.open(OpenOptions::READ, FileMode::empty()).unwrap(); - let stdout = console.open(OpenOptions::WRITE, FileMode::empty()).unwrap(); + let stdin = console.open(OpenOptions::READ).unwrap(); + let stdout = console.open(OpenOptions::WRITE).unwrap(); let stderr = stdout.clone(); { - let user_init = proc::exec::load_elf("init", file, &["/init", "xxx"], &[]).unwrap(); + // XXX + let (user_init, user_init_main) = + proc::exec::load_elf("init", file, &["/init", "xxx"], &[]).unwrap(); let mut io = user_init.io.lock(); io.set_ioctx(ioctx); io.set_file(RawFd::STDIN, stdin).unwrap(); @@ -70,6 +72,6 @@ pub fn kinit() { user_init.set_session_terminal(console); - user_init.enqueue_somewhere(); + user_init_main.enqueue_somewhere(); } } diff --git a/src/panic.rs b/src/panic.rs index f59a9110..312b31db 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -51,78 +51,78 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { infoln!("{:?}", pi); - if PANIC_HAPPENED - .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) - .is_ok() - { - let id = Cpu::local_id(); - // Let other CPUs know we're screwed - unsafe { - ARCHITECTURE - .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) - .ok(); - } + // if PANIC_HAPPENED + // .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) + // .is_ok() + // { + // let id = Cpu::local_id(); + // // Let other CPUs know we're screwed + // unsafe { + // ARCHITECTURE + // .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) + // .ok(); + // } - let ap_count = ArchitectureImpl::cpu_count() - 1; - PANIC_HANDLED_FENCE.wait_all(ap_count); + // let ap_count = ArchitectureImpl::cpu_count() - 1; + // PANIC_HANDLED_FENCE.wait_all(ap_count); - unsafe { - hack_locks(); - } + // unsafe { + // hack_locks(); + // } - log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); - log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); - log_print_raw!(LogLevel::Fatal, "Kernel panic "); + // log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); + // log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); + // log_print_raw!(LogLevel::Fatal, "Kernel panic "); - if let Some(location) = pi.location() { - log_print_raw!( - LogLevel::Fatal, - "at {}:{}:", - location.file(), - location.line() - ); - } else { - log_print_raw!(LogLevel::Fatal, ":"); - } + // if let Some(location) = pi.location() { + // log_print_raw!( + // LogLevel::Fatal, + // "at {}:{}:", + // location.file(), + // location.line() + // ); + // } else { + // log_print_raw!(LogLevel::Fatal, ":"); + // } - log_print_raw!(LogLevel::Fatal, "\n"); + // log_print_raw!(LogLevel::Fatal, "\n"); - if let Some(msg) = pi.message() { - debug_internal(*msg, LogLevel::Fatal); - log_print_raw!(LogLevel::Fatal, "\n"); - } + // if let Some(msg) = pi.message() { + // debug_internal(*msg, LogLevel::Fatal); + // log_print_raw!(LogLevel::Fatal, "\n"); + // } - for (i, queue) in CpuQueue::all().enumerate() { - log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); - let lock = unsafe { queue.grab() }; - for item in lock.iter() { - log_print_raw!( - LogLevel::Fatal, - "* {} {:?} {:?}\n", - item.id(), - item.name(), - item.state() - ); - } - } + // for (i, queue) in CpuQueue::all().enumerate() { + // log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); + // let lock = unsafe { queue.grab() }; + // for item in lock.iter() { + // log_print_raw!( + // LogLevel::Fatal, + // "* {} {:?} {:?}\n", + // item.id(), + // item.name(), + // item.state() + // ); + // } + // } - log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); + // log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); - PANIC_FINISHED_FENCE.signal(); - while PANIC_SEQUENCE.load(Ordering::Acquire) != id { - core::hint::spin_loop(); - } + // PANIC_FINISHED_FENCE.signal(); + // while PANIC_SEQUENCE.load(Ordering::Acquire) != id { + // core::hint::spin_loop(); + // } - log_print_raw!(LogLevel::Fatal, "X"); + // log_print_raw!(LogLevel::Fatal, "X"); - flush_consoles(); + // flush_consoles(); - PANIC_SEQUENCE.fetch_add(1, Ordering::Release); + // PANIC_SEQUENCE.fetch_add(1, Ordering::Release); - unsafe { - ARCHITECTURE.reset(); - } - } + // unsafe { + // ARCHITECTURE.reset(); + // } + // } loop { ArchitectureImpl::wait_for_interrupt(); diff --git a/src/proc/exec.rs b/src/proc/exec.rs index f81dca62..2ac0b8a5 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -15,7 +15,7 @@ use crate::{ ForeignPointer, }, proc, - task::{context::TaskContextImpl, process::Process, TaskContext}, + task::{context::TaskContextImpl, process::Process, thread::Thread, TaskContext}, }; pub struct BufferPlacer<'a> { @@ -102,7 +102,7 @@ fn setup_binary>( entry: usize, args: &[&str], envs: &[&str], -) -> Result, Error> { +) -> Result<(Arc, Arc), Error> { const USER_STACK_PAGES: usize = 16; let virt_stack_base = 0x3000000; @@ -140,7 +140,8 @@ fn setup_binary>( let context = TaskContext::user(entry, arg, space.as_address_with_asid(), user_sp)?; - Ok(Process::new_with_context(name, Some(space), context)) + Ok(Process::new_with_main(name, Arc::new(space), context)) + // Ok(Process::new_with_context(name, Some(space), context)) } /// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory @@ -149,7 +150,7 @@ pub fn load_elf>( file: FileRef, args: &[&str], envs: &[&str], -) -> Result, Error> { +) -> Result<(Arc, Arc), Error> { let space = ProcessAddressSpace::new()?; let elf_entry = proc::elf::load_elf_from_file(&space, file)?; diff --git a/src/syscall/arg.rs b/src/syscall/arg.rs index dede36c3..b772c851 100644 --- a/src/syscall/arg.rs +++ b/src/syscall/arg.rs @@ -3,17 +3,19 @@ use yggdrasil_abi::error::Error; use crate::{ mem::{validate_user_region, ForeignPointer}, - task::process::Process, + task::thread::Thread, }; +// XXX + pub(super) fn arg_buffer_ref<'a>(base: usize, len: usize) -> Result<&'a [u8], Error> { - let proc = Process::current(); + let proc = Thread::current(); validate_user_region(proc.address_space(), base, len, false)?; Ok(unsafe { core::slice::from_raw_parts(base as *const u8, len) }) } pub(super) fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8], Error> { - let proc = Process::current(); + let proc = Thread::current(); validate_user_region(proc.address_space(), base, len, true)?; Ok(unsafe { core::slice::from_raw_parts_mut(base as *mut u8, len) }) } @@ -28,13 +30,13 @@ pub(super) fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error } pub(super) fn arg_user_ref<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { - let proc = Process::current(); + let proc = Thread::current(); let ptr = addr as *const T; unsafe { ptr.validate_user_ptr(proc.address_space()) } } pub(super) fn arg_user_mut<'a, T: Sized>(addr: usize) -> Result<&'a mut T, Error> { - let proc = Process::current(); + let proc = Thread::current(); let ptr = addr as *mut T; unsafe { ptr.validate_user_mut(proc.address_space()) } } @@ -43,7 +45,7 @@ pub(super) fn arg_user_slice_mut<'a, T: Sized>( base: usize, count: usize, ) -> Result<&'a mut [T], Error> { - let proc = Process::current(); + let proc = Thread::current(); let ptr = base as *mut T; unsafe { ptr.validate_user_slice_mut(count, proc.address_space()) } } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 1817665b..d17eb0d6 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -15,20 +15,24 @@ use yggdrasil_abi::{ }; use crate::{ - block, fs, + block, + debug::LogLevel, + fs, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo}, sync::IrqSafeSpinlockGuard, - task::{process::Process, runtime, ProcessId}, + task::{ + process::{Process, ProcessId}, + runtime, + thread::Thread, + }, }; mod arg; use arg::*; -fn run_with_io) -> T>(f: F) -> T { - let proc = Process::current(); +fn run_with_io) -> T>(proc: &Process, f: F) -> T { let io = proc.io.lock(); - f(io) } @@ -36,10 +40,10 @@ fn run_with_io_at< T, F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, >( + proc: &Process, at: Option, f: F, ) -> Result { - let proc = Process::current(); let io = proc.io.lock(); let at = at .map(|fd| io.file(fd).and_then(|f| f.borrow().node())) @@ -49,34 +53,36 @@ fn run_with_io_at< } fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { + let thread = Thread::current(); + let process = thread.process(); + match func { SyscallFunction::DebugTrace => { - let pid = Process::current().id(); + let pid = process.id(); + let tid = thread.id(); + let arg = arg_user_str(args[0] as usize, args[1] as usize)?; - debugln!("[{}] TRACE: {:?}", pid, arg); + + log_print_raw!(LogLevel::Debug, "[{}:{}] TRACE: {}\n", pid, tid, arg); Ok(0) } SyscallFunction::Nanosleep => { - let seconds = args[0]; - let nanos = args[1] as u32; - let duration = Duration::new(seconds, nanos); + todo!(); + // let seconds = args[0]; + // let nanos = args[1] as u32; + // let duration = Duration::new(seconds, nanos); - block! { - runtime::sleep(duration).await - } - .map(|_| 0) - } - SyscallFunction::Exit => { - let code = ExitCode::from(args[0] as i32); - Process::current().exit(code); - panic!(); + // block! { + // runtime::sleep(duration).await + // } + // .map(|_| 0) } + // Resource management SyscallFunction::MapMemory => { let len = args[1] as usize; - let proc = Process::current(); - let space = proc.address_space(); + let space = thread.address_space(); if len & 0xFFF != 0 { todo!(); @@ -93,8 +99,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let addr = args[0] as usize; let len = args[1] as usize; - let proc = Process::current(); - let space = proc.address_space(); + let space = thread.address_space(); if len & 0xFFF != 0 { todo!(); @@ -106,11 +111,20 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } + SyscallFunction::SetSignalEntry => { + let entry = args[0] as usize; + let sp = args[1] as usize; + + thread.set_signal_entry(entry, sp); + + Ok(0) + } + // I/O SyscallFunction::Write => { let fd = RawFd(args[0] as u32); let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - run_with_io(|io| { + run_with_io(&process, |io| { let file = io.file(fd)?; let mut file_borrow = file.borrow_mut(); @@ -121,7 +135,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - run_with_io(|io| { + run_with_io(&process, |io| { let file = io.file(fd)?; let mut file_borrow = file.borrow_mut(); @@ -133,15 +147,15 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; let opts = OpenOptions::from(args[3] as u32); let mode = FileMode::from(args[4] as u32); - let proc = Process::current(); - run_with_io_at(at, |at, mut io| { + run_with_io_at(&process, at, |at, mut io| { let file = io.ioctx().open(at, path, opts, mode)?; - if proc.session_terminal().is_none() && + // TODO NO_CTTY? + if process.session_terminal().is_none() && let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { - debugln!("Session terminal set for #{}: {}", proc.id(), path); - proc.set_session_terminal(node); + debugln!("Session terminal set for #{}: {}", process.id(), path); + process.set_session_terminal(node); } let fd = io.place_file(file)?; @@ -151,46 +165,16 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::Close => { let fd = RawFd(args[0] as u32); - run_with_io(|mut io| { + run_with_io(&process, |mut io| { io.close_file(fd)?; Ok(0) }) } - SyscallFunction::Mount => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(|mut io| { - let target_node = io.ioctx().find(None, options.target, true, false)?; - if !target_node.is_directory() { - return Err(Error::NotADirectory); - } - - let fs_root = fs::create_filesystem(options)?; - - target_node.mount(fs_root)?; - - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - - Ok(0) - }) - } - SyscallFunction::Unmount => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(|mut io| { - let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; - mountpoint.unmount_target()?; - - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - - Ok(0) - }) - } SyscallFunction::OpenDirectory => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - run_with_io_at(at, |at, mut io| { + run_with_io_at(&process, at, |at, mut io| { let node = io.ioctx().find(at, path, true, true)?; let file = node.open_directory()?; let fd = io.place_file(file)?; @@ -205,7 +189,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result args[2] as usize, )?; - run_with_io(|io| { + run_with_io(&process, |io| { let file = io.file(fd)?; let mut file_borrow = file.borrow_mut(); @@ -217,10 +201,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; let _mode = FileMode::from(args[3] as u32); - run_with_io_at(at, |at, mut io| { + run_with_io_at(&process, at, |at, mut io| { let (parent, name) = abi::path::split_right(path); let parent_node = io.ioctx().find(at, parent, true, true)?; - parent_node.create(name, VnodeKind::Directory)?; + todo!(); + // parent_node.create(name, VnodeKind::Directory)?; Ok(0) }) @@ -230,7 +215,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; let recurse = args[3] != 0; - run_with_io_at(at, |at, mut io| { + run_with_io_at(&process, at, |at, mut io| { let node = io.ioctx().find(at, path, false, false)?; if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { @@ -250,7 +235,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let buffer = arg_user_mut::>(args[3] as usize)?; let follow = args[4] != 0; - run_with_io_at(at, |at, mut io| { + run_with_io_at(&process, at, |at, mut io| { let node = if path.is_empty() { at.ok_or(Error::InvalidArgument)? } else { @@ -267,37 +252,79 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let pos = SeekFrom::from(args[1]); - run_with_io(|io| { + run_with_io(&process, |io| { let file = io.file(fd)?; let mut file_borrow = file.borrow_mut(); file_borrow.seek(pos).map(|v| v as usize) }) } + SyscallFunction::Mount => { + let options = arg_user_ref::(args[0] as usize)?; + + run_with_io(&process, |mut io| { + let target_node = io.ioctx().find(None, options.target, true, false)?; + if !target_node.is_directory() { + return Err(Error::NotADirectory); + } + + let fs_root = fs::create_filesystem(options)?; + + target_node.mount(fs_root)?; + + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); + + Ok(0) + }) + } + SyscallFunction::Unmount => { + let options = arg_user_ref::(args[0] as usize)?; + + run_with_io(&process, |mut io| { + let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; + mountpoint.unmount_target()?; + + debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); + + Ok(0) + }) + } + SyscallFunction::DeviceRequest => { + let fd = RawFd(args[0] as u32); + let req = arg_user_mut::(args[1] as usize)?; + + run_with_io(&process, |io| { + let file = io.file(fd)?; + let node = file.borrow().node()?; + + node.device_request(req)?; + + Ok(0) + }) + } + // Process management SyscallFunction::Spawn => { let options = arg_user_ref::(args[0] as usize)?; - let proc = Process::current(); - - run_with_io(|mut io| { + run_with_io(&process, |mut io| { let node = io.ioctx().find(None, options.program, true, true)?; // Setup a new process from the file - let file = node.open(OpenOptions::READ, FileMode::empty())?; - let child = proc::exec::load_elf( + let file = node.open(OpenOptions::READ)?; + let (child_process, child_main) = proc::exec::load_elf( options.program, file, options.arguments, options.environment, )?; - let pid = child.id() as u32; + let pid: u32 = child_process.id().into(); // Inherit group and session from the creator - child.inherit(&proc)?; + child_process.inherit(&process)?; // Inherit root from the creator let child_ioctx = IoContext::new(io.ioctx().root().clone()); - let mut child_io = child.io.lock(); + let mut child_io = child_process.io.lock(); child_io.set_ioctx(child_ioctx); for opt in options.optional { @@ -307,7 +334,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result child_io.set_file(child, src_file)?; } &SpawnOption::SetProcessGroup(pgroup) => { - child.set_group_id(pgroup as _); + child_process.set_group_id(pgroup.into()); } _ => (), } @@ -323,21 +350,63 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result debugln!("{} requested terminal {:?}", pid, fd); let file = child_io.file(fd)?; let node = file.borrow().node()?; - let mut req = DeviceRequest::SetTerminalGroup(child.group_id() as _); + let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); node.device_request(&mut req)?; } drop(child_io); - child.enqueue_somewhere(); + child_main.enqueue_somewhere(); Ok(pid as _) }) } + SyscallFunction::Exit => { + let code = ExitCode::from(args[0] as i32); + // TODO separate handlers for process exit and thread exit? + thread.exit(code); + // Process::current().exit(code); + panic!(); + } + SyscallFunction::GetPid => Ok(u32::from(process.id()) as usize), + SyscallFunction::GetSessionId => todo!(), + SyscallFunction::GetProcessGroupId => todo!(), + SyscallFunction::SetProcessGroupId => { + let pid = ProcessId::from(args[0] as u32); + let group_id = ProcessId::from(args[1] as u32); + + // Other syscall variants are not currently supported + assert_eq!(pid, group_id); + assert_eq!(process.id(), pid); + + process.set_group_id(group_id); + Ok(0) + } + SyscallFunction::StartSession => { + let session_terminal = process.clear_session_terminal(); + + if let Some(ctty) = session_terminal { + // Drop all FDs referring to the old session terminal + run_with_io(&process, |mut io| { + io.retain(|_, f| { + f.borrow() + .node() + .map(|node| !Rc::ptr_eq(&node, &ctty)) + .unwrap_or(true) + }); + }); + } + + process.set_session_id(process.id()); + process.set_group_id(process.id()); + + Ok(0) + } + // Waiting and polling SyscallFunction::WaitProcess => { - let pid = args[0] as ProcessId; - let status = arg_user_mut::(args[1] as _)?; + let pid = ProcessId::from(args[0] as u32); debugln!("WaitProcess #{}", pid); + let status = arg_user_mut::(args[1] as _)?; let target = Process::get(pid).ok_or(Error::DoesNotExist)?; @@ -350,78 +419,20 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } - SyscallFunction::SendSignal => { - let pid = args[0] as u32; - let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + _ => todo!("{:?}", func), + // SyscallFunction::SendSignal => { + // let pid = args[0] as u32; + // let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; - target.raise_signal(signal); + // let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; + // target.raise_signal(signal); - Ok(0) - } - SyscallFunction::SetSignalEntry => { - let entry = args[0] as usize; - let sp = args[1] as usize; - - Process::current().set_signal_entry(entry, sp); - - Ok(0) - } - SyscallFunction::ExitSignal => { - panic!("Handled elsewhere"); - // Process::current().exit_signal(); - } - SyscallFunction::GetPid => Ok(Process::current().id()), - SyscallFunction::GetSessionId => todo!(), - SyscallFunction::GetProcessGroupId => todo!(), - SyscallFunction::SetProcessGroupId => { - let pid = args[0] as ProcessId; - let group_id = args[1] as ProcessId; - - let proc = Process::current(); - // Other syscall variants are not currently supported - assert_eq!(pid, group_id); - assert_eq!(proc.id(), pid); - - proc.set_group_id(group_id); - Ok(0) - } - SyscallFunction::StartSession => { - let proc = Process::current(); - - let session_terminal = proc.clear_session_terminal(); - - if let Some(ctty) = session_terminal { - // Drop all FDs referring to the old session terminal - run_with_io(|mut io| { - io.retain(|_, f| { - f.borrow() - .node() - .map(|node| !Rc::ptr_eq(&node, &ctty)) - .unwrap_or(true) - }); - }); - } - - proc.set_session_id(proc.id()); - proc.set_group_id(proc.id()); - - Ok(0) - } - - SyscallFunction::DeviceRequest => { - let fd = RawFd(args[0] as u32); - let req = arg_user_mut::(args[1] as usize)?; - - run_with_io(|io| { - let file = io.file(fd)?; - let node = file.borrow().node()?; - - node.device_request(req)?; - - Ok(0) - }) - } + // Ok(0) + // } + // SyscallFunction::ExitSignal => { + // panic!("Handled elsewhere"); + // // Process::current().exit_signal(); + // } } } diff --git a/src/task/context.rs b/src/task/context.rs index 81ebaafa..0c011f96 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -4,7 +4,9 @@ use abi::{arch::SavedFrame, error::Error, process::ExitCode}; use alloc::boxed::Box; use cfg_if::cfg_if; -use crate::task::process::Process; +use crate::task::thread::Thread; + +// use crate::task::process::Process; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -77,7 +79,8 @@ pub trait TaskContextImpl: Sized { extern "C" fn closure_wrapper(closure_addr: usize) -> ! { let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; closure(); - Process::current().exit(ExitCode::SUCCESS); + + Thread::current().exit(ExitCode::SUCCESS); unreachable!(); } diff --git a/src/task/mod.rs b/src/task/mod.rs index 7f1ac299..0a71c811 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -8,7 +8,7 @@ use alloc::{string::String, sync::Arc, vec::Vec}; use crate::{ arch::{Architecture, ArchitectureImpl}, sync::{IrqSafeSpinlock, SpinFence}, - task::sched::CpuQueue, + task::{sched::CpuQueue, thread::Thread}, }; use self::{context::TaskContextImpl, process::Process}; @@ -17,58 +17,59 @@ pub mod context; pub mod process; pub mod runtime; pub mod sched; +pub mod thread; pub use context::{Cpu, TaskContext}; -/// Process identifier alias for clarity -pub type ProcessId = usize; +// /// Process identifier alias for clarity +// pub type ProcessId = usize; -/// Wrapper structure to hold all the system's processes -pub struct ProcessList { - data: Vec<(ProcessId, Arc)>, - last_process_id: ProcessId, -} - -impl ProcessList { - /// Constructs an empty process list - pub const fn new() -> Self { - Self { - last_process_id: 0, - data: Vec::new(), - } - } - - /// Inserts a new process into the list. - /// - /// # Safety - /// - /// Only meant to be called from inside the Process impl, as this function does not perform any - /// accounting information updates. - pub unsafe fn push(&mut self, process: Arc) -> ProcessId { - self.last_process_id += 1; - debugln!("Insert process with ID {}", self.last_process_id); - self.data.push((self.last_process_id, process)); - self.last_process_id - } - - /// Looks up a process by its ID - pub fn get(&self, id: ProcessId) -> Option<&Arc> { - self.data - .iter() - .find_map(|(i, p)| if *i == id { Some(p) } else { None }) - } -} - -/// Global shared process list -pub static PROCESSES: IrqSafeSpinlock = IrqSafeSpinlock::new(ProcessList::new()); +// /// Wrapper structure to hold all the system's processes +// pub struct ProcessList { +// data: Vec<(ProcessId, Arc)>, +// last_process_id: ProcessId, +// } +// +// impl ProcessList { +// /// Constructs an empty process list +// pub const fn new() -> Self { +// Self { +// last_process_id: 0, +// data: Vec::new(), +// } +// } +// +// /// Inserts a new process into the list. +// /// +// /// # Safety +// /// +// /// Only meant to be called from inside the Process impl, as this function does not perform any +// /// accounting information updates. +// pub unsafe fn push(&mut self, process: Arc) -> ProcessId { +// self.last_process_id += 1; +// debugln!("Insert process with ID {}", self.last_process_id); +// self.data.push((self.last_process_id, process)); +// self.last_process_id +// } +// +// /// Looks up a process by its ID +// pub fn get(&self, id: ProcessId) -> Option<&Arc> { +// self.data +// .iter() +// .find_map(|(i, p)| if *i == id { Some(p) } else { None }) +// } +// } +// +// /// Global shared process list +// pub static PROCESSES: IrqSafeSpinlock = IrqSafeSpinlock::new(ProcessList::new()); /// Creates a new kernel-space process to execute a closure and queues it to some CPU pub fn spawn_kernel_closure, F: Fn() + Send + 'static>( name: S, f: F, ) -> Result<(), Error> { - let proc = Process::new_with_context(name, None, TaskContext::kernel_closure(f)?); - proc.enqueue_somewhere(); + let thread = Thread::new_kthread(name, TaskContext::kernel_closure(f)?); + thread.enqueue_somewhere(); Ok(()) } diff --git a/src/task/process.rs b/src/task/process.rs index 071dd576..556d85e0 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,387 +1,142 @@ //! Process data structures + use core::{ - mem::size_of, - ops::Deref, + fmt, pin::Pin, - sync::atomic::{AtomicU32, Ordering}, + sync::atomic::{AtomicU64, Ordering}, task::{Context, Poll}, }; use abi::{ error::Error, - process::{ExitCode, Signal, SignalEntryData}, + process::{ExitCode, Signal}, }; -use alloc::{collections::VecDeque, string::String, sync::Arc}; -use atomic_enum::atomic_enum; -use futures_util::{task::ArcWake, Future}; +use alloc::{ + collections::{BTreeMap, VecDeque}, + string::String, + sync::Arc, + vec::Vec, +}; +use futures_util::Future; use kernel_util::util::OneTimeInit; use vfs::VnodeRef; -use crate::{ - mem::{process::ProcessAddressSpace, ForeignPointer}, - proc::io::ProcessIo, - sync::{IrqGuard, IrqSafeSpinlock}, - task::context::TaskContextImpl, -}; +use crate::{mem::process::ProcessAddressSpace, proc::io::ProcessIo, sync::IrqSafeSpinlock}; use super::{ - context::TaskFrame, runtime::QueueWaker, sched::CpuQueue, Cpu, ProcessId, TaskContext, - PROCESSES, + runtime::QueueWaker, + thread::{Thread, ThreadId, ThreadState}, + TaskContext, }; -/// Represents the states a process can be at some point in time -#[atomic_enum] #[derive(PartialEq)] pub enum ProcessState { - /// Process is ready for execution and is present in some CPU's queue - Ready, - /// Process is currently being executed by some CPU Running, - /// Process is present in a global list, but is not queued for execution until it is resumed - Suspended, - /// Process is terminated and waits to be reaped - Terminated, + Terminated(ExitCode), } -/// Describes signal entry information -#[derive(Debug, Clone)] -pub struct SignalEntry { - entry: usize, - stack: usize, -} +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct ProcessId(u64); struct ProcessInner { - exit_status: i32, + state: ProcessState, session_id: ProcessId, group_id: ProcessId, - session_terminal: Option, - signal_entry: Option, - signal_stack: VecDeque, - queue: Option<&'static CpuQueue>, + session_terminal: Option, + + threads: Vec>, } -/// Process data and state structure pub struct Process { - normal_context: TaskContext, name: String, + id: ProcessId, - // Process state info - id: OneTimeInit, - state: AtomicProcessState, - cpu_id: AtomicU32, - space: Option, + space: Arc, inner: IrqSafeSpinlock, exit_waker: QueueWaker, - - /// I/O state of the task pub io: IrqSafeSpinlock, } -/// Guard type that provides [Process] operations only available for current processes -pub struct CurrentProcess(Arc, IrqGuard); +static PROCESSES: IrqSafeSpinlock>> = + IrqSafeSpinlock::new(BTreeMap::new()); impl Process { - /// Creates a process from raw architecture-specific [TaskContext]. - /// - /// # Note - /// - /// Has side-effect of allocating a new PID for itself. - pub fn new_with_context>( + pub fn new_with_main>( name: S, - space: Option, - normal_context: TaskContext, - ) -> Arc { - let this = Arc::new(Self { - normal_context, + space: Arc, + context: TaskContext, + ) -> (Arc, Arc) { + let name = name.into(); + let id = ProcessId::next(); - id: OneTimeInit::new(), - name: name.into(), - state: AtomicProcessState::new(ProcessState::Suspended), - cpu_id: AtomicU32::new(0), - space, + let process = Arc::new(Self { + name, + id, + + space: space.clone(), + inner: IrqSafeSpinlock::new(ProcessInner { + state: ProcessState::Running, + session_id: id, + group_id: id, + session_terminal: None, + threads: Vec::new(), + }), exit_waker: QueueWaker::new(), - - inner: IrqSafeSpinlock::new(ProcessInner { - exit_status: 0, - - session_id: 0, - group_id: 0, - session_terminal: None, - signal_entry: None, - signal_stack: VecDeque::new(), - - queue: None, - }), io: IrqSafeSpinlock::new(ProcessIo::new()), }); - let id = unsafe { PROCESSES.lock().push(this.clone()) }; - this.id.init(id); + // Create "main" thread + let thread = Thread::new_uthread(process.clone(), space, context); + process.inner.lock().threads.push(thread.clone()); - { - let mut inner = this.inner.lock(); - inner.session_id = id; - inner.group_id = id; - } + PROCESSES.lock().insert(id, process.clone()); - this + (process, thread) } - /// Returns a reference to the inner architecture-specific [TaskContext]. - pub fn current_context(&self) -> &TaskContext { - &self.normal_context - } - - /// Returns this process' ID pub fn id(&self) -> ProcessId { - *self.id.get() + self.id } - /// Returns the binary name of the process - pub fn name(&self) -> &str { - self.name.as_str() - } - - /// Returns the state of the process. - /// - /// # Note - /// - /// Maybe I should remove this and make ALL state changes atomic. - pub fn state(&self) -> ProcessState { - self.state.load(Ordering::Acquire) - } - - /// Atomically updates the state of the process and returns the previous one. - pub fn set_state(&self, state: ProcessState) -> ProcessState { - self.state.swap(state, Ordering::SeqCst) - } - - /// Marks the task as running on the specified CPU. - /// - /// # Safety - /// - /// Only meant to be called from scheduler routines. - pub unsafe fn set_running(&self, cpu: u32) { - self.cpu_id.store(cpu, Ordering::Release); - self.state.store(ProcessState::Running, Ordering::Release); - } - - /// Returns the address space of the task - pub fn address_space(&self) -> &ProcessAddressSpace { - self.space.as_ref().unwrap() - } - - /// Returns the address space of the task, if one is set - pub fn get_address_space(&self) -> Option<&ProcessAddressSpace> { - self.space.as_ref() - } - - /// Replaces the task's session terminal device with another one - pub fn set_session_terminal(&self, terminal: VnodeRef) { - self.inner.lock().session_terminal.replace(terminal); - } - - /// Removes the task's current terminal - pub fn clear_session_terminal(&self) -> Option { - self.inner.lock().session_terminal.take() - } - - /// Returns the current terminal of the task - pub fn session_terminal(&self) -> Option { - self.inner.lock().session_terminal.clone() - } - - /// Sets the session ID of the task - pub fn set_session_id(&self, sid: ProcessId) { - self.inner.lock().session_id = sid; - } - - /// Sets the process group ID of the task - pub fn set_group_id(&self, mut gid: ProcessId) { - if gid == 0 { - gid = self.id(); - } - self.inner.lock().group_id = gid; - } - - /// Returns the process group ID of the task pub fn group_id(&self) -> ProcessId { self.inner.lock().group_id } - /// Returns the CPU number this task in running on (or the last one) - pub fn cpu_id(&self) -> u32 { - self.cpu_id.load(Ordering::Acquire) + pub fn session_id(&self) -> ProcessId { + self.inner.lock().session_id } - /// Selects a suitable CPU queue and submits the process for execution. - /// - /// # Panics - /// - /// Currently, the code will panic if the process is queued/executing on any queue. - pub fn enqueue_somewhere(self: Arc) -> usize { - // Doesn't have to be precise, so even if something changes, we can still be rebalanced - // to another CPU - let (index, queue) = CpuQueue::least_loaded().unwrap(); - - self.enqueue_to(queue); - - index + pub fn name(&self) -> &str { + self.name.as_ref() } - /// Submits the process to a specific queue. - /// - /// # Panics - /// - /// Currently, the code will panic if the process is queued/executing on any queue. - pub fn enqueue_to(self: Arc, queue: &'static CpuQueue) { - let _irq = IrqGuard::acquire(); - - { - let mut inner = self.inner.lock(); - let old_queue = inner.queue.replace(queue); - if old_queue.is_some() { - // Already in some queue - return; - } - } - match self.state.compare_exchange( - ProcessState::Suspended, - ProcessState::Ready, - Ordering::SeqCst, - Ordering::Relaxed, - ) { - Err(ProcessState::Terminated) => { - // Process might've been killed while `await`ing in a `block!` - debugln!( - "Process {} {:?} already terminated, dropping", - self.id(), - self.name() - ); - } - Err(state) => { - todo!("Unexpected process state when enqueueing: {:?}", state) - } - Ok(_) => unsafe { - queue.enqueue(self); - }, - } + pub fn set_group_id(&self, id: ProcessId) { + self.inner.lock().group_id = id; } - fn dequeue(&self, new_state: ProcessState) { - let _irq = IrqGuard::acquire(); - assert_ne!(new_state, ProcessState::Ready); - assert_ne!(new_state, ProcessState::Running); - - let mut inner = self.inner.lock(); - let current_state = self.state.swap(new_state, Ordering::SeqCst); - let proc_queue = inner.queue.take().unwrap(); - - proc_queue.dequeue(self.id()); - - match current_state { - // NOTE: I'm not sure if the process could've been queued between the store and this - // but most likely not (if I'm not that bad with atomics) - // Do nothing, its queue will just drop the process - ProcessState::Ready => (), - // Do nothing, not in a queue already - ProcessState::Suspended => (), - ProcessState::Terminated => panic!("Process is terminated"), - ProcessState::Running => { - let cpu_id = self.cpu_id.load(Ordering::Acquire); - let local_cpu_id = Cpu::local_id(); - let queue = Cpu::local().queue(); - - if cpu_id == local_cpu_id { - assert_eq!(queue as *const _, proc_queue as *const _, "Process queue mismatch: process says cpu{}, queue {:p}, actual cpu{}, queue {:p}", cpu_id, proc_queue, local_cpu_id, queue); - drop(inner); - // Suspending a process running on local CPU - unsafe { queue.yield_cpu() } - } else { - todo!(); - } - } - } + pub fn set_session_id(&self, id: ProcessId) { + self.inner.lock().session_id = id; } - /// Marks the process as suspended, blocking it from being run until it's resumed. - /// - /// # Note - /// - /// The process may not halt its execution immediately when this function is called, only when - /// this function is called targeting the *current process* running on *local* CPU. - pub fn suspend(&self) { - self.dequeue(ProcessState::Suspended); + // Resources + pub fn session_terminal(&self) -> Option { + self.inner.lock().session_terminal.clone() } - /// Returns an exit code if the process exited, [None] if it didn't - pub fn get_exit_status(&self) -> Option { - if self.state() == ProcessState::Terminated { - Some(ExitCode::from(self.inner.lock().exit_status)) - } else { - None - } + pub fn set_session_terminal(&self, node: VnodeRef) { + self.inner.lock().session_terminal.replace(node); } - /// Returns the [Process] currently executing on local CPU, None if idling. - pub fn get_current() -> Option { - let queue = Cpu::local().queue(); - queue.current_process() + pub fn clear_session_terminal(&self) -> Option { + self.inner.lock().session_terminal.take() } - /// Returns a process by its ID - pub fn get(pid: ProcessId) -> Option> { - PROCESSES.lock().get(pid).cloned() - } - - /// Wraps [Process::get_current()] for cases when the caller is absolutely sure there is a - /// running process (e.g. the call itself comes from a process). - pub fn current() -> CurrentProcess { - Self::get_current().unwrap() - } - - /// Handles the cleanup of an exited process - pub fn handle_exit(&self) { - // Scheduler still holds a lock of this process? - // TODO cancel Wait if a process was killed while suspended? - { - let inner = self.inner.lock(); - let exit_status = ExitCode::from(inner.exit_status); - debugln!( - "Handling exit of #{} with status {:?}", - self.id(), - exit_status - ); - - // TODO cleanup address space - // if let Some(space) = self.get_address_space() { - // } - - self.io.lock().handle_exit(); - } - - // Notify any waiters we're done - self.exit_waker.wake_all(); - } - - /// Raises a signal for the currentprocess - pub fn raise_signal(self: &Arc, signal: Signal) { - { - let mut inner = self.inner.lock(); - inner.signal_stack.push_back(signal); - } - - if self.state() == ProcessState::Suspended { - self.clone().enqueue_somewhere(); - } - } - - /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. - pub fn inherit(&self, parent: &Arc) -> Result<(), Error> { + pub fn inherit(&self, parent: &Process) -> Result<(), Error> { let mut our_inner = self.inner.lock(); let their_inner = parent.inner.lock(); @@ -392,20 +147,15 @@ impl Process { Ok(()) } - /// Raises a signal for the specified process group - pub fn signal_group(group_id: ProcessId, signal: Signal) { - let processes = PROCESSES.lock(); - for (_, proc) in processes.data.iter() { - let inner = proc.inner.lock(); - if proc.state() != ProcessState::Terminated && inner.group_id == group_id { - debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); - drop(inner); - proc.raise_signal(signal); - } + // State + pub fn get_exit_status(&self) -> Option { + match self.inner.lock().state { + ProcessState::Running => None, + ProcessState::Terminated(x) => Some(x), } } - pub fn wait_for_exit(process: Arc) -> impl Future { + pub fn wait_for_exit(process: Arc) -> impl Future { struct ProcessExitFuture { process: Arc, } @@ -429,114 +179,338 @@ impl Process { ProcessExitFuture { process } } -} -impl ArcWake for Process { - fn wake_by_ref(arc_self: &Arc) { - arc_self.clone().enqueue_somewhere(); - } -} - -impl Drop for Process { - fn drop(&mut self) { - infoln!("Drop process!"); - } -} - -impl CurrentProcess { - /// Wraps a process in this structure. - /// - /// # Safety - /// - /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. - pub unsafe fn new(inner: Arc, guard: IrqGuard) -> Self { - Self(inner, guard) - } - - /// Configures signal entry information for the process - pub fn set_signal_entry(&self, entry: usize, stack: usize) { + pub fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { + debugln!("Thread {} of process {}: {:?}", thread, self.id, code); let mut inner = self.inner.lock(); - inner.signal_entry.replace(SignalEntry { entry, stack }); - } - pub fn suspend(&self) -> Result<(), Error> { - self.dequeue(ProcessState::Suspended); + // TODO make this cleaner + let old_len = inner.threads.len(); + inner.threads.retain(|t| t.id() != thread); + assert_ne!(inner.threads.len(), old_len); - let inner = self.inner.lock(); - if !inner.signal_stack.is_empty() { - return Err(Error::Interrupted); + let last_thread = inner.threads.is_empty(); + + if last_thread { + debugln!("Last thread of {} exited", self.id); + + inner.state = ProcessState::Terminated(code); + + self.io.lock().handle_exit(); + + drop(inner); + + self.exit_waker.wake_all(); } - - Ok(()) + // // Scheduler still holds a lock of this process? + // // TODO cancel Wait if a process was killed while suspended? + // { + // let inner = self.inner.lock(); + // let exit_status = ExitCode::from(inner.exit_status); + // debugln!( + // "Handling exit of #{} with status {:?}", + // self.id(), + // exit_status + // ); + // + // // TODO cleanup address space + // // if let Some(space) = self.get_address_space() { + // // } + // + // self.io.lock().handle_exit(); + // } + // + // // Notify any waiters we're done + // self.exit_waker.wake_all(); } - /// Terminate the current process - pub fn exit(&self, status: ExitCode) { - self.inner.lock().exit_status = status.into(); - debugln!("Process {} exited with code {:?}", self.id(), status); + /// Raises a signal for the specified process + pub fn raise_signal(self: &Arc, signal: Signal) { + let thread = self.inner.lock().threads[0].clone(); + thread.raise_signal(signal); - self.handle_exit(); - self.dequeue(ProcessState::Terminated); + // // TODO if the process does not have any running/ready threads, pick one and wake it up + // if inner + // .threads + // .iter() + // .all(|t| t.state.load(Ordering::Acquire) == ThreadState::Suspended) + // { + // let thread = inner.threads[0].clone(); + + // drop(inner); + + // thread.enqueue_somewhere(); + + // return; + // } } - /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. - /// - /// # Safety - /// - /// This function is only meant to be called right before returning from an userspace - /// exception handler. - pub unsafe fn handle_signal(&self, frame: &mut F) { - let mut inner = self.inner.lock(); - if let Some(signal) = inner.signal_stack.pop_front() { - let Some(entry) = inner.signal_entry.clone() else { - todo!(); - }; - - debugln!( - "Enter signal handler from: pc={:#x}, sp={:#x}", - frame.user_ip(), - frame.user_sp() - ); - - // TODO check if really in a syscall, lol - let syscall_return = -(u32::from(Error::Interrupted) as isize); - frame.set_return_value(syscall_return as u64); - - // Setup signal frame - let usp = ((entry.stack - size_of::()) & !0xF) - - TaskContext::SIGNAL_STACK_EXTRA_ALIGN; - let frame_ptr = usp as *mut SignalEntryData; - - let saved_frame = frame.store(); - frame_ptr.write_foreign_volatile( - self.address_space(), - SignalEntryData { - signal, - frame: saved_frame, - }, - ); - - // Setup return to signal handler - debugln!( - "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", - entry.entry, - usp, - entry.stack - ); - - frame.set_user_sp(usp); - frame.set_user_ip(entry.entry); - - // Pass the frame pointer as an argument to signal handler entry - frame.set_argument(usp as _); + /// Raises a signal for the specified process group + pub fn signal_group(group_id: ProcessId, signal: Signal) { + let processes = PROCESSES.lock(); + for (_, proc) in processes.iter() { + let inner = proc.inner.lock(); + if !matches!(inner.state, ProcessState::Terminated(_)) && inner.group_id == group_id { + debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); + drop(inner); + proc.raise_signal(signal); + } } } -} -impl Deref for CurrentProcess { - type Target = Arc; - - fn deref(&self) -> &Self::Target { - &self.0 + // Process list + pub fn get(id: ProcessId) -> Option> { + PROCESSES.lock().get(&id).cloned() } } + +impl fmt::Display for ProcessId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use fmt::Write; + + write!(f, "", self.0) + } +} + +// XXX TODO Remove this +impl From for u32 { + fn from(value: ProcessId) -> Self { + value.0 as _ + } +} + +impl From for ProcessId { + fn from(value: u32) -> Self { + Self(value as _) + } +} + +impl ProcessId { + pub fn next() -> Self { + static COUNTER: AtomicU64 = AtomicU64::new(1); + let id = COUNTER.fetch_add(1, Ordering::SeqCst); + Self(id) + } +} + +// /// Returns the state of the process. +// /// +// /// # Note +// /// +// /// Maybe I should remove this and make ALL state changes atomic. +// pub fn state(&self) -> ProcessState { +// self.state.load(Ordering::Acquire) +// } +// +// /// Atomically updates the state of the process and returns the previous one. +// pub fn set_state(&self, state: ProcessState) -> ProcessState { +// self.state.swap(state, Ordering::SeqCst) +// } +// +// /// Marks the task as running on the specified CPU. +// /// +// /// # Safety +// /// +// /// Only meant to be called from scheduler routines. +// pub unsafe fn set_running(&self, cpu: u32) { +// self.cpu_id.store(cpu, Ordering::Release); +// self.state.store(ProcessState::Running, Ordering::Release); +// } +// +// /// Returns the address space of the task +// pub fn address_space(&self) -> &ProcessAddressSpace { +// self.space.as_ref().unwrap() +// } +// +// /// Returns the address space of the task, if one is set +// pub fn get_address_space(&self) -> Option<&ProcessAddressSpace> { +// self.space.as_ref() +// } +// +// /// Replaces the task's session terminal device with another one +// pub fn set_session_terminal(&self, terminal: VnodeRef) { +// self.inner.lock().session_terminal.replace(terminal); +// } +// +// /// Removes the task's current terminal +// pub fn clear_session_terminal(&self) -> Option { +// } +// +// /// Returns the current terminal of the task +// pub fn session_terminal(&self) -> Option { +// self.inner.lock().session_terminal.clone() +// } +// +// /// Sets the session ID of the task +// pub fn set_session_id(&self, sid: ProcessId) { +// self.inner.lock().session_id = sid; +// } +// +// /// Sets the process group ID of the task +// pub fn set_group_id(&self, mut gid: ProcessId) { +// if gid == 0 { +// gid = self.id(); +// } +// self.inner.lock().group_id = gid; +// } +// +// /// Returns the process group ID of the task +// pub fn group_id(&self) -> ProcessId { +// self.inner.lock().group_id +// } +// +// /// Returns the CPU number this task in running on (or the last one) +// pub fn cpu_id(&self) -> u32 { +// self.cpu_id.load(Ordering::Acquire) +// } +// +// /// Selects a suitable CPU queue and submits the process for execution. +// /// +// /// # Panics +// /// +// /// Currently, the code will panic if the process is queued/executing on any queue. +// pub fn enqueue_somewhere(self: Arc) -> usize { +// // Doesn't have to be precise, so even if something changes, we can still be rebalanced +// // to another CPU +// let (index, queue) = CpuQueue::least_loaded().unwrap(); +// +// self.enqueue_to(queue); +// +// index +// } +// +// /// Submits the process to a specific queue. +// /// +// /// # Panics +// /// +// /// Currently, the code will panic if the process is queued/executing on any queue. +// pub fn enqueue_to(self: Arc, queue: &'static CpuQueue) { +// let _irq = IrqGuard::acquire(); +// +// { +// let mut inner = self.inner.lock(); +// let old_queue = inner.queue.replace(queue); +// if old_queue.is_some() { +// // Already in some queue +// return; +// } +// } +// match self.state.compare_exchange( +// ProcessState::Suspended, +// ProcessState::Ready, +// Ordering::SeqCst, +// Ordering::Relaxed, +// ) { +// Err(ProcessState::Terminated) => { +// // Process might've been killed while `await`ing in a `block!` +// debugln!( +// "Process {} {:?} already terminated, dropping", +// self.id(), +// self.name() +// ); +// } +// Err(state) => { +// todo!("Unexpected process state when enqueueing: {:?}", state) +// } +// Ok(_) => unsafe { +// queue.enqueue(self); +// }, +// } +// } +// +// /// Returns an exit code if the process exited, [None] if it didn't +// pub fn get_exit_status(&self) -> Option { +// if self.state() == ProcessState::Terminated { +// Some(ExitCode::from(self.inner.lock().exit_status)) +// } else { +// None +// } +// } +// +// /// Returns the [Process] currently executing on local CPU, None if idling. +// pub fn get_current() -> Option { +// let queue = Cpu::local().queue(); +// queue.current_process() +// } +// +// /// Returns a process by its ID +// pub fn get(pid: ProcessId) -> Option> { +// PROCESSES.lock().get(pid).cloned() +// } +// +// /// Wraps [Process::get_current()] for cases when the caller is absolutely sure there is a +// /// running process (e.g. the call itself comes from a process). +// pub fn current() -> CurrentProcess { +// Self::get_current().unwrap() +// } +// +// /// Handles the cleanup of an exited process +// pub fn handle_exit(&self) { +// } +// +// /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. +// pub fn inherit(&self, parent: &Arc) -> Result<(), Error> { +// } +// +// +// pub fn wait_for_exit(process: Arc) -> impl Future { +// } +// } +// +// impl ArcWake for Process { +// fn wake_by_ref(arc_self: &Arc) { +// arc_self.clone().enqueue_somewhere(); +// } +// } +// +// impl Drop for Process { +// fn drop(&mut self) { +// infoln!("Drop process!"); +// } +// } +// +// impl CurrentProcess { +// /// Wraps a process in this structure. +// /// +// /// # Safety +// /// +// /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. +// pub unsafe fn new(inner: Arc, guard: IrqGuard) -> Self { +// Self(inner, guard) +// } +// +// /// Configures signal entry information for the process +// pub fn set_signal_entry(&self, entry: usize, stack: usize) { +// let mut inner = self.inner.lock(); +// inner.signal_entry.replace(SignalEntry { entry, stack }); +// } +// +// pub fn suspend(&self) -> Result<(), Error> { +// self.dequeue(ProcessState::Suspended); +// +// let inner = self.inner.lock(); +// if !inner.signal_stack.is_empty() { +// return Err(Error::Interrupted); +// } +// +// Ok(()) +// } +// +// /// Terminate the current process +// pub fn exit(&self, status: ExitCode) { +// self.inner.lock().exit_status = status.into(); +// debugln!("Process {} exited with code {:?}", self.id(), status); +// +// self.handle_exit(); +// self.dequeue(ProcessState::Terminated); +// } +// +// } +// +// impl Deref for CurrentProcess { +// type Target = Arc; +// +// fn deref(&self) -> &Self::Target { +// &self.0 +// } +// } diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs index 0d69ac8f..7509bc7f 100644 --- a/src/task/runtime/executor.rs +++ b/src/task/runtime/executor.rs @@ -4,7 +4,7 @@ use abi::error::Error; use alloc::{boxed::Box, format, sync::Arc}; use futures_util::{task::waker_ref, Future}; -use crate::task::{process::Process, spawn_kernel_closure}; +use crate::task::{spawn_kernel_closure, thread::Thread}; use super::{ task::Task, @@ -40,17 +40,17 @@ pub fn spawn + Send + 'static>(future: F) -> Result<(), E } pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { - let process = Process::current(); + let thread = Thread::current(); let mut future = Box::pin(future); loop { - let waker = waker_ref(&process); + let waker = waker_ref(&thread); let context = &mut Context::from_waker(&waker); match future.as_mut().poll(context) { Poll::Ready(value) => break Ok(value), Poll::Pending => { - if let Err(error) = process.suspend() { + if let Err(error) = thread.suspend() { break Err(error); } } diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs index ded29ee2..0fc0715d 100644 --- a/src/task/runtime/task_queue.rs +++ b/src/task/runtime/task_queue.rs @@ -6,7 +6,7 @@ use kernel_util::util::OneTimeInit; use crate::{ arch::{Architecture, ArchitectureImpl}, sync::IrqGuard, - task::process::Process, + task::thread::Thread, }; use super::task::Task; @@ -15,7 +15,7 @@ pub(super) static TASK_QUEUE: OneTimeInit = OneTimeInit::new(); pub(super) struct TaskQueue { // Queue of workers waiting for an item - pending_workers: ArrayQueue>, + pending_workers: ArrayQueue>, task_queue: ArrayQueue>, } @@ -44,19 +44,19 @@ impl TaskQueue { } pub fn dequeue(&self) -> Result, Error> { - let process = Process::current(); + let thread = Thread::current(); assert!(ArchitectureImpl::interrupt_mask()); loop { if let Some(task) = self.task_queue.pop() { return Ok(task); } - if self.pending_workers.push(process.clone()).is_err() { + if self.pending_workers.push(thread.clone()).is_err() { panic!("Pending worker queue overflow"); } // This must not fail. Signals must not be raised. - process.suspend().unwrap(); + thread.suspend().unwrap(); } } } diff --git a/src/task/sched.rs b/src/task/sched.rs index a3f97548..174a5520 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -1,7 +1,13 @@ //! Per-CPU queue implementation +use core::sync::atomic::Ordering; + // use aarch64_cpu::registers::CNTPCT_EL0; -use alloc::{collections::VecDeque, sync::Arc, vec::Vec}; +use alloc::{ + collections::{BTreeMap, VecDeque}, + sync::Arc, + vec::Vec, +}; use cfg_if::cfg_if; use kernel_util::util::OneTimeInit; @@ -9,12 +15,13 @@ use crate::{ // arch::aarch64::{context::TaskContext, cpu::Cpu}, arch::{Architecture, ArchitectureImpl}, sync::{IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard}, + task::thread::ThreadState, }; use super::{ context::TaskContextImpl, - process::{CurrentProcess, Process, ProcessState}, - Cpu, ProcessId, TaskContext, + thread::{Thread, ThreadId}, + Cpu, TaskContext, }; /// Per-CPU statistics @@ -29,15 +36,21 @@ pub struct CpuQueueStats { measure_time: u64, } -/// Per-CPU queue's inner data, normally resides under a lock -pub struct CpuQueueInner { - /// Current process, None if idling - pub current: Option>, - /// LIFO queue for processes waiting for execution - pub queue: VecDeque>, +// /// Per-CPU queue's inner data, normally resides under a lock +// pub struct CpuQueueInner { +// /// Current process, None if idling +// pub current: Option>, +// /// LIFO queue for processes waiting for execution +// pub queue: VecDeque>, +// +// /// CPU time usage statistics +// pub stats: CpuQueueStats, +// } - /// CPU time usage statistics - pub stats: CpuQueueStats, +struct CpuQueueInner { + current: Option, + queue: VecDeque, + stats: CpuQueueStats, } /// Per-CPU queue @@ -80,21 +93,22 @@ impl CpuQueueInner { /// Picks a next task for execution, skipping (dropping) those that were suspended. May return /// None if the queue is empty or no valid task was found, in which case the scheduler should /// go idle. - pub fn next_ready_task(&mut self) -> Option> { + pub fn next_ready_task(&mut self) -> Option> { #[allow(clippy::never_loop)] while !self.queue.is_empty() { let task = self.queue.pop_front().unwrap(); + let task_t = Thread::get(task).unwrap(); - match task.state() { - ProcessState::Ready => { - return Some(task); + match task_t.state.load(Ordering::Acquire) { + ThreadState::Ready => { + return Some(task_t); } e => panic!( - "Unexpected process state in CpuQueue: {:?} ({} {:?}, cpu_id={})", + "Unexpected thread state in CpuQueue: {:?} ({} {:?}, cpu_id={})", e, - task.id(), - task.name(), - task.cpu_id() + task_t.id(), + task_t.name(), + 1234 // task_t.cpu_id() ), } } @@ -102,15 +116,15 @@ impl CpuQueueInner { None } - /// Returns an iterator over all the processes in the queue plus the currently running process, + /// Returns an iterator over all the threads in the queue plus the currently running thread, /// if there is one. - pub fn iter(&self) -> impl Iterator> { + pub fn iter(&self) -> impl Iterator { Iterator::chain(self.queue.iter(), self.current.iter()) } } impl CpuQueue { - /// Constructs an empty queue with its own idle task + // /// Constructs an empty queue with its own idle task pub fn new(index: usize) -> Self { let idle = TaskContext::kernel(__idle, Cpu::local_id() as usize) .expect("Could not construct an idle task"); @@ -158,57 +172,66 @@ impl CpuQueue { // inner.stats.measure_time = t; let current = inner.current.clone(); + let current_t = current.and_then(Thread::get); - if let Some(current) = current.as_ref() { - if current.state() == ProcessState::Running { - current.set_state(ProcessState::Ready); - inner.queue.push_back(current.clone()); + if let Some(current_t) = current_t.as_ref() { + if current_t + .state + .compare_exchange( + ThreadState::Running, + ThreadState::Ready, + Ordering::SeqCst, + Ordering::Relaxed, + ) + .is_ok() + { + inner.queue.push_back(current_t.id()); + // inner.stats.cpu_time += delta; } - - // inner.stats.cpu_time += delta; - } else { - // inner.stats.idle_time += delta; } + // else + // inner.stats.idle_time += delta; - let next = inner.next_ready_task(); + let next_t = inner.next_ready_task(); + // let next_t = next.and_then(Thread::get); - inner.current = next.clone(); + inner.current = next_t.as_deref().map(Thread::id); // Can drop the lock, we hold current and next Arc's drop(inner); - let (from, _from_rc) = if let Some(current) = current.as_ref() { - (current.current_context(), Arc::strong_count(current)) + let (from, _from_rc) = if let Some(current_t) = current_t.as_ref() { + (current_t.context(), Arc::strong_count(current_t)) } else { (&self.idle, 0) }; - let (to, _to_rc) = if let Some(next) = next.as_ref() { - next.set_running(Cpu::local_id()); - (next.current_context(), Arc::strong_count(next)) + let (to, _to_rc) = if let Some(next_t) = next_t.as_ref() { + next_t.set_running(Cpu::local_id()); + (next_t.context(), Arc::strong_count(next_t)) } else { (&self.idle, 0) }; - // log_print_raw!(crate::debug::LogLevel::Info, "{}: ", Cpu::local_id()); - // if let Some(from) = current.as_ref() { - // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id(),); - // } else { - // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); - // } - - // log_print_raw!(crate::debug::LogLevel::Info, " -> "); - - // if let Some(to) = next.as_ref() { - // log_print_raw!(crate::debug::LogLevel::Info, "{}", to.id(),); - // } else { - // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); - // } - - // log_print_raw!(crate::debug::LogLevel::Info, "\n"); - assert!(ArchitectureImpl::interrupt_mask()); to.switch(from) + + // // log_print_raw!(crate::debug::LogLevel::Info, "{}: ", Cpu::local_id()); + // // if let Some(from) = current.as_ref() { + // // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id(),); + // // } else { + // // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); + // // } + + // // log_print_raw!(crate::debug::LogLevel::Info, " -> "); + + // // if let Some(to) = next.as_ref() { + // // log_print_raw!(crate::debug::LogLevel::Info, "{}", to.id(),); + // // } else { + // // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); + // // } + + // // log_print_raw!(crate::debug::LogLevel::Info, "\n"); } /// Pushes the process to the back of the execution queue. @@ -217,20 +240,20 @@ impl CpuQueue { /// /// Only meant to be called from Process impl. The function does not set any process accounting /// information, which may lead to invalid states. - pub unsafe fn enqueue(&self, p: Arc) { + pub unsafe fn enqueue(&self, tid: ThreadId) { let mut inner = self.inner.lock(); assert!(ArchitectureImpl::interrupt_mask()); - assert_eq!(p.state(), ProcessState::Ready); + // assert_eq!(p.state(), ProcessState::Ready); - inner.queue.push_back(p); + inner.queue.push_back(tid); } - /// Removes process with given PID from the exeuction queue. - pub fn dequeue(&self, pid: ProcessId) { + /// Removes thread with given TID from the exeuction queue. + pub fn dequeue(&self, tid: ThreadId) { assert!(ArchitectureImpl::interrupt_mask()); let mut inner = self.inner.lock(); - inner.queue.retain(|p| p.id() != pid) + inner.queue.retain(|&p| p != tid) } /// Returns the queue length at this moment. @@ -251,38 +274,42 @@ impl CpuQueue { self.inner.lock().queue.is_empty() } - /// Returns a safe reference to the inner data structure. - pub fn lock(&self) -> IrqSafeSpinlockGuard { - self.inner.lock() - } + // /// Returns a safe reference to the inner data structure. + // pub fn lock(&self) -> IrqSafeSpinlockGuard { + // self.inner.lock() + // } - /// Returns an unsafe reference to the queue. - /// - /// # Safety - /// - /// Only meant to be called to dump the queue contents when panicking. - #[allow(clippy::mut_from_ref)] - pub unsafe fn grab(&self) -> &mut CpuQueueInner { - self.inner.grab() - } + // /// Returns an unsafe reference to the queue. + // /// + // /// # Safety + // /// + // /// Only meant to be called to dump the queue contents when panicking. + // #[allow(clippy::mut_from_ref)] + // pub unsafe fn grab(&self) -> &mut CpuQueueInner { + // self.inner.grab() + // } + // + // /// Returns the process currently being executed. + // /// + // /// # Note + // /// + // /// This function should be safe in all kernel thread/interrupt contexts: + // /// + // /// * (in kthread) the code calling this will still remain on the same thread. + // /// * (in irq) the code cannot be interrupted and other CPUs shouldn't change this queue, so it + // /// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu] + // /// is called. + // pub fn current_process(&self) -> Option { + // let guard = IrqGuard::acquire(); + // self.inner + // .lock() + // .current + // .clone() + // .map(|p| unsafe { CurrentProcess::new(p, guard) }) + // } - /// Returns the process currently being executed. - /// - /// # Note - /// - /// This function should be safe in all kernel thread/interrupt contexts: - /// - /// * (in kthread) the code calling this will still remain on the same thread. - /// * (in irq) the code cannot be interrupted and other CPUs shouldn't change this queue, so it - /// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu] - /// is called. - pub fn current_process(&self) -> Option { - let guard = IrqGuard::acquire(); - self.inner - .lock() - .current - .clone() - .map(|p| unsafe { CurrentProcess::new(p, guard) }) + pub fn current_id(&self) -> Option { + self.inner.lock().current } /// Returns a queue for given CPU index diff --git a/src/task/thread.rs b/src/task/thread.rs new file mode 100644 index 00000000..19f47c14 --- /dev/null +++ b/src/task/thread.rs @@ -0,0 +1,430 @@ +use core::{ + fmt, + mem::size_of, + ops::Deref, + sync::atomic::{AtomicU32, AtomicU64, Ordering}, +}; + +use abi::{ + error::Error, + process::{ExitCode, Signal, SignalEntryData}, +}; +use alloc::{ + collections::{BTreeMap, VecDeque}, + string::String, + sync::Arc, +}; +use atomic_enum::atomic_enum; +use futures_util::task::ArcWake; +use kernel_util::util::OneTimeInit; + +use crate::{ + mem::{process::ProcessAddressSpace, ForeignPointer}, + sync::{IrqGuard, IrqSafeSpinlock}, + task::{context::TaskContextImpl, Cpu}, +}; + +use super::{context::TaskFrame, process::Process, sched::CpuQueue, TaskContext}; + +/// Represents the states a thread can be at some point in time +#[atomic_enum] +#[derive(PartialEq)] +pub enum ThreadState { + /// Thread is ready for execution and is present in some CPU's queue + Ready, + /// Thread is currently being executed by some CPU + Running, + /// Thread is present in a global list, but is not queued for execution until it is resumed + Suspended, + /// Thread is terminated and waits to be reaped + Terminated, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Ord, PartialOrd)] +pub enum ThreadId { + Kernel(u64), + User(u64), +} + +pub struct CurrentThread(Arc, IrqGuard); + +struct SignalEntry { + entry: usize, + stack: usize, +} + +struct ThreadInner { + exit_status: i32, + queue: Option<&'static CpuQueue>, + + signal_entry: Option, + signal_stack: VecDeque, +} + +pub struct Thread { + context: TaskContext, + name: Option, + id: ThreadId, + + cpu_id: AtomicU32, + pub(super) state: AtomicThreadState, + + process: Option>, + space: Option>, + + inner: IrqSafeSpinlock, +} + +static THREADS: IrqSafeSpinlock>> = + IrqSafeSpinlock::new(BTreeMap::new()); + +impl Thread { + // pub fn new(index: usize, parent: &Process, context: TaskContext) -> Self { + // todo!() + // } + + fn new( + id: ThreadId, + name: Option, + process: Option>, + space: Option>, + context: TaskContext, + ) -> Arc { + let thread = Arc::new(Self { + context, + name, + id, + + cpu_id: AtomicU32::new(0), + state: AtomicThreadState::new(ThreadState::Suspended), + + process, + space, + + inner: IrqSafeSpinlock::new(ThreadInner { + exit_status: 0, + queue: None, + + signal_stack: VecDeque::new(), + signal_entry: None, + }), + }); + + THREADS.lock().insert(id, thread.clone()); + + thread + } + + pub fn new_uthread( + parent: Arc, + space: Arc, + context: TaskContext, + ) -> Arc { + Self::new( + ThreadId::next_user(), + None, + Some(parent), + Some(space), + context, + ) + } + + pub fn new_kthread>(name: S, context: TaskContext) -> Arc { + Self::new( + ThreadId::next_kernel(), + Some(name.into()), + None, + None, + context, + ) + } + + // Info + pub fn id(&self) -> ThreadId { + self.id + } + + pub fn name(&self) -> Option<&String> { + self.name.as_ref() + } + + pub fn context(&self) -> &TaskContext { + &self.context + } + + pub fn address_space(&self) -> &Arc { + self.space.as_ref().unwrap() + } + + pub fn process(&self) -> &Arc { + self.process.as_ref().unwrap() + } + + // Queue operation + pub fn current() -> CurrentThread { + Self::get_current().unwrap() + } + + pub fn get_current() -> Option { + let guard = IrqGuard::acquire(); + Cpu::local() + .queue() + .current_id() + .and_then(Self::get) + .map(|t| unsafe { CurrentThread(t, guard) }) + } + + pub fn enqueue_somewhere(&self) { + // Doesn't have to be precise, so even if something changes, we can still be rebalanced + // to another CPU + let (index, queue) = CpuQueue::least_loaded().unwrap(); + + self.enqueue_to(queue); + + // index + } + + pub fn enqueue_to(&self, queue: &'static CpuQueue) { + let _irq = IrqGuard::acquire(); + + { + let mut inner = self.inner.lock(); + let old_queue = inner.queue.replace(queue); + if old_queue.is_some() { + // Already in some queue + return; + } + } + + match self.state.compare_exchange( + ThreadState::Suspended, + ThreadState::Ready, + Ordering::SeqCst, + Ordering::Relaxed, + ) { + Err(ThreadState::Terminated) => { + // Process might've been killed while `await`ing in a `block!` + debugln!( + "Thread {} {:?} already terminated, dropping", + self.id(), + self.name() + ); + } + Err(state) => { + todo!("Unexpected process state when enqueueing: {:?}", state) + } + Ok(_) => unsafe { + queue.enqueue(self.id); + }, + } + } + + // TODO maybe separate dequeue for current and "other" threads + fn dequeue(&self, new_state: ThreadState) { + let _irq = IrqGuard::acquire(); + assert_ne!(new_state, ThreadState::Ready); + assert_ne!(new_state, ThreadState::Running); + + let mut inner = self.inner.lock(); + let current_state = self.state.swap(new_state, Ordering::SeqCst); + let proc_queue = inner.queue.take().unwrap(); + + proc_queue.dequeue(self.id()); + + match current_state { + // NOTE: I'm not sure if the process could've been queued between the store and this + // but most likely not (if I'm not that bad with atomics) + // Do nothing, its queue will just drop the process + ThreadState::Ready => (), + // Do nothing, not in a queue already + ThreadState::Suspended => (), + ThreadState::Terminated => panic!("Thread is terminated"), + ThreadState::Running => { + let cpu_id = self.cpu_id.load(Ordering::Acquire); + let local_cpu_id = Cpu::local_id(); + let queue = Cpu::local().queue(); + + if cpu_id == local_cpu_id { + assert_eq!(queue as *const _, proc_queue as *const _, "Thread queue mismatch: process says cpu{}, queue {:p}, actual cpu{}, queue {:p}", cpu_id, proc_queue, local_cpu_id, queue); + drop(inner); + // Suspending a process running on local CPU + unsafe { queue.yield_cpu() } + } else { + todo!(); + } + } + } + } + + pub unsafe fn set_running(&self, cpu: u32) { + self.cpu_id.store(cpu, Ordering::Release); + self.state.store(ThreadState::Running, Ordering::Release); + } + + pub fn suspend(&self) { + self.dequeue(ThreadState::Suspended); + } + + // Accounting + pub fn get(id: ThreadId) -> Option> { + THREADS.lock().get(&id).cloned() + } + + // Thread inner + /// Handles the cleanup of an exited thread + pub fn handle_exit(&self) { + // Scheduler still holds a lock of this process? + // TODO cancel Wait if a process was killed while suspended? + let code = { + let inner = self.inner.lock(); + let exit_status = ExitCode::from(inner.exit_status); + exit_status + }; + + if let Some(process) = self.process.as_ref() { + process.handle_thread_exit(self.id(), code); + } + + // TODO WaitThread, notify any waiters we're done + // self.exit_waker.wake_all(); + } + + pub fn set_signal_entry(&self, entry: usize, stack: usize) { + let mut inner = self.inner.lock(); + inner.signal_entry.replace(SignalEntry { entry, stack }); + } + + pub fn raise_signal(self: &Arc, signal: Signal) { + self.inner.lock().signal_stack.push_back(signal); + + if self.state.load(Ordering::Acquire) == ThreadState::Suspended { + self.clone().enqueue_somewhere(); + } + } +} + +impl ArcWake for Thread { + fn wake_by_ref(arc_self: &Arc) { + arc_self.clone().enqueue_somewhere(); + } +} + +impl CurrentThread { + /// Terminate the current process + pub fn exit(&self, status: ExitCode) { + self.inner.lock().exit_status = status.into(); + debugln!("Thread {} exited with code {:?}", self.id(), status); + + self.handle_exit(); + self.dequeue(ThreadState::Terminated); + } + + pub fn suspend(&self) -> Result<(), Error> { + self.dequeue(ThreadState::Suspended); + + let inner = self.inner.lock(); + if !inner.signal_stack.is_empty() { + return Err(Error::Interrupted); + } + + Ok(()) + } + + /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. + /// + /// # Safety + /// + /// This function is only meant to be called right before returning from an userspace + /// exception handler. + pub unsafe fn handle_pending_signals(&self, frame: &mut F) { + let ThreadId::User(id) = self.id else { + return; + }; + + let mut inner = self.inner.lock(); + + if let Some(signal) = inner.signal_stack.pop_front() { + let Some(entry) = inner.signal_entry.as_ref() else { + todo!(); + }; + + // TODO check if really in a syscall, lol + let syscall_return = -(u32::from(Error::Interrupted) as isize); + frame.set_return_value(syscall_return as u64); + + // Setup signal frame + let usp = ((entry.stack - size_of::()) & !0xF) + - TaskContext::SIGNAL_STACK_EXTRA_ALIGN; + let frame_ptr = usp as *mut SignalEntryData; + + let saved_frame = frame.store(); + frame_ptr.write_foreign_volatile( + self.address_space(), + SignalEntryData { + signal, + frame: saved_frame, + }, + ); + + // Setup return to signal handler + debugln!( + "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", + entry.entry, + usp, + entry.stack + ); + + frame.set_user_sp(usp); + frame.set_user_ip(entry.entry); + + // Pass the frame pointer as an argument to signal handler entry + frame.set_argument(usp as _); + } + + // let process = self.process(); + + // if let Some((entry, signal)) = process.pop_signal() { + // debugln!( + // "{} of {}, enter signal handler from pc={:#x}, sp={:#x}", + // self.id, + // process.id(), + // frame.user_ip(), + // frame.user_sp() + // ); + + // } + } +} + +impl Deref for CurrentThread { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl ThreadId { + pub fn next_kernel() -> Self { + static COUNT: AtomicU64 = AtomicU64::new(1); + let id = COUNT.fetch_add(1, Ordering::SeqCst); + Self::Kernel(id) + } + + pub fn next_user() -> Self { + static COUNT: AtomicU64 = AtomicU64::new(1); + let id = COUNT.fetch_add(1, Ordering::SeqCst); + Self::User(id) + } +} + +impl fmt::Display for ThreadId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use fmt::Write; + + match self { + Self::Kernel(id) => write!(f, "#[{id}]"), + Self::User(id) => write!(f, "#{id}"), + } + } +} From 76d823541163e3d37eb5a1afb8f0c3296e4eb3c1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 21 Nov 2023 14:18:22 +0200 Subject: [PATCH 089/211] =?UTF-8?q?sys:=20Spawn=20=E2=86=92=20SpawnProcess?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syscall/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index d17eb0d6..39dd9b05 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -303,7 +303,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result }) } // Process management - SyscallFunction::Spawn => { + SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; run_with_io(&process, |mut io| { From 8bc8b30362e453fceea9a57cbb9d7f5eeefe0e93 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 23 Nov 2023 10:41:27 +0200 Subject: [PATCH 090/211] proc: Implement simple static TLS support --- src/arch/aarch64/context.rs | 16 ++- src/arch/x86_64/context.S | 23 ++- src/arch/x86_64/context.rs | 22 ++- src/proc/elf.rs | 270 ++++++++++++++++++++++++++++-------- src/proc/exec.rs | 37 +++-- src/syscall/mod.rs | 49 ++++--- src/task/context.rs | 8 +- src/task/process.rs | 124 ++++++++++++++++- src/task/thread.rs | 7 + 9 files changed, 448 insertions(+), 108 deletions(-) diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 9e1cd482..06f50d9f 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -58,9 +58,9 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, ttbr0: u64) { + fn init_common(&mut self, entry: usize, ttbr0: u64, tpidr_el0: u64) { self.push(ttbr0 as _); // ttbr0_el1 - self.push(0); // tpidr_el0 + self.push(tpidr_el0 as _); // tpidr_el0 self.push(entry); // x30/lr self.push(0); // x29 @@ -91,7 +91,7 @@ impl TaskContextImpl for TaskContext { stack.push(entry as _); stack.push(arg); - stack.init_common(__aarch64_task_enter_kernel as _, 0); + stack.init_common(__aarch64_task_enter_kernel as _, 0, 0); let sp = stack.build(); @@ -104,7 +104,13 @@ impl TaskContextImpl for TaskContext { }) } - fn user(entry: usize, arg: usize, ttbr0: u64, user_stack_sp: usize) -> Result { + fn user( + entry: usize, + arg: usize, + ttbr0: u64, + user_stack_sp: usize, + tpidr_el0: usize, + ) -> Result { const USER_TASK_PAGES: usize = 16; let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); @@ -115,7 +121,7 @@ impl TaskContextImpl for TaskContext { stack.push(0); stack.push(user_stack_sp); - stack.init_common(__aarch64_task_enter_user as _, ttbr0); + stack.init_common(__aarch64_task_enter_user as _, ttbr0, tpidr_el0 as _); let sp = stack.build(); diff --git a/src/arch/x86_64/context.S b/src/arch/x86_64/context.S index 1892e63e..d89ff421 100644 --- a/src/arch/x86_64/context.S +++ b/src/arch/x86_64/context.S @@ -1,5 +1,7 @@ // vi: set ft=asm : +.set MSR_IA32_FS_BASE, 0xC0000100 + .macro SAVE_TASK_STATE sub ${context_size}, %rsp @@ -8,6 +10,16 @@ mov %r13, 16(%rsp) mov %r14, 24(%rsp) mov %r15, 32(%rsp) + + // Store FS_BASE + mov $MSR_IA32_FS_BASE, %ecx + rdmsr + mov %edx, %ecx + shl $32, %rcx + or %rax, %rcx + + mov %rcx, 40(%rsp) + // TODO save %fs mov %rbp, 48(%rsp) @@ -24,7 +36,16 @@ mov 16(%rsp), %r13 mov 24(%rsp), %r14 mov 32(%rsp), %r15 - // TODO + + // Load FS_BASE + // edx:eax = fs_base + mov 40(%rsp), %rdx + mov %edx, %eax + shr $32, %rdx + + mov $MSR_IA32_FS_BASE, %ecx + wrmsr + // mov 40(%rsp), %fs mov 48(%rsp), %rbp diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 2c4458ea..430e9337 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -65,14 +65,14 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, cr3: u64) { + fn init_common(&mut self, entry: usize, cr3: u64, fs_base: usize) { self.push(entry); // address for ret // End of common context self.push(cr3 as _); // %cr3 self.push(0); // %rbp - self.push(0); // %fs (TODO) + self.push(fs_base); // fs_base self.push(0); // %r15 self.push(0); // %r14 self.push(0); // %r13 @@ -98,9 +98,11 @@ impl TaskContextImpl for TaskContext { stack.push(entry as _); stack.push(arg); - stack.init_common(__x86_64_task_enter_kernel as _, unsafe { - KERNEL_TABLES.as_physical_address().into_raw() - }); + stack.init_common( + __x86_64_task_enter_kernel as _, + unsafe { KERNEL_TABLES.as_physical_address().into_raw() }, + 0, + ); let sp = stack.build(); infoln!("Kernel stack {:#x}", sp); @@ -114,7 +116,13 @@ impl TaskContextImpl for TaskContext { }) } - fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result { + fn user( + entry: usize, + arg: usize, + cr3: u64, + user_stack_sp: usize, + fs_base: usize, + ) -> Result { const USER_TASK_PAGES: usize = 8; let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); @@ -125,7 +133,7 @@ impl TaskContextImpl for TaskContext { stack.push(arg); stack.push(user_stack_sp); - stack.init_common(__x86_64_task_enter_user as _, cr3); + stack.init_common(__x86_64_task_enter_user as _, cr3, fs_base); let sp = stack.build(); let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 06afd6aa..16ed50f2 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -2,15 +2,22 @@ use core::ops::DerefMut; use elf::{ - abi::{PF_W, PF_X, PT_LOAD}, + abi::{PF_W, PF_X, PT_LOAD, PT_TLS}, endian::AnyEndian, + segment::ProgramHeader, ElfStream, ParseError, }; use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; -use crate::mem::{ - phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, table::MapAttributes, +use crate::{ + mem::{ + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + process::ProcessAddressSpace, + table::{EntryLevel, MapAttributes}, + }, + task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; #[derive(Clone, Copy)] @@ -56,6 +63,54 @@ fn from_parse_error(v: ParseError) -> Error { Error::InvalidFile } +pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result { + let Some(tls) = image.tls.as_ref() else { + // No TLS + return Ok(0); + }; + + assert_ne!(tls.master_copy_base, 0); + assert_ne!(tls.layout.mem_size, 0); + + let address = space.allocate( + None, + 0x1000, + |_| phys::alloc_page(), + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, + )?; + + debugln!( + "Clone TLS from {:#x} (data={:#x}) to {:#x} (data={:#x})", + tls.master_copy_base, + tls.master_copy_base + tls.layout.data_offset, + address, + address + tls.layout.data_offset + ); + debugln!("tls ptr = {:#x}", address + tls.layout.ptr_offset); + + let src_phys = space.translate(tls.master_copy_base)?; + let dst_phys = space.translate(address)?; + + // Copy data + unsafe { + let src = + PhysicalRef::::map_slice(src_phys.add(tls.layout.data_offset), tls.layout.mem_size); + let mut dst = + PhysicalRefMut::map_slice(dst_phys.add(tls.layout.data_offset), tls.layout.mem_size); + + dst.copy_from_slice(&src); + } + + // Setup self-pointer + unsafe { + let mut dst = PhysicalRefMut::::map(dst_phys.add(tls.layout.ptr_offset)); + + *dst = address + tls.layout.ptr_offset; + } + + Ok(address + tls.layout.ptr_offset) +} + fn load_bytes( space: &ProcessAddressSpace, addr: usize, @@ -92,69 +147,162 @@ where Ok(()) } -/// Loads an ELF binary from `file` into the target address space -pub fn load_elf_from_file(space: &ProcessAddressSpace, file: FileRef) -> Result { - let file = FileReader { file: &file }; +fn load_segment( + space: &ProcessAddressSpace, + phdr: &ProgramHeader, + file: &FileReader, +) -> Result<(), Error> { + if phdr.p_memsz == 0 { + return Ok(()); + } + let attrs = match (phdr.p_flags & PF_W, phdr.p_flags & PF_X) { + (0, 0) => MapAttributes::USER_READ, + (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + (0, _) => MapAttributes::USER_READ, + (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + } | MapAttributes::NON_GLOBAL; + + // Map the range + let aligned_start = (phdr.p_vaddr as usize) & !0xFFF; + let aligned_end = ((phdr.p_vaddr + phdr.p_memsz) as usize + 0xFFF) & !0xFFF; + + space.map( + aligned_start, + aligned_end - aligned_start, + |_| phys::alloc_page(), + attrs, + )?; + + if phdr.p_filesz > 0 { + load_bytes( + space, + phdr.p_vaddr as usize, + |off, mut dst| { + let mut source = file.file.borrow_mut(); + source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + source.read_exact(dst.deref_mut()) + }, + phdr.p_filesz as usize, + )?; + } + + if phdr.p_memsz > phdr.p_filesz { + let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; + let len = (phdr.p_memsz - phdr.p_filesz) as usize; + + load_bytes( + space, + addr, + |_, mut dst| { + dst.fill(0); + Ok(()) + }, + len, + )?; + } + + Ok(()) +} + +fn tls_segment( + space: &ProcessAddressSpace, + phdr: &ProgramHeader, + file: &FileReader, +) -> Result { + assert_ne!(phdr.p_memsz, 0); + + if !phdr.p_align.is_power_of_two() { + return Err(Error::InvalidArgument); + } + + let layout = ProcessTlsLayout::new(phdr.p_align as _, phdr.p_filesz as _, phdr.p_memsz as _); + let data_offset = layout.data_offset; + let data_size = layout.data_size; + let mem_size = layout.mem_size; + let aligned_size = (layout.full_size + 0xFFF) & !0xFFF; + assert!(aligned_size <= 0x1000); + + let base_address = space.allocate( + None, + aligned_size, + |_| phys::alloc_page(), + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, + )?; + + debugln!( + "Allocated TLS master copy @ {:#x}, tls={:#x}, tls_data={:#x}", + base_address, + base_address + layout.ptr_offset, + base_address + layout.data_offset + ); + + let tls = ProcessTlsInfo { + master_copy_base: base_address, + layout, + }; + + if data_size > 0 { + load_bytes( + space, + base_address + data_offset, + |off, mut dst| { + let mut source = file.file.borrow_mut(); + source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + source.read_exact(dst.deref_mut()) + }, + data_size, + ); + } + + if mem_size > data_size { + load_bytes( + space, + base_address + data_offset + data_size, + |off, mut dst| { + dst.fill(0); + Ok(()) + }, + mem_size - data_size, + ); + } + + Ok(tls) +} + +/// Loads an ELF binary from `file` into the target address space +pub fn load_elf_from_file( + space: &ProcessAddressSpace, + file: FileRef, +) -> Result { + let file = FileReader { file: &file }; let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; + struct TlsInfo { + master_address: usize, + offset: usize, + size: usize, + } + + let mut tls = None; + for phdr in elf.segments() { - if phdr.p_type != PT_LOAD { - continue; - } - - debugln!("LOAD {:#x?}", phdr.p_vaddr..phdr.p_vaddr + phdr.p_memsz); - - let attrs = match (phdr.p_flags & PF_W, phdr.p_flags & PF_X) { - (0, 0) => MapAttributes::USER_READ, - (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, - (0, _) => MapAttributes::USER_READ, - (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, - } | MapAttributes::NON_GLOBAL; - - if phdr.p_memsz > 0 { - // Map the range - let aligned_start = (phdr.p_vaddr as usize) & !0xFFF; - let aligned_end = ((phdr.p_vaddr + phdr.p_memsz) as usize + 0xFFF) & !0xFFF; - - space.map( - aligned_start, - aligned_end - aligned_start, - |_| phys::alloc_page(), - attrs, - )?; - } else { - continue; - } - - if phdr.p_filesz > 0 { - load_bytes( - space, - phdr.p_vaddr as usize, - |off, mut dst| { - let mut source = file.file.borrow_mut(); - source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; - source.read_exact(dst.deref_mut()) - }, - phdr.p_filesz as usize, - )?; - } - - if phdr.p_memsz > phdr.p_filesz { - let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; - let len = (phdr.p_memsz - phdr.p_filesz) as usize; - - load_bytes( - space, - addr, - |_, mut dst| { - dst.fill(0); - Ok(()) - }, - len, - )?; + match phdr.p_type { + PT_LOAD => { + load_segment(space, phdr, &file)?; + } + PT_TLS => { + assert!(tls.is_none()); + tls.replace(tls_segment(space, phdr, &file)?); + // tls_master_address = tls_segment(space, phdr, &file)?; + // tls_size = phdr.p_memsz as usize; + } + _ => (), } } - Ok(elf.ehdr.e_entry as usize) + Ok(ProcessImage { + entry: elf.ehdr.e_entry as usize, + tls, + }) } diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 2ac0b8a5..f1566591 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -11,11 +11,19 @@ use vfs::FileRef; use crate::{ mem::{ - phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, table::MapAttributes, + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + process::ProcessAddressSpace, + table::MapAttributes, ForeignPointer, }, proc, - task::{context::TaskContextImpl, process::Process, thread::Thread, TaskContext}, + task::{ + context::TaskContextImpl, + process::{Process, ProcessImage}, + thread::Thread, + TaskContext, + }, }; pub struct BufferPlacer<'a> { @@ -99,7 +107,7 @@ fn setup_program_env( fn setup_binary>( name: S, space: ProcessAddressSpace, - entry: usize, + image: ProcessImage, args: &[&str], envs: &[&str], ) -> Result<(Arc, Arc), Error> { @@ -120,7 +128,7 @@ fn setup_binary>( debugln!( "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", - entry, + image.entry, virt_stack_base, virt_stack_base + USER_STACK_PAGES * 0x1000, virt_args_base @@ -138,9 +146,22 @@ fn setup_binary>( } } - let context = TaskContext::user(entry, arg, space.as_address_with_asid(), user_sp)?; + let tls_address = proc::elf::clone_tls(&space, &image)?; - Ok(Process::new_with_main(name, Arc::new(space), context)) + let context = TaskContext::user( + image.entry, + arg, + space.as_address_with_asid(), + user_sp, + tls_address, + )?; + + Ok(Process::new_with_main( + name, + Arc::new(space), + context, + Some(image), + )) // Ok(Process::new_with_context(name, Some(space), context)) } @@ -152,7 +173,7 @@ pub fn load_elf>( envs: &[&str], ) -> Result<(Arc, Arc), Error> { let space = ProcessAddressSpace::new()?; - let elf_entry = proc::elf::load_elf_from_file(&space, file)?; + let image = proc::elf::load_elf_from_file(&space, file)?; - setup_binary(name, space, elf_entry, args, envs) + setup_binary(name, space, image, args, envs) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 39dd9b05..fa2d8d20 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,7 +4,7 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, - process::{ExitCode, Signal, SpawnOption, SpawnOptions}, + process::{ExitCode, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; use alloc::rc::Rc; @@ -68,15 +68,14 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } SyscallFunction::Nanosleep => { - todo!(); - // let seconds = args[0]; - // let nanos = args[1] as u32; - // let duration = Duration::new(seconds, nanos); + let seconds = args[0]; + let nanos = args[1] as u32; + let duration = Duration::new(seconds, nanos); - // block! { - // runtime::sleep(duration).await - // } - // .map(|_| 0) + block! { + runtime::sleep(duration).await + } + .map(|_| 0) } // Resource management SyscallFunction::MapMemory => { @@ -361,6 +360,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(pid as _) }) } + SyscallFunction::SpawnThread => { + let options = arg_user_ref::(args[0] as usize)?; + let id = process.spawn_thread(options)?; + Ok(id.as_user() as _) + } SyscallFunction::Exit => { let code = ExitCode::from(args[0] as i32); // TODO separate handlers for process exit and thread exit? @@ -368,6 +372,19 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Process::current().exit(code); panic!(); } + SyscallFunction::SendSignal => { + let pid = ProcessId::from(args[0] as u32); + let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + + let target = Process::get(pid).ok_or(Error::DoesNotExist)?; + target.raise_signal(signal); + + Ok(0) + } + SyscallFunction::ExitSignal => { + panic!("Handled elsewhere"); + // Process::current().exit_signal(); + } SyscallFunction::GetPid => Ok(u32::from(process.id()) as usize), SyscallFunction::GetSessionId => todo!(), SyscallFunction::GetProcessGroupId => todo!(), @@ -419,20 +436,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } - _ => todo!("{:?}", func), - // SyscallFunction::SendSignal => { - // let pid = args[0] as u32; - // let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - - // let target = Process::get(pid as _).ok_or(Error::DoesNotExist)?; - // target.raise_signal(signal); - - // Ok(0) - // } - // SyscallFunction::ExitSignal => { - // panic!("Handled elsewhere"); - // // Process::current().exit_signal(); - // } } } diff --git a/src/task/context.rs b/src/task/context.rs index 0c011f96..c6c36c14 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -58,7 +58,13 @@ pub trait TaskContextImpl: Sized { /// Constructs a user thread context. The caller is responsible for allocating the userspace /// stack and setting up a valid address space for the context. - fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result; + fn user( + entry: usize, + arg: usize, + cr3: u64, + user_stack_sp: usize, + tls_address: usize, + ) -> Result; /// Performs an entry into a context. /// diff --git a/src/task/process.rs b/src/task/process.rs index 556d85e0..3e81ddb6 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -2,6 +2,7 @@ use core::{ fmt, + mem::size_of, pin::Pin, sync::atomic::{AtomicU64, Ordering}, task::{Context, Poll}, @@ -9,7 +10,7 @@ use core::{ use abi::{ error::Error, - process::{ExitCode, Signal}, + process::{ExitCode, Signal, ThreadSpawnOptions}, }; use alloc::{ collections::{BTreeMap, VecDeque}, @@ -21,7 +22,17 @@ use futures_util::Future; use kernel_util::util::OneTimeInit; use vfs::VnodeRef; -use crate::{mem::process::ProcessAddressSpace, proc::io::ProcessIo, sync::IrqSafeSpinlock}; +use crate::{ + mem::{ + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + process::ProcessAddressSpace, + table::MapAttributes, + }, + proc::{self, io::ProcessIo}, + sync::IrqSafeSpinlock, + task::context::TaskContextImpl, +}; use super::{ runtime::QueueWaker, @@ -39,6 +50,80 @@ pub enum ProcessState { #[repr(transparent)] pub struct ProcessId(u64); +// TLS layout (x86-64): +// | mem_size | uthread_size | +// | Data .......| self, ??? | +// +// TLS layout (aarch64): +// | uthread_size (0x10?) | mem_size | +// | ??? | Data .....| +#[derive(Debug)] +pub struct ProcessTlsInfo { + pub master_copy_base: usize, + pub layout: ProcessTlsLayout, +} + +#[derive(Debug)] +pub struct ProcessTlsLayout { + pub data_offset: usize, + pub uthread_offset: usize, + pub ptr_offset: usize, + + pub data_size: usize, + pub mem_size: usize, + + pub full_size: usize, +} + +#[cfg(target_arch = "aarch64")] +impl ProcessTlsLayout { + pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { + debug_assert!(align.is_power_of_two()); + let tls_block0_offset = (size_of::() * 2 + align - 1) & !(align - 1); + + let full_size = (tls_block0_offset + mem_size + align - 1) & !(align - 1); + + Self { + data_offset: tls_block0_offset, + uthread_offset: 0, + ptr_offset: 0, + + data_size, + mem_size, + full_size, + } + } +} + +#[cfg(target_arch = "x86_64")] +impl ProcessTlsLayout { + pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { + // The static TLS blocks are placed below TP + // TP points to the TCB + debug_assert!(align.is_power_of_two()); + let back_size = (mem_size + align - 1) & !(align - 1); + // Self-pointer + let forward_size = size_of::(); + + let full_size = back_size + forward_size; + + Self { + data_offset: 0, + uthread_offset: back_size, + ptr_offset: back_size, + + data_size, + mem_size, + full_size, + } + } +} + +pub struct ProcessImage { + pub entry: usize, + pub tls: Option, +} + struct ProcessInner { state: ProcessState, @@ -56,6 +141,7 @@ pub struct Process { space: Arc, inner: IrqSafeSpinlock, + image: Option, exit_waker: QueueWaker, pub io: IrqSafeSpinlock, @@ -69,6 +155,7 @@ impl Process { name: S, space: Arc, context: TaskContext, + image: Option, ) -> (Arc, Arc) { let name = name.into(); let id = ProcessId::next(); @@ -77,6 +164,8 @@ impl Process { name, id, + image, + space: space.clone(), inner: IrqSafeSpinlock::new(ProcessInner { state: ProcessState::Running, @@ -99,6 +188,37 @@ impl Process { (process, thread) } + pub fn spawn_thread(self: &Arc, options: &ThreadSpawnOptions) -> Result { + debugln!( + "Spawn thread in {} with options: {:#x?}", + self.id(), + options + ); + + let tls_address = if let Some(image) = self.image.as_ref() { + proc::elf::clone_tls(&self.space, image)? + } else { + 0 + }; + + let space = self.space.clone(); + let context = TaskContext::user( + options.entry as _, + options.argument as _, + space.as_address_with_asid(), + options.stack_top, + tls_address, + )?; + let thread = Thread::new_uthread(self.clone(), space, context); + let id = thread.id(); + + self.inner.lock().threads.push(thread.clone()); + + thread.enqueue_somewhere(); + + Ok(id) + } + pub fn id(&self) -> ProcessId { self.id } diff --git a/src/task/thread.rs b/src/task/thread.rs index 19f47c14..ec52067e 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -416,6 +416,13 @@ impl ThreadId { let id = COUNT.fetch_add(1, Ordering::SeqCst); Self::User(id) } + + pub fn as_user(&self) -> u64 { + match self { + Self::Kernel(_) => panic!(), + &Self::User(id) => id, + } + } } impl fmt::Display for ThreadId { From 307f14678f2f539e540c3729307ba5edb1096710 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 24 Nov 2023 13:30:26 +0200 Subject: [PATCH 091/211] proc: add userspace mutex system call --- src/proc/elf.rs | 8 ++---- src/syscall/mod.rs | 18 +++++++++++- src/task/mod.rs | 1 + src/task/process.rs | 49 +++++++++------------------------ src/task/sync.rs | 67 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 42 deletions(-) create mode 100644 src/task/sync.rs diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 16ed50f2..a3c6820b 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -15,7 +15,7 @@ use crate::{ phys, pointer::{PhysicalRef, PhysicalRefMut}, process::ProcessAddressSpace, - table::{EntryLevel, MapAttributes}, + table::MapAttributes, }, task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; @@ -252,7 +252,7 @@ fn tls_segment( source.read_exact(dst.deref_mut()) }, data_size, - ); + )?; } if mem_size > data_size { @@ -264,7 +264,7 @@ fn tls_segment( Ok(()) }, mem_size - data_size, - ); + )?; } Ok(tls) @@ -294,8 +294,6 @@ pub fn load_elf_from_file( PT_TLS => { assert!(tls.is_none()); tls.replace(tls_segment(space, phdr, &file)?); - // tls_master_address = tls_segment(space, phdr, &file)?; - // tls_size = phdr.p_memsz as usize; } _ => (), } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index fa2d8d20..a3fe19e4 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,7 +4,7 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, - process::{ExitCode, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, + process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; use alloc::rc::Rc; @@ -365,6 +365,22 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let id = process.spawn_thread(options)?; Ok(id.as_user() as _) } + SyscallFunction::Mutex => { + let address = args[0] as usize; + let op = arg_user_ref::(args[1] as usize)?; + + let mutex = process.get_or_insert_mutex(address); + + match op { + &MutexOperation::Wait(value, _timeout) => { + block! { mutex.wait(value).await }.unwrap() + } + MutexOperation::Wake => mutex.wake(), + MutexOperation::WakeAll => mutex.wake_all(), + } + + Ok(0) + } SyscallFunction::Exit => { let code = ExitCode::from(args[0] as i32); // TODO separate handlers for process exit and thread exit? diff --git a/src/task/mod.rs b/src/task/mod.rs index 0a71c811..f472bdc6 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -17,6 +17,7 @@ pub mod context; pub mod process; pub mod runtime; pub mod sched; +pub mod sync; pub mod thread; pub use context::{Cpu, TaskContext}; diff --git a/src/task/process.rs b/src/task/process.rs index 3e81ddb6..5bae9a4a 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -4,7 +4,7 @@ use core::{ fmt, mem::size_of, pin::Pin, - sync::atomic::{AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, task::{Context, Poll}, }; @@ -36,6 +36,7 @@ use crate::{ use super::{ runtime::QueueWaker, + sync::UserspaceMutex, thread::{Thread, ThreadId, ThreadState}, TaskContext, }; @@ -133,6 +134,7 @@ struct ProcessInner { session_terminal: Option, threads: Vec>, + mutexes: BTreeMap>, } pub struct Process { @@ -173,6 +175,7 @@ impl Process { group_id: id, session_terminal: None, threads: Vec::new(), + mutexes: BTreeMap::new(), }), exit_waker: QueueWaker::new(), @@ -322,47 +325,12 @@ impl Process { self.exit_waker.wake_all(); } - // // Scheduler still holds a lock of this process? - // // TODO cancel Wait if a process was killed while suspended? - // { - // let inner = self.inner.lock(); - // let exit_status = ExitCode::from(inner.exit_status); - // debugln!( - // "Handling exit of #{} with status {:?}", - // self.id(), - // exit_status - // ); - // - // // TODO cleanup address space - // // if let Some(space) = self.get_address_space() { - // // } - // - // self.io.lock().handle_exit(); - // } - // - // // Notify any waiters we're done - // self.exit_waker.wake_all(); } /// Raises a signal for the specified process pub fn raise_signal(self: &Arc, signal: Signal) { let thread = self.inner.lock().threads[0].clone(); thread.raise_signal(signal); - - // // TODO if the process does not have any running/ready threads, pick one and wake it up - // if inner - // .threads - // .iter() - // .all(|t| t.state.load(Ordering::Acquire) == ThreadState::Suspended) - // { - // let thread = inner.threads[0].clone(); - - // drop(inner); - - // thread.enqueue_somewhere(); - - // return; - // } } /// Raises a signal for the specified process group @@ -378,6 +346,15 @@ impl Process { } } + pub fn get_or_insert_mutex(&self, address: usize) -> Arc { + let mut inner = self.inner.lock(); + inner + .mutexes + .entry(address) + .or_insert_with(|| Arc::new(UserspaceMutex::new(address))) + .clone() + } + // Process list pub fn get(id: ProcessId) -> Option> { PROCESSES.lock().get(&id).cloned() diff --git a/src/task/sync.rs b/src/task/sync.rs new file mode 100644 index 00000000..a0baa1a9 --- /dev/null +++ b/src/task/sync.rs @@ -0,0 +1,67 @@ +use core::{ + pin::Pin, + sync::atomic::{AtomicU32, Ordering}, + task::{Context, Poll}, +}; + +use alloc::sync::Arc; +use futures_util::Future; + +use super::runtime::QueueWaker; + +pub struct UserspaceMutex { + queue: QueueWaker, + address: usize, +} + +impl UserspaceMutex { + pub fn new(address: usize) -> Self { + Self { + queue: QueueWaker::new(), + address, + } + } + + pub fn wait(self: Arc, compare_value: u32) -> impl Future { + struct WaitFuture { + mutex: Arc, + compare_value: u32, + } + + impl WaitFuture { + fn load(&self, ordering: Ordering) -> u32 { + let value = unsafe { &*(self.mutex.address as *const AtomicU32) }; + value.load(ordering) + } + } + + impl Future for WaitFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.mutex.queue.register(cx.waker()); + + // Test the mutex + if self.load(Ordering::Acquire) != self.compare_value { + self.mutex.queue.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + } + + WaitFuture { + mutex: self, + compare_value, + } + } + + pub fn wake(&self) { + self.queue.wake_one(); + } + + pub fn wake_all(&self) { + self.queue.wake_all(); + } +} From dbbddaa4f504e273ad07b7f516630afad898a981 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 24 Nov 2023 14:40:37 +0200 Subject: [PATCH 092/211] fs: add a basic sysfs --- lib/memfs/src/dir.rs | 10 ++- lib/memfs/src/file.rs | 21 +++-- lib/vfs/src/char.rs | 19 ++++- lib/vfs/src/file.rs | 16 ++-- lib/vfs/src/node.rs | 31 +++++-- src/fs/devfs.rs | 13 ++- src/fs/mod.rs | 2 + src/fs/sysfs.rs | 188 ++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 4 + src/util/mod.rs | 11 +++ 10 files changed, 289 insertions(+), 26 deletions(-) create mode 100644 src/fs/sysfs.rs diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs index 024662f2..421a7908 100644 --- a/lib/memfs/src/dir.rs +++ b/lib/memfs/src/dir.rs @@ -1,4 +1,4 @@ -use core::{cell::RefCell, marker::PhantomData}; +use core::{any::Any, cell::RefCell, marker::PhantomData}; use alloc::boxed::Box; @@ -39,8 +39,12 @@ impl VnodeImpl for DirectoryNode { Ok(child) } - fn open(&self, _node: &VnodeRef, _opts: OpenOptions) -> Result { - Ok(DIR_POSITION_FROM_CACHE) + fn open( + &self, + _node: &VnodeRef, + _opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Ok((DIR_POSITION_FROM_CACHE, None)) } fn remove(&self, _at: &VnodeRef, _name: &str) -> Result<(), Error> { diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs index c8f7c436..8fe15a13 100644 --- a/lib/memfs/src/file.rs +++ b/lib/memfs/src/file.rs @@ -1,5 +1,6 @@ -use core::cell::RefCell; +use core::{any::Any, cell::RefCell}; +use alloc::boxed::Box; use vfs::{VnodeImpl, VnodeRef}; use yggdrasil_abi::{ error::Error, @@ -16,11 +17,15 @@ pub(crate) struct FileNode { } impl VnodeImpl for FileNode { - fn open(&self, _node: &VnodeRef, opts: OpenOptions) -> Result { + fn open( + &self, + _node: &VnodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { if opts.contains(OpenOptions::APPEND) { - Ok(self.data.borrow().size() as u64) + Ok((self.data.borrow().size() as u64, None)) } else { - Ok(0) + Ok((0, None)) } } @@ -28,7 +33,13 @@ impl VnodeImpl for FileNode { Ok(()) } - fn read(&self, _node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + fn read( + &self, + _node: &VnodeRef, + pos: u64, + _inner: Option<&mut Box>, + data: &mut [u8], + ) -> Result { self.data.borrow().read(pos, data) } diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs index 2ca3853f..abf98923 100644 --- a/lib/vfs/src/char.rs +++ b/lib/vfs/src/char.rs @@ -1,3 +1,6 @@ +use core::any::Any; + +use alloc::boxed::Box; use yggdrasil_abi::{ error::Error, io::{DeviceRequest, FileAttr, FileMode, FileType, OpenOptions}, @@ -23,15 +26,25 @@ impl CharDeviceWrapper { } impl VnodeImpl for CharDeviceWrapper { - fn open(&self, _node: &VnodeRef, _opts: OpenOptions) -> Result { - Ok(0) + fn open( + &self, + _node: &VnodeRef, + _opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Ok((0, None)) } fn close(&self, _node: &VnodeRef) -> Result<(), Error> { Ok(()) } - fn read(&self, _node: &VnodeRef, _pos: u64, data: &mut [u8]) -> Result { + fn read( + &self, + _node: &VnodeRef, + _pos: u64, + _inner: Option<&mut Box>, + data: &mut [u8], + ) -> Result { self.device.read(true, data) } diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs index 2c8c4cc2..51fe0b25 100644 --- a/lib/vfs/src/file.rs +++ b/lib/vfs/src/file.rs @@ -1,6 +1,6 @@ -use core::{cell::RefCell, mem::MaybeUninit}; +use core::{any::Any, cell::RefCell, mem::MaybeUninit}; -use alloc::rc::Rc; +use alloc::{boxed::Box, rc::Rc}; use bitflags::bitflags; use yggdrasil_abi::{ error::Error, @@ -34,6 +34,7 @@ enum DirectoryPosition { pub struct NormalFile { vnode: VnodeRef, + data: Option>, pos: u64, } @@ -53,9 +54,14 @@ pub struct File { } impl File { - pub fn normal(vnode: VnodeRef, pos: u64, flags: FileFlags) -> FileRef { + pub fn normal( + vnode: VnodeRef, + pos: u64, + data: Option>, + flags: FileFlags, + ) -> FileRef { Rc::new(RefCell::new(Self { - inner: FileInner::Normal(NormalFile { vnode, pos }), + inner: FileInner::Normal(NormalFile { vnode, pos, data }), flags, })) } @@ -109,7 +115,7 @@ impl Read for File { match &mut self.inner { FileInner::Normal(inner) => { - let count = inner.vnode.read(inner.pos, data)?; + let count = inner.vnode.read(inner.pos, inner.data.as_mut(), data)?; if inner.vnode.kind() != VnodeKind::Char { inner.pos += count as u64; } diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs index 78bc6ac5..547d19ee 100644 --- a/lib/vfs/src/node.rs +++ b/lib/vfs/src/node.rs @@ -1,4 +1,5 @@ use core::{ + any::Any, cell::{Ref, RefCell, RefMut}, fmt, }; @@ -68,7 +69,11 @@ pub trait VnodeImpl { Err(Error::NotImplemented) } - fn open(&self, node: &VnodeRef, opts: OpenOptions) -> Result { + fn open( + &self, + node: &VnodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { Err(Error::NotImplemented) } @@ -76,7 +81,13 @@ pub trait VnodeImpl { Err(Error::NotImplemented) } - fn read(&self, node: &VnodeRef, pos: u64, data: &mut [u8]) -> Result { + fn read( + &self, + node: &VnodeRef, + pos: u64, + inner: Option<&mut Box>, + data: &mut [u8], + ) -> Result { Err(Error::NotImplemented) } @@ -288,8 +299,8 @@ impl Vnode { } if let Some(data) = self.data() { - let pos = data.open(self, flags)?; - Ok(File::normal(self.clone(), pos, open_flags)) + let (pos, inner) = data.open(self, flags)?; + Ok(File::normal(self.clone(), pos, inner, open_flags)) } else { todo!() } @@ -301,7 +312,8 @@ impl Vnode { } if let Some(data) = self.data() { - let pos = data.open(self, OpenOptions::READ)?; + // TODO don't discard directory's Box + let (pos, _) = data.open(self, OpenOptions::READ)?; Ok(File::directory(self.clone(), pos)) } else { // TODO: some options here? @@ -386,13 +398,18 @@ impl Vnode { } } - pub fn read(self: &VnodeRef, pos: u64, buf: &mut [u8]) -> Result { + pub fn read( + self: &VnodeRef, + pos: u64, + inner: Option<&mut Box>, + buf: &mut [u8], + ) -> Result { if self.kind == VnodeKind::Directory { todo!(); } if let Some(data) = self.data() { - data.read(self, pos, buf) + data.read(self, pos, inner, buf) } else { todo!() } diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index 0dba0878..279a82de 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -1,5 +1,8 @@ //! Device virtual file system -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::{ + any::Any, + sync::atomic::{AtomicUsize, Ordering}, +}; use abi::{ error::Error, @@ -23,8 +26,12 @@ pub enum CharDeviceType { struct DevfsDirectory; impl VnodeImpl for DevfsDirectory { - fn open(&self, _node: &VnodeRef, _opts: OpenOptions) -> Result { - Ok(DIR_POSITION_FROM_CACHE) + fn open( + &self, + _node: &VnodeRef, + _opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Ok((DIR_POSITION_FROM_CACHE, None)) } fn metadata(&self, _node: &VnodeRef) -> Result { diff --git a/src/fs/mod.rs b/src/fs/mod.rs index ac6107cc..f0cd4033 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -10,6 +10,7 @@ use yggdrasil_abi::{error::Error, io::MountOptions}; use crate::mem::{phys, PhysicalAddress}; pub mod devfs; +pub mod sysfs; /// Describes in-memory filesystem image used as initial root pub struct Initrd { @@ -48,6 +49,7 @@ pub fn create_filesystem(options: &MountOptions) -> Result { match fs_name { "devfs" => Ok(devfs::root().clone()), + "sysfs" => Ok(sysfs::root().clone()), _ => todo!(), } } diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs new file mode 100644 index 00000000..94b7194e --- /dev/null +++ b/src/fs/sysfs.rs @@ -0,0 +1,188 @@ +use core::{any::Any, marker::PhantomData}; + +use abi::{ + error::Error, + io::{FileAttr, FileMode, FileType, OpenOptions}, +}; +use alloc::{ + boxed::Box, + string::{String, ToString}, +}; +use git_version::git_version; +use kernel_util::util::OneTimeInit; +use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; + +use crate::util; + +trait GetterFn = Fn() -> Result; +trait ReaderFn = Fn(u64, &mut [u8]) -> Result; + +struct SysfsDirectory; + +struct SysfsGetterNode> { + getter: R, + _pd: PhantomData, +} + +struct SysfsReaderNode { + reader: R, +} + +impl> SysfsGetterNode { + pub const fn new(getter: R) -> Self { + Self { + getter, + _pd: PhantomData, + } + } +} + +impl SysfsReaderNode { + pub const fn new(reader: R) -> Self { + Self { reader } + } +} + +impl VnodeImpl for SysfsDirectory { + fn open( + &self, + _node: &VnodeRef, + _opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Ok((DIR_POSITION_FROM_CACHE, None)) + } + + fn metadata(&self, _node: &VnodeRef) -> Result { + Ok(FileAttr { + size: 0, + // TODO mutable directory mode + mode: FileMode::from(0o755), + ty: FileType::Directory, + }) + } +} + +impl> VnodeImpl for SysfsGetterNode { + fn open( + &self, + _node: &VnodeRef, + _opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + let value = (self.getter)()?; + let inner = Box::new(value.to_string()); + Ok((0, Some(inner))) + } + + fn read( + &self, + node: &VnodeRef, + pos: u64, + inner: Option<&mut Box>, + data: &mut [u8], + ) -> Result { + let string = inner.unwrap().downcast_ref::().unwrap(); + let pos = pos as usize; + let bytes = string.as_bytes(); + + if pos >= bytes.len() { + return Ok(0); + } + + let len = core::cmp::min(bytes.len() - pos, data.len()); + + data[..len].copy_from_slice(&bytes[pos..pos + len]); + + Ok(len) + } + + fn size(&self, node: &VnodeRef) -> Result { + Ok(0) + } + + fn metadata(&self, node: &VnodeRef) -> Result { + Ok(FileAttr { + size: 0, + mode: FileMode::from(0o444), + ty: FileType::File, + }) + } +} + +impl VnodeImpl for SysfsReaderNode { + fn open( + &self, + node: &VnodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Ok((0, None)) + } + + fn read( + &self, + node: &VnodeRef, + pos: u64, + _inner: Option<&mut Box>, + data: &mut [u8], + ) -> Result { + (self.reader)(pos, data) + } + + fn size(&self, node: &VnodeRef) -> Result { + todo!() + } + + fn metadata(&self, node: &VnodeRef) -> Result { + todo!() + } +} + +static ROOT: OneTimeInit = OneTimeInit::new(); + +fn getter, T: ToString + 'static, R: GetterFn + 'static>( + name: S, + get: R, +) -> VnodeRef { + let data = Box::new(SysfsGetterNode::new(get)); + let node = Vnode::new(name, VnodeKind::Regular); + node.set_data(data); + node +} + +fn reader, R: ReaderFn + 'static>(name: S, read: R) -> VnodeRef { + let data = Box::new(SysfsReaderNode::new(read)); + let node = Vnode::new(name, VnodeKind::Regular); + node.set_data(data); + node +} + +fn dir, I: IntoIterator>(name: S, items: I) -> VnodeRef { + let node = Vnode::new(name, VnodeKind::Directory); + node.set_data(Box::new(SysfsDirectory)); + for item in items { + node.add_child(item); + } + node +} + +pub fn root() -> &'static VnodeRef { + ROOT.get() +} + +fn read_kernel_log(pos: u64, buffer: &mut [u8]) -> Result { + // TODO actual kernel log buffer + todo!() +} + +pub fn init() { + let kernel = dir( + "kernel", + [ + getter("version", || Ok(env!("CARGO_PKG_VERSION"))), + getter("rev", || Ok(git_version!())), + reader("log", read_kernel_log), + ], + ); + let root = dir("", [kernel, getter("arch", || Ok(util::arch_str()))]); + + ROOT.init(root); +} diff --git a/src/main.rs b/src/main.rs index bd3f75e0..c4d7df49 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,6 +34,7 @@ use arch::Architecture; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, + fs::sysfs, mem::heap, sync::SpinFence, task::{spawn_kernel_closure, Cpu}, @@ -87,6 +88,9 @@ pub fn kernel_secondary_main() -> ! { pub fn kernel_main() -> ! { debugln!("Heap: {:#x?}", heap::heap_range()); + // Setup the sysfs + sysfs::init(); + unsafe { ARCHITECTURE.start_application_processors(); } diff --git a/src/util/mod.rs b/src/util/mod.rs index e7a12c05..1ee02742 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -18,6 +18,17 @@ impl>> ResultIterator for I { } } +pub const fn arch_str() -> &'static str { + #[cfg(target_arch = "aarch64")] + { + "aarch64" + } + #[cfg(target_arch = "x86_64")] + { + "x86_64" + } +} + // /// Performs a busy-loop sleep until the specified duration has passed // pub fn polling_sleep(duration: Duration) -> Result<(), Error> { // // TODO no non-IRQ mode timestamp provider From 0dc2cfa159c7e32d87334e22d5b9fdcddba70367 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 26 Nov 2023 13:28:50 +0200 Subject: [PATCH 093/211] fs: simple kernel log buffer --- src/debug.rs | 55 ++++++++++++++++++++++++++++++++++++-- src/device/serial/pl011.rs | 2 +- src/fs/sysfs.rs | 5 ++-- src/util/ring.rs | 45 ++++++++++++++++++++++--------- 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/src/debug.rs b/src/debug.rs index 9ba51389..34062f6c 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,14 +2,30 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use kernel_util::util::StaticVector; +use alloc::sync::Arc; +use futures_util::Future; +use kernel_util::util::{OneTimeInit, StaticVector}; -use crate::{sync::IrqSafeSpinlock, task::process::Process}; +use crate::{ + sync::IrqSafeSpinlock, + task::{process::Process, runtime::QueueWaker}, + util::ring::RingBuffer, +}; const MAX_DEBUG_SINKS: usize = 4; +const RING_LOGGER_CAPACITY: usize = 65536; struct SimpleLogger; +struct RingLoggerInner { + data: RingBuffer, +} + +pub struct RingLoggerSink { + inner: IrqSafeSpinlock, + waker: QueueWaker, +} + /// Defines the severity of the message #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum LogLevel { @@ -151,11 +167,44 @@ impl fmt::Write for DebugSinkWrapper { } } +impl RingLoggerSink { + pub const fn new() -> Self { + Self { + inner: IrqSafeSpinlock::new(RingLoggerInner { + data: RingBuffer::new(0), + }), + waker: QueueWaker::new(), + } + } + + pub fn read(&self, pos: usize, buffer: &mut [u8]) -> usize { + unsafe { self.inner.lock().data.read_all_static(pos, buffer) } + } + + fn write_fmt(&self, args: fmt::Arguments<'_>) -> fmt::Result { + use fmt::Write; + self.inner.lock().write_fmt(args) + } +} + +impl fmt::Write for RingLoggerInner { + fn write_str(&mut self, s: &str) -> fmt::Result { + for ch in s.bytes() { + unsafe { + self.data.write_unchecked(ch); + } + } + Ok(()) + } +} + static LOGGER: SimpleLogger = SimpleLogger; static DEBUG_SINKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(StaticVector::new()); +pub static RING_LOGGER_SINK: RingLoggerSink = RingLoggerSink::new(); + /// Prints a hex-dump of a slice, appending a virtual address offset to the output pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { for (i, b) in data.iter().enumerate() { @@ -208,6 +257,8 @@ pub fn init() { pub fn debug_internal(args: Arguments, level: LogLevel) { use fmt::Write; + RING_LOGGER_SINK.write_fmt(args).ok(); + for sink in DEBUG_SINKS.lock().iter_mut() { if level < sink.level { continue; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 22136890..8fe55aac 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -192,7 +192,7 @@ impl Device for Pl011 { self.inner.init(IrqSafeSpinlock::new(inner)); - debug::add_sink(self, LogLevel::Debug); + // debug::add_sink(self, LogLevel::Debug); devfs::add_char_device(self, CharDeviceType::TtySerial)?; Ok(()) diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 94b7194e..994574fe 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -12,7 +12,7 @@ use git_version::git_version; use kernel_util::util::OneTimeInit; use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; -use crate::util; +use crate::{debug, util}; trait GetterFn = Fn() -> Result; trait ReaderFn = Fn(u64, &mut [u8]) -> Result; @@ -169,8 +169,7 @@ pub fn root() -> &'static VnodeRef { } fn read_kernel_log(pos: u64, buffer: &mut [u8]) -> Result { - // TODO actual kernel log buffer - todo!() + Ok(debug::RING_LOGGER_SINK.read(pos as usize, buffer)) } pub fn init() { diff --git a/src/util/ring.rs b/src/util/ring.rs index 9f903b04..41205e47 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -9,24 +9,36 @@ use futures_util::Future; use crate::{sync::IrqSafeSpinlock, task::runtime::QueueWaker}; -struct Inner { +pub struct RingBuffer { rd: usize, wr: usize, data: [T; N], } pub struct AsyncRing { - inner: Arc>>, + inner: Arc>>, read_waker: Arc, } -impl Inner { +impl RingBuffer { + pub const fn new(value: T) -> Self { + Self { + rd: 0, + wr: 0, + data: [value; N], + } + } + #[inline] const fn is_readable(&self) -> bool { - if self.rd <= self.wr { - (self.wr - self.rd) > 0 + self.is_readable_at(self.rd) + } + + const fn is_readable_at(&self, at: usize) -> bool { + if at <= self.wr { + (self.wr - at) > 0 } else { - (self.wr + (N - self.rd)) > 0 + (self.wr + (N - at)) > 0 } } @@ -37,8 +49,19 @@ impl Inner { res } + pub unsafe fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize { + let mut pos = (self.rd + pos) % N; + let mut off = 0; + while off < buffer.len() && self.is_readable_at(pos) { + buffer[off] = self.data[pos]; + pos += 1; + off += 1; + } + off + } + #[inline] - unsafe fn write_unchecked(&mut self, ch: T) { + pub unsafe fn write_unchecked(&mut self, ch: T) { self.data[self.wr] = ch; self.wr = (self.wr + 1) % N; } @@ -47,11 +70,7 @@ impl Inner { impl AsyncRing { pub fn new(value: T) -> Self { Self { - inner: Arc::new(IrqSafeSpinlock::new(Inner { - rd: 0, - wr: 0, - data: [value; N], - })), + inner: Arc::new(IrqSafeSpinlock::new(RingBuffer::new(value))), read_waker: Arc::new(QueueWaker::new()), } } @@ -69,7 +88,7 @@ impl AsyncRing { pub fn read(&self) -> impl Future { struct ReadFuture { - inner: Arc>>, + inner: Arc>>, read_waker: Arc, } From 6eef11a4e488494f3a159463e25cefd8336b4b6e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 26 Nov 2023 15:05:16 +0200 Subject: [PATCH 094/211] proc: separate thread and process exit --- src/device/serial/pl011.rs | 2 +- src/init.rs | 39 +++++------- src/syscall/mod.rs | 14 +++-- src/task/context.rs | 45 ++++++++++++-- src/task/mod.rs | 7 ++- src/task/process.rs | 15 +++++ src/task/sched.rs | 4 ++ src/task/thread.rs | 119 ++++++++++++++++++++++++++++--------- 8 files changed, 180 insertions(+), 65 deletions(-) diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 8fe55aac..22136890 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -192,7 +192,7 @@ impl Device for Pl011 { self.inner.init(IrqSafeSpinlock::new(inner)); - // debug::add_sink(self, LogLevel::Debug); + debug::add_sink(self, LogLevel::Debug); devfs::add_char_device(self, CharDeviceType::TtySerial)?; Ok(()) diff --git a/src/init.rs b/src/init.rs index dfdcdbdc..37bc02cc 100644 --- a/src/init.rs +++ b/src/init.rs @@ -1,7 +1,7 @@ //! Kernel main process implementation: filesystem initialization and userspace init start use abi::{ error::Error, - io::{FileMode, OpenOptions, RawFd}, + io::{OpenOptions, RawFd}, }; use memfs::MemoryFilesystem; use vfs::{Filesystem, IoContext, VnodeRef}; @@ -23,7 +23,7 @@ fn setup_root() -> Result { /// /// This function is meant to be used as a kernel-space process after all the platform-specific /// initialization has finished. -pub fn kinit() { +pub fn kinit() -> Result<(), Error> { infoln!("In main"); #[cfg(feature = "fb_console")] @@ -32,46 +32,39 @@ pub fn kinit() { runtime::spawn(async move { update_consoles_task().await; - }) - .expect("Could not start periodic console auto-flush task"); + })?; } - let root = match setup_root() { - Ok(root) => root, - Err(err) => { - warnln!("Could not setup root from initrd: {:?}", err); - return; - } - }; + let root = setup_root()?; let ioctx = IoContext::new(root); - let node = ioctx.find(None, "/init", true, true).unwrap(); - let file = node.open(OpenOptions::READ).unwrap(); + let node = ioctx.find(None, "/init", true, true)?; + let file = node.open(OpenOptions::READ)?; let devfs = devfs::root(); #[cfg(target_arch = "x86_64")] - let console = ioctx.find(Some(devfs.clone()), "tty0", true, true).unwrap(); + let console = ioctx.find(Some(devfs.clone()), "tty0", true, true)?; #[cfg(target_arch = "aarch64")] - let console = ioctx - .find(Some(devfs.clone()), "ttyS0", true, true) - .unwrap(); - let stdin = console.open(OpenOptions::READ).unwrap(); - let stdout = console.open(OpenOptions::WRITE).unwrap(); + let console = ioctx.find(Some(devfs.clone()), "ttyS0", true, true)?; + let stdin = console.open(OpenOptions::READ)?; + let stdout = console.open(OpenOptions::WRITE)?; let stderr = stdout.clone(); { // XXX let (user_init, user_init_main) = - proc::exec::load_elf("init", file, &["/init", "xxx"], &[]).unwrap(); + proc::exec::load_elf("init", file, &["/init", "xxx"], &[])?; let mut io = user_init.io.lock(); io.set_ioctx(ioctx); - io.set_file(RawFd::STDIN, stdin).unwrap(); - io.set_file(RawFd::STDOUT, stdout).unwrap(); - io.set_file(RawFd::STDERR, stderr).unwrap(); + io.set_file(RawFd::STDIN, stdin)?; + io.set_file(RawFd::STDOUT, stdout)?; + io.set_file(RawFd::STDERR, stderr)?; drop(io); user_init.set_session_terminal(console); user_init_main.enqueue_somewhere(); } + + Ok(()) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index a3fe19e4..d26de8cb 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -381,12 +381,18 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } - SyscallFunction::Exit => { + SyscallFunction::ExitThread => { let code = ExitCode::from(args[0] as i32); - // TODO separate handlers for process exit and thread exit? thread.exit(code); - // Process::current().exit(code); - panic!(); + unreachable!() + } + SyscallFunction::ExitProcess => { + // A bit different from thread exit: wait for other threads to finish and exit only + // after that + let code = ExitCode::from(args[0] as i32); + + thread.exit_process(code); + unreachable!() } SyscallFunction::SendSignal => { let pid = ProcessId::from(args[0] as u32); diff --git a/src/task/context.rs b/src/task/context.rs index c6c36c14..96fff708 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -1,5 +1,7 @@ //! Platform-specific task context manipulation interfaces +use core::fmt; + use abi::{arch::SavedFrame, error::Error, process::ExitCode}; use alloc::boxed::Box; use cfg_if::cfg_if; @@ -16,6 +18,34 @@ cfg_if! { } } +pub trait Termination { + fn into_exit_code(self) -> ExitCode; +} + +impl Termination for Result { + fn into_exit_code(self) -> ExitCode { + match self { + Ok(_) => ExitCode::SUCCESS, + Err(err) => { + warnln!("Kernel thread failed: {:?}", err); + ExitCode::Exited(1) + } + } + } +} + +impl Termination for ExitCode { + fn into_exit_code(self) -> ExitCode { + self + } +} + +impl Termination for () { + fn into_exit_code(self) -> ExitCode { + ExitCode::SUCCESS + } +} + /// Interface for task state save/restore mechanisms pub trait TaskFrame { /// Creates a "snapshot" of a exception/syscall frame @@ -81,17 +111,20 @@ pub trait TaskContextImpl: Sized { unsafe fn switch(&self, from: &Self); /// Constructs a safe wrapper process to execute a kernel-space closure - fn kernel_closure(f: F) -> Result { - extern "C" fn closure_wrapper(closure_addr: usize) -> ! { + fn kernel_closure T + Send + 'static>( + f: F, + ) -> Result { + extern "C" fn closure_wrapper T + Send + 'static>( + closure_addr: usize, + ) -> ! { let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; - closure(); - - Thread::current().exit(ExitCode::SUCCESS); + let result = closure(); + Thread::current().exit(result.into_exit_code()); unreachable!(); } let closure = Box::new(f); debugln!("closure: {:p}", closure); - Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) + Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) } } diff --git a/src/task/mod.rs b/src/task/mod.rs index f472bdc6..1f7658aa 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -11,7 +11,10 @@ use crate::{ task::{sched::CpuQueue, thread::Thread}, }; -use self::{context::TaskContextImpl, process::Process}; +use self::{ + context::{TaskContextImpl, Termination}, + process::Process, +}; pub mod context; pub mod process; @@ -65,7 +68,7 @@ pub use context::{Cpu, TaskContext}; // pub static PROCESSES: IrqSafeSpinlock = IrqSafeSpinlock::new(ProcessList::new()); /// Creates a new kernel-space process to execute a closure and queues it to some CPU -pub fn spawn_kernel_closure, F: Fn() + Send + 'static>( +pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send + 'static>( name: S, f: F, ) -> Result<(), Error> { diff --git a/src/task/process.rs b/src/task/process.rs index 5bae9a4a..4156bab2 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -359,6 +359,21 @@ impl Process { pub fn get(id: ProcessId) -> Option> { PROCESSES.lock().get(&id).cloned() } + + pub async fn terminate_others(&self, except: ThreadId) { + let mut inner = self.inner.lock(); + + for thread in inner.threads.iter() { + if thread.id() == except { + continue; + } + + infoln!("Terminate thread {}", thread.id()); + thread.terminate().await; + } + + inner.threads.retain(|t| t.id() == except); + } } impl fmt::Display for ProcessId { diff --git a/src/task/sched.rs b/src/task/sched.rs index 174a5520..f36b12b4 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -175,6 +175,10 @@ impl CpuQueue { let current_t = current.and_then(Thread::get); if let Some(current_t) = current_t.as_ref() { + if current_t.state.load(Ordering::Acquire) == ThreadState::Terminated { + current_t.exit_notify.wake_all(); + } + if current_t .state .compare_exchange( diff --git a/src/task/thread.rs b/src/task/thread.rs index ec52067e..1719db56 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -2,7 +2,9 @@ use core::{ fmt, mem::size_of, ops::Deref, + pin::Pin, sync::atomic::{AtomicU32, AtomicU64, Ordering}, + task::{Context, Poll}, }; use abi::{ @@ -15,16 +17,19 @@ use alloc::{ sync::Arc, }; use atomic_enum::atomic_enum; -use futures_util::task::ArcWake; +use futures_util::{task::ArcWake, Future}; use kernel_util::util::OneTimeInit; use crate::{ + block, mem::{process::ProcessAddressSpace, ForeignPointer}, sync::{IrqGuard, IrqSafeSpinlock}, task::{context::TaskContextImpl, Cpu}, }; -use super::{context::TaskFrame, process::Process, sched::CpuQueue, TaskContext}; +use super::{ + context::TaskFrame, process::Process, runtime::QueueWaker, sched::CpuQueue, TaskContext, +}; /// Represents the states a thread can be at some point in time #[atomic_enum] @@ -54,7 +59,6 @@ struct SignalEntry { } struct ThreadInner { - exit_status: i32, queue: Option<&'static CpuQueue>, signal_entry: Option, @@ -73,6 +77,7 @@ pub struct Thread { space: Option>, inner: IrqSafeSpinlock, + pub(super) exit_notify: Arc, } static THREADS: IrqSafeSpinlock>> = @@ -102,12 +107,13 @@ impl Thread { space, inner: IrqSafeSpinlock::new(ThreadInner { - exit_status: 0, queue: None, signal_stack: VecDeque::new(), signal_entry: None, }), + + exit_notify: Arc::new(QueueWaker::new()), }); THREADS.lock().insert(id, thread.clone()); @@ -225,8 +231,8 @@ impl Thread { assert_ne!(new_state, ThreadState::Ready); assert_ne!(new_state, ThreadState::Running); - let mut inner = self.inner.lock(); let current_state = self.state.swap(new_state, Ordering::SeqCst); + let mut inner = self.inner.lock(); let proc_queue = inner.queue.take().unwrap(); proc_queue.dequeue(self.id()); @@ -271,24 +277,6 @@ impl Thread { } // Thread inner - /// Handles the cleanup of an exited thread - pub fn handle_exit(&self) { - // Scheduler still holds a lock of this process? - // TODO cancel Wait if a process was killed while suspended? - let code = { - let inner = self.inner.lock(); - let exit_status = ExitCode::from(inner.exit_status); - exit_status - }; - - if let Some(process) = self.process.as_ref() { - process.handle_thread_exit(self.id(), code); - } - - // TODO WaitThread, notify any waiters we're done - // self.exit_waker.wake_all(); - } - pub fn set_signal_entry(&self, entry: usize, stack: usize) { let mut inner = self.inner.lock(); inner.signal_entry.replace(SignalEntry { entry, stack }); @@ -301,6 +289,32 @@ impl Thread { self.clone().enqueue_somewhere(); } } + + pub fn terminate(self: &Arc) -> impl Future { + struct F(Arc); + + impl Future for F { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let F(thread) = self.deref(); + + thread.exit_notify.register(cx.waker()); + + if thread.state.load(Ordering::Acquire) == ThreadState::Terminated { + thread.exit_notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + } + + // Will not abort the execution: called from another thread + self.dequeue(ThreadState::Terminated); + + F(self.clone()) + } } impl ArcWake for Thread { @@ -310,13 +324,60 @@ impl ArcWake for Thread { } impl CurrentThread { - /// Terminate the current process - pub fn exit(&self, status: ExitCode) { - self.inner.lock().exit_status = status.into(); - debugln!("Thread {} exited with code {:?}", self.id(), status); + fn dequeue_terminate(&self, code: ExitCode) { + self.state + .compare_exchange( + ThreadState::Running, + ThreadState::Terminated, + Ordering::AcqRel, + Ordering::Relaxed, + ) + .unwrap(); - self.handle_exit(); - self.dequeue(ThreadState::Terminated); + if let Some(process) = self.process.as_ref() { + process.handle_thread_exit(self.id(), code); + } + + let mut inner = self.inner.lock(); + let proc_queue = inner.queue.take().unwrap(); + let queue = Cpu::local().queue(); + assert_eq!(proc_queue.index(), queue.index()); + + drop(inner); + + queue.dequeue(self.id()); + unsafe { queue.yield_cpu() } + + unreachable!() + } + + /// Terminate the current thread + pub fn exit(&self, code: ExitCode) { + // May already have been terminated by process exit + if self.state.load(Ordering::Acquire) == ThreadState::Terminated { + return; + } + + self.dequeue_terminate(code) + } + + pub fn exit_process(&self, code: ExitCode) { + let _guard = IrqGuard::acquire(); + + let process = self + .process + .clone() + .expect("exit_process() called on a detached thread"); + + let p = process.clone(); + + block! { + p.terminate_others(self.id()).await; + } + .unwrap(); + + self.exit(code); + unreachable!(); } pub fn suspend(&self) -> Result<(), Error> { From cc816920b0e6fe4653ea022e31bd4b21f0ecb6cd Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 27 Nov 2023 14:56:11 +0200 Subject: [PATCH 095/211] fs: remove old vfs --- lib/memfs/src/lib.rs | 692 +++++++++++++++++------------------ lib/vfs/Cargo.toml | 4 - lib/vfs/src/block.rs | 6 - lib/vfs/src/char.rs | 66 ---- lib/vfs/src/file.rs | 224 ------------ lib/vfs/src/fs.rs | 15 - lib/vfs/src/ioctx.rs | 274 -------------- lib/vfs/src/lib.rs | 68 ---- lib/vfs/src/node.rs | 519 -------------------------- src/arch/aarch64/boot/mod.rs | 4 +- src/arch/aarch64/mod.rs | 12 +- src/device/serial/pl011.rs | 66 ++-- src/device/tty.rs | 72 ++-- src/init.rs | 80 ++-- src/main.rs | 5 +- src/proc/exec.rs | 28 +- src/proc/io.rs | 71 +--- src/proc/mod.rs | 2 +- src/syscall/mod.rs | 307 ++-------------- src/task/process.rs | 46 +-- 20 files changed, 524 insertions(+), 2037 deletions(-) delete mode 100644 lib/vfs/src/block.rs delete mode 100644 lib/vfs/src/char.rs delete mode 100644 lib/vfs/src/file.rs delete mode 100644 lib/vfs/src/fs.rs delete mode 100644 lib/vfs/src/ioctx.rs delete mode 100644 lib/vfs/src/node.rs diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index 22344364..10aec7fc 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -1,348 +1,348 @@ //! In-memory filesystem driver #![no_std] -#![warn(missing_docs)] -#![allow(clippy::new_without_default)] -#![feature( - const_mut_refs, - maybe_uninit_uninit_array, - const_maybe_uninit_uninit_array, - maybe_uninit_array_assume_init -)] - -use core::{ - any::Any, - cell::{Ref, RefCell}, - marker::PhantomData, -}; - -use alloc::{boxed::Box, rc::Rc}; -use block::BlockAllocator; -use vfs::{BlockDevice, CreateInfo, Filesystem, Vnode, VnodeKind, VnodeRef}; -use yggdrasil_abi::{error::Error, io::FileMode, path}; - -use crate::{bvec::BVec, dir::DirectoryNode, file::FileNode, tar::TarIterator}; - -#[cfg(test)] -extern crate std; - -extern crate alloc; - -#[cfg(test)] -macro_rules! test_allocator_with_counter { - ($counter:ident, $allocator:ident) => { - static $counter: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); - - struct $allocator; - - unsafe impl $crate::block::BlockAllocator for $allocator { - fn alloc() -> Result, yggdrasil_abi::error::Error> { - let b = std::boxed::Box::into_raw(std::boxed::Box::new([0; $crate::block::SIZE])); - $counter.fetch_add(1, core::sync::atomic::Ordering::Release); - Ok(unsafe { core::ptr::NonNull::new_unchecked(b as _) }) - } - - unsafe fn dealloc(block: core::ptr::NonNull) { - $counter.fetch_sub(1, core::sync::atomic::Ordering::Release); - drop(std::boxed::Box::from_raw( - block.as_ptr() as *mut [u8; $crate::block::SIZE] - )); - } - } - }; -} - -pub mod block; -pub mod bvec; - -mod dir; -mod file; -mod tar; - -const DEFAULT_FILE_MODE: FileMode = FileMode::new(0o644); -const DEFAULT_DIR_MODE: FileMode = FileMode::new(0o755); - -/// In-memory read/write filesystem -pub struct MemoryFilesystem { - root: RefCell>, - _pd: PhantomData, -} - -impl Filesystem for MemoryFilesystem { - fn dev(self: Rc) -> Option<&'static dyn BlockDevice> { - todo!() - } - - fn data(&self) -> Option> { - todo!() - } - - fn root(self: Rc) -> Result { - Ok(self.root.borrow().clone().unwrap()) - } -} - -impl MemoryFilesystem { - fn make_path( - self: &Rc, - at: &VnodeRef, - path: &str, - kind: VnodeKind, - create: bool, - ) -> Result { - if path.is_empty() { - return Ok(at.clone()); - } - let (element, rest) = path::split_left(path); - assert!(!element.is_empty()); - - let node = at.lookup(element); - let node = match node { - Some(node) => node, - None => { - if !create { - return Err(Error::DoesNotExist); - } - - let node = self.create_node_initial(element, kind); - // TODO require .tar's to have all the directories present to extract their metadata? - if kind == VnodeKind::Directory { - node.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); - } - at.add_child(node.clone()); - - node - } - }; - - if rest.is_empty() { - Ok(node) - } else { - assert!(node.is_directory()); - self.make_path(&node, rest, kind, create) - } - } - - fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { - assert!(!name.is_empty()); - assert!(!name.contains('/')); - - let node = Vnode::new(name, kind); - node.set_fs(self.clone()); - - // match kind { - // VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new( - // info.uid, info.gid, info.mode, - // ))), - // VnodeKind::Regular => {} - // _ => todo!(), - // } - - node - } - - fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { - let root = Vnode::new("", VnodeKind::Directory); - root.set_fs(self.clone()); - root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); - - // 1. Create paths in tar - for item in TarIterator::new(tar_data) { - let Ok((hdr, _)) = item else { - return Err(Error::InvalidArgument); - }; - - let path = hdr.name.as_str()?.trim_matches('/'); - let (dirname, filename) = path::split_right(path); - let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; - let node = self.create_node_initial(filename, hdr.node_kind()); - - parent.add_child(node); - } - - // 2. Associate files with their data - for item in TarIterator::new(tar_data) { - let Ok((hdr, data)) = item else { - panic!("Unreachable"); - }; - - let path = hdr.name.as_str()?.trim_matches('/'); - let node = self.make_path(&root, path, VnodeKind::Directory, false)?; - assert_eq!(node.kind(), hdr.node_kind()); - - if hdr.node_kind() == VnodeKind::Regular { - let uid = usize::from(&hdr.uid).try_into().unwrap(); - let gid = usize::from(&hdr.gid).try_into().unwrap(); - let mode = convert_mode(usize::from(&hdr.mode))?; - - let data = data.unwrap(); - let bvec = BVec::::try_from(data)?; - assert_eq!(bvec.size(), data.len()); - node.set_data(Box::new(FileNode { - uid, - gid, - mode, - data: RefCell::new(bvec), - })); - } - } - - Ok(root) - } - - /// Constructs a filesystem tree from a tar image in memory - pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { - let fs = Rc::new(Self { - root: RefCell::new(None), - _pd: PhantomData, - }); - let root = fs.from_slice_internal(tar_data)?; - fs.root.replace(Some(root)); - - Ok(fs) - } - - /// Constructs an empty memory filesystem - pub fn empty() -> Rc { - let fs = Rc::new(Self { - root: RefCell::new(None), - _pd: PhantomData, - }); - let root = Vnode::new("", VnodeKind::Directory); - root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); - root.set_fs(fs.clone()); - fs.root.replace(Some(root)); - fs - } -} - -fn convert_mode(mode: usize) -> Result { - Ok(FileMode::new(mode as u32 & 0o777)) -} - -#[cfg(test)] -mod tests { - use core::sync::atomic::Ordering; - use std::rc::Rc; - use vfs::{Filesystem, IoContext, Read, Seek, VnodeKind, Write}; - use yggdrasil_abi::io::{FileMode, OpenOptions, SeekFrom}; - - use crate::MemoryFilesystem; - - #[repr(C)] - struct AlignedTo { - _align: [Align; 0], - bytes: Bytes, - } - - static TEST_IMAGE: &'static AlignedTo = &AlignedTo { - _align: [], - bytes: *include_bytes!("../test/test_image.tar"), - }; - - #[test] - fn test_memfs_construction() { - fn check_file(ioctx: &IoContext, path: &str, expected_data: &str) { - let node = ioctx.find(None, path, false, false).unwrap(); - - assert_eq!(node.kind(), VnodeKind::Regular); - assert_eq!(node.size().unwrap(), expected_data.len() as u64); - - let file = node.open(OpenOptions::READ).unwrap(); - let mut buf = [0; 512]; - - assert_eq!( - file.borrow_mut().read(&mut buf).unwrap(), - expected_data.len() - ); - - assert_eq!(&buf[..expected_data.len()], expected_data.as_bytes()); - } - - test_allocator_with_counter!(A_COUNTER, A); - - let fs = MemoryFilesystem::::from_slice(&TEST_IMAGE.bytes).unwrap(); - let root = fs.root().unwrap(); - - // Files are small, so no indirect blocks allocated - assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); - - let ioctx = IoContext::new(root.clone()); - - assert!(Rc::ptr_eq( - &root, - &ioctx.find(None, "/", false, false).unwrap() - )); - - let old_data = include_str!("../test/test1.txt"); - check_file(&ioctx, "/test1.txt", old_data); - - // Write to the file - { - let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); - let file = node.open(OpenOptions::WRITE).unwrap(); - - assert_eq!(file.borrow_mut().write(b"Hello").unwrap(), 5); - } - - assert_eq!(A_COUNTER.load(Ordering::Acquire), 1); - - // Read back - { - let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); - let file = node.open(OpenOptions::READ).unwrap(); - - let mut buf = [0; 512]; - assert_eq!(file.borrow_mut().read(&mut buf).unwrap(), old_data.len()); - - assert_eq!(&buf[..5], b"Hello"); - assert_eq!(&buf[5..old_data.len()], &old_data.as_bytes()[5..]); - } - } - - // #[test] - // fn test_memfs_create_and_write() { - // test_allocator_with_counter!(A_COUNTER, A); - - // let fs = MemoryFilesystem::::empty(); - // let root = fs.root().unwrap(); - - // let ioctx = IoContext::new(root.clone()); - - // // Create, write, seek and read file - // { - // // TODO CREATE option handling - // root.create("test1.txt", VnodeKind::Regular).unwrap(); - - // let file = ioctx - // .open( - // None, - // "/test1.txt", - // OpenOptions::WRITE | OpenOptions::READ, - // FileMode::empty(), - // ) - // .unwrap(); - - // let write_data = [1, 2, 3, 4]; - // let mut read_data = [0; 512]; - - // let mut file = file.borrow_mut(); - // assert_eq!(file.write(&write_data).unwrap(), write_data.len()); - // assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); - // assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); - // assert_eq!(&read_data[..write_data.len()], &write_data[..]); - // } - - // // Create a directory - // { - // // TODO read directory - // root.create("dir1", VnodeKind::Directory).unwrap(); - - // let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); - // let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); - // assert!(Rc::ptr_eq( - // &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), - // &node - // )); - // } - // } -} +// #![warn(missing_docs)] +// #![allow(clippy::new_without_default)] +// #![feature( +// const_mut_refs, +// maybe_uninit_uninit_array, +// const_maybe_uninit_uninit_array, +// maybe_uninit_array_assume_init +// )] +// +// use core::{ +// any::Any, +// cell::{Ref, RefCell}, +// marker::PhantomData, +// }; +// +// use alloc::{boxed::Box, rc::Rc}; +// use block::BlockAllocator; +// use vfs::{BlockDevice, CreateInfo, Filesystem, Vnode, VnodeKind, VnodeRef}; +// use yggdrasil_abi::{error::Error, io::FileMode, path}; +// +// use crate::{bvec::BVec, dir::DirectoryNode, file::FileNode, tar::TarIterator}; +// +// #[cfg(test)] +// extern crate std; +// +// extern crate alloc; +// +// #[cfg(test)] +// macro_rules! test_allocator_with_counter { +// ($counter:ident, $allocator:ident) => { +// static $counter: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); +// +// struct $allocator; +// +// unsafe impl $crate::block::BlockAllocator for $allocator { +// fn alloc() -> Result, yggdrasil_abi::error::Error> { +// let b = std::boxed::Box::into_raw(std::boxed::Box::new([0; $crate::block::SIZE])); +// $counter.fetch_add(1, core::sync::atomic::Ordering::Release); +// Ok(unsafe { core::ptr::NonNull::new_unchecked(b as _) }) +// } +// +// unsafe fn dealloc(block: core::ptr::NonNull) { +// $counter.fetch_sub(1, core::sync::atomic::Ordering::Release); +// drop(std::boxed::Box::from_raw( +// block.as_ptr() as *mut [u8; $crate::block::SIZE] +// )); +// } +// } +// }; +// } +// +// pub mod block; +// pub mod bvec; +// +// mod dir; +// mod file; +// mod tar; +// +// const DEFAULT_FILE_MODE: FileMode = FileMode::new(0o644); +// const DEFAULT_DIR_MODE: FileMode = FileMode::new(0o755); +// +// /// In-memory read/write filesystem +// pub struct MemoryFilesystem { +// root: RefCell>, +// _pd: PhantomData, +// } +// +// impl Filesystem for MemoryFilesystem { +// fn dev(self: Rc) -> Option<&'static dyn BlockDevice> { +// todo!() +// } +// +// fn data(&self) -> Option> { +// todo!() +// } +// +// fn root(self: Rc) -> Result { +// Ok(self.root.borrow().clone().unwrap()) +// } +// } +// +// impl MemoryFilesystem { +// fn make_path( +// self: &Rc, +// at: &VnodeRef, +// path: &str, +// kind: VnodeKind, +// create: bool, +// ) -> Result { +// if path.is_empty() { +// return Ok(at.clone()); +// } +// let (element, rest) = path::split_left(path); +// assert!(!element.is_empty()); +// +// let node = at.lookup(element); +// let node = match node { +// Some(node) => node, +// None => { +// if !create { +// return Err(Error::DoesNotExist); +// } +// +// let node = self.create_node_initial(element, kind); +// // TODO require .tar's to have all the directories present to extract their metadata? +// if kind == VnodeKind::Directory { +// node.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); +// } +// at.add_child(node.clone()); +// +// node +// } +// }; +// +// if rest.is_empty() { +// Ok(node) +// } else { +// assert!(node.is_directory()); +// self.make_path(&node, rest, kind, create) +// } +// } +// +// fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { +// assert!(!name.is_empty()); +// assert!(!name.contains('/')); +// +// let node = Vnode::new(name, kind); +// node.set_fs(self.clone()); +// +// // match kind { +// // VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new( +// // info.uid, info.gid, info.mode, +// // ))), +// // VnodeKind::Regular => {} +// // _ => todo!(), +// // } +// +// node +// } +// +// fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { +// let root = Vnode::new("", VnodeKind::Directory); +// root.set_fs(self.clone()); +// root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); +// +// // 1. Create paths in tar +// for item in TarIterator::new(tar_data) { +// let Ok((hdr, _)) = item else { +// return Err(Error::InvalidArgument); +// }; +// +// let path = hdr.name.as_str()?.trim_matches('/'); +// let (dirname, filename) = path::split_right(path); +// let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; +// let node = self.create_node_initial(filename, hdr.node_kind()); +// +// parent.add_child(node); +// } +// +// // 2. Associate files with their data +// for item in TarIterator::new(tar_data) { +// let Ok((hdr, data)) = item else { +// panic!("Unreachable"); +// }; +// +// let path = hdr.name.as_str()?.trim_matches('/'); +// let node = self.make_path(&root, path, VnodeKind::Directory, false)?; +// assert_eq!(node.kind(), hdr.node_kind()); +// +// if hdr.node_kind() == VnodeKind::Regular { +// let uid = usize::from(&hdr.uid).try_into().unwrap(); +// let gid = usize::from(&hdr.gid).try_into().unwrap(); +// let mode = convert_mode(usize::from(&hdr.mode))?; +// +// let data = data.unwrap(); +// let bvec = BVec::::try_from(data)?; +// assert_eq!(bvec.size(), data.len()); +// node.set_data(Box::new(FileNode { +// uid, +// gid, +// mode, +// data: RefCell::new(bvec), +// })); +// } +// } +// +// Ok(root) +// } +// +// /// Constructs a filesystem tree from a tar image in memory +// pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { +// let fs = Rc::new(Self { +// root: RefCell::new(None), +// _pd: PhantomData, +// }); +// let root = fs.from_slice_internal(tar_data)?; +// fs.root.replace(Some(root)); +// +// Ok(fs) +// } +// +// /// Constructs an empty memory filesystem +// pub fn empty() -> Rc { +// let fs = Rc::new(Self { +// root: RefCell::new(None), +// _pd: PhantomData, +// }); +// let root = Vnode::new("", VnodeKind::Directory); +// root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); +// root.set_fs(fs.clone()); +// fs.root.replace(Some(root)); +// fs +// } +// } +// +// fn convert_mode(mode: usize) -> Result { +// Ok(FileMode::new(mode as u32 & 0o777)) +// } +// +// #[cfg(test)] +// mod tests { +// use core::sync::atomic::Ordering; +// use std::rc::Rc; +// use vfs::{Filesystem, IoContext, Read, Seek, VnodeKind, Write}; +// use yggdrasil_abi::io::{FileMode, OpenOptions, SeekFrom}; +// +// use crate::MemoryFilesystem; +// +// #[repr(C)] +// struct AlignedTo { +// _align: [Align; 0], +// bytes: Bytes, +// } +// +// static TEST_IMAGE: &'static AlignedTo = &AlignedTo { +// _align: [], +// bytes: *include_bytes!("../test/test_image.tar"), +// }; +// +// #[test] +// fn test_memfs_construction() { +// fn check_file(ioctx: &IoContext, path: &str, expected_data: &str) { +// let node = ioctx.find(None, path, false, false).unwrap(); +// +// assert_eq!(node.kind(), VnodeKind::Regular); +// assert_eq!(node.size().unwrap(), expected_data.len() as u64); +// +// let file = node.open(OpenOptions::READ).unwrap(); +// let mut buf = [0; 512]; +// +// assert_eq!( +// file.borrow_mut().read(&mut buf).unwrap(), +// expected_data.len() +// ); +// +// assert_eq!(&buf[..expected_data.len()], expected_data.as_bytes()); +// } +// +// test_allocator_with_counter!(A_COUNTER, A); +// +// let fs = MemoryFilesystem::::from_slice(&TEST_IMAGE.bytes).unwrap(); +// let root = fs.root().unwrap(); +// +// // Files are small, so no indirect blocks allocated +// assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); +// +// let ioctx = IoContext::new(root.clone()); +// +// assert!(Rc::ptr_eq( +// &root, +// &ioctx.find(None, "/", false, false).unwrap() +// )); +// +// let old_data = include_str!("../test/test1.txt"); +// check_file(&ioctx, "/test1.txt", old_data); +// +// // Write to the file +// { +// let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); +// let file = node.open(OpenOptions::WRITE).unwrap(); +// +// assert_eq!(file.borrow_mut().write(b"Hello").unwrap(), 5); +// } +// +// assert_eq!(A_COUNTER.load(Ordering::Acquire), 1); +// +// // Read back +// { +// let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); +// let file = node.open(OpenOptions::READ).unwrap(); +// +// let mut buf = [0; 512]; +// assert_eq!(file.borrow_mut().read(&mut buf).unwrap(), old_data.len()); +// +// assert_eq!(&buf[..5], b"Hello"); +// assert_eq!(&buf[5..old_data.len()], &old_data.as_bytes()[5..]); +// } +// } +// +// // #[test] +// // fn test_memfs_create_and_write() { +// // test_allocator_with_counter!(A_COUNTER, A); +// +// // let fs = MemoryFilesystem::::empty(); +// // let root = fs.root().unwrap(); +// +// // let ioctx = IoContext::new(root.clone()); +// +// // // Create, write, seek and read file +// // { +// // // TODO CREATE option handling +// // root.create("test1.txt", VnodeKind::Regular).unwrap(); +// +// // let file = ioctx +// // .open( +// // None, +// // "/test1.txt", +// // OpenOptions::WRITE | OpenOptions::READ, +// // FileMode::empty(), +// // ) +// // .unwrap(); +// +// // let write_data = [1, 2, 3, 4]; +// // let mut read_data = [0; 512]; +// +// // let mut file = file.borrow_mut(); +// // assert_eq!(file.write(&write_data).unwrap(), write_data.len()); +// // assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); +// // assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); +// // assert_eq!(&read_data[..write_data.len()], &write_data[..]); +// // } +// +// // // Create a directory +// // { +// // // TODO read directory +// // root.create("dir1", VnodeKind::Directory).unwrap(); +// +// // let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); +// // let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); +// // assert!(Rc::ptr_eq( +// // &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), +// // &node +// // )); +// // } +// // } +// } diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 10c8bf13..6d86b5ea 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -6,7 +6,3 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../kernel-util" } -bitflags = "2.3.3" -log = "0.4.20" diff --git a/lib/vfs/src/block.rs b/lib/vfs/src/block.rs deleted file mode 100644 index 1c6faa90..00000000 --- a/lib/vfs/src/block.rs +++ /dev/null @@ -1,6 +0,0 @@ -use yggdrasil_abi::error::Error; - -pub trait BlockDevice { - fn read(&self, pos: usize, buf: &mut [u8]) -> Result<(), Error>; - fn write(&self, pos: usize, buf: &[u8]) -> Result<(), Error>; -} diff --git a/lib/vfs/src/char.rs b/lib/vfs/src/char.rs deleted file mode 100644 index abf98923..00000000 --- a/lib/vfs/src/char.rs +++ /dev/null @@ -1,66 +0,0 @@ -use core::any::Any; - -use alloc::boxed::Box; -use yggdrasil_abi::{ - error::Error, - io::{DeviceRequest, FileAttr, FileMode, FileType, OpenOptions}, -}; - -use crate::{node::VnodeImpl, VnodeRef}; - -pub trait CharDevice { - fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result; - fn write(&self, blocking: bool, data: &[u8]) -> Result; - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error>; -} - -pub struct CharDeviceWrapper { - device: &'static dyn CharDevice, -} - -impl CharDeviceWrapper { - pub const fn new(device: &'static dyn CharDevice) -> Self { - Self { device } - } -} - -impl VnodeImpl for CharDeviceWrapper { - fn open( - &self, - _node: &VnodeRef, - _opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - Ok((0, None)) - } - - fn close(&self, _node: &VnodeRef) -> Result<(), Error> { - Ok(()) - } - - fn read( - &self, - _node: &VnodeRef, - _pos: u64, - _inner: Option<&mut Box>, - data: &mut [u8], - ) -> Result { - self.device.read(true, data) - } - - fn write(&self, _node: &VnodeRef, _pos: u64, data: &[u8]) -> Result { - self.device.write(true, data) - } - - fn metadata(&self, _node: &VnodeRef) -> Result { - Ok(FileAttr { - size: 0, - ty: FileType::Char, - mode: FileMode::default_file(), - }) - } - - fn device_request(&self, _node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { - self.device.device_request(req) - } -} diff --git a/lib/vfs/src/file.rs b/lib/vfs/src/file.rs deleted file mode 100644 index 51fe0b25..00000000 --- a/lib/vfs/src/file.rs +++ /dev/null @@ -1,224 +0,0 @@ -use core::{any::Any, cell::RefCell, mem::MaybeUninit}; - -use alloc::{boxed::Box, rc::Rc}; -use bitflags::bitflags; -use yggdrasil_abi::{ - error::Error, - io::{DirectoryEntry, FileType}, -}; - -use crate::{ - node::{VnodeKind, VnodeRef}, - Read, ReadDirectory, Seek, SeekFrom, Write, DIR_POSITION_FROM_CACHE, -}; - -bitflags! { - pub struct FileFlags: u32 { - const READ = 1 << 0; - const WRITE = 1 << 1; - } -} - -pub type FileRef = Rc>; - -enum DirectoryPosition { - // TODO not the best implementation, but at least somewhat safe? - CachePosition(usize), - Dot, - DotDot, - #[allow(dead_code)] - DiskPosition(u64), - #[allow(dead_code)] - End, -} - -pub struct NormalFile { - vnode: VnodeRef, - data: Option>, - pos: u64, -} - -pub struct Directory { - vnode: VnodeRef, - pos: DirectoryPosition, -} - -pub enum FileInner { - Normal(NormalFile), - Directory(Directory), -} - -pub struct File { - inner: FileInner, - flags: FileFlags, -} - -impl File { - pub fn normal( - vnode: VnodeRef, - pos: u64, - data: Option>, - flags: FileFlags, - ) -> FileRef { - Rc::new(RefCell::new(Self { - inner: FileInner::Normal(NormalFile { vnode, pos, data }), - flags, - })) - } - - pub fn directory(vnode: VnodeRef, pos: u64) -> FileRef { - let pos = if pos == DIR_POSITION_FROM_CACHE { - // Read from cache - DirectoryPosition::Dot - } else { - // Reading from a "physical" directory - todo!() - }; - Rc::new(RefCell::new(Self { - inner: FileInner::Directory(Directory { vnode, pos }), - flags: FileFlags::READ, - })) - } - - pub fn node(&self) -> Result { - match &self.inner { - FileInner::Normal(inner) => Ok(inner.vnode.clone()), - FileInner::Directory(inner) => Ok(inner.vnode.clone()), - } - } -} - -impl Write for File { - fn write(&mut self, data: &[u8]) -> Result { - if !self.flags.contains(FileFlags::WRITE) { - panic!(); - } - - match &mut self.inner { - FileInner::Normal(inner) => { - let count = inner.vnode.write(inner.pos, data)?; - if inner.vnode.kind() != VnodeKind::Char { - inner.pos += count as u64; - } - Ok(count) - } - FileInner::Directory(_) => unimplemented!(), - } - } -} - -impl Read for File { - fn read(&mut self, data: &mut [u8]) -> Result { - if !self.flags.contains(FileFlags::READ) { - panic!(); - } - - match &mut self.inner { - FileInner::Normal(inner) => { - let count = inner.vnode.read(inner.pos, inner.data.as_mut(), data)?; - if inner.vnode.kind() != VnodeKind::Char { - inner.pos += count as u64; - } - Ok(count) - } - FileInner::Directory(_) => Err(Error::IsADirectory), - } - } -} - -impl Seek for File { - fn seek(&mut self, pos: SeekFrom) -> Result { - match &mut self.inner { - FileInner::Normal(inner) => { - // TODO check if the file is actually seekable - - let size = inner.vnode.size()?; - let pos = match pos { - SeekFrom::Start(offset) => { - if offset > size { - todo!(); - } - offset - } - SeekFrom::End(0) => size, - _ => todo!(), - }; - inner.pos = pos; - - Ok(pos) - } - FileInner::Directory(_) => todo!(), - } - } -} - -impl ReadDirectory for File { - fn read_dir_entries( - &mut self, - entries: &mut [MaybeUninit], - ) -> Result { - let FileInner::Directory(inner) = &mut self.inner else { - return Err(Error::NotADirectory); - }; - - let mut nread = 0; - let mut rem = entries.len(); - - while rem != 0 { - let mut entry = DirectoryEntry { - name: [0; 256], - ty: FileType::File, - }; - - let next_position = match inner.pos { - DirectoryPosition::End => { - break; - } - DirectoryPosition::Dot => { - entry.name[0] = b'.'; - entry.ty = FileType::Directory; - DirectoryPosition::DotDot - } - DirectoryPosition::DotDot => { - entry.name[..2].copy_from_slice(b".."); - entry.ty = FileType::Directory; - DirectoryPosition::CachePosition(0) - } - DirectoryPosition::CachePosition(index) => { - let child = inner.vnode.child_at(index); - - if let Some(child) = child { - let child_name = child.name(); - entry.name[..child_name.len()].copy_from_slice(child_name.as_bytes()); - entry.ty = FileType::from(child.kind()); - DirectoryPosition::CachePosition(index + 1) - } else { - break; - } - } - DirectoryPosition::DiskPosition(_) => todo!(), - }; - inner.pos = next_position; - - entries[nread].write(entry); - - nread += 1; - rem -= 1; - } - - Ok(nread) - } -} - -impl Drop for File { - fn drop(&mut self) { - match &mut self.inner { - FileInner::Normal(inner) => { - inner.vnode.close().ok(); - } - FileInner::Directory(inner) => { - inner.vnode.close().ok(); - } - } - } -} diff --git a/lib/vfs/src/fs.rs b/lib/vfs/src/fs.rs deleted file mode 100644 index 27d51d91..00000000 --- a/lib/vfs/src/fs.rs +++ /dev/null @@ -1,15 +0,0 @@ -use core::{any::Any, cell::Ref}; - -use alloc::rc::Rc; - -use yggdrasil_abi::error::Error; - -use crate::{block::BlockDevice, node::VnodeRef}; - -pub trait Filesystem { - fn root(self: Rc) -> Result; - - fn dev(self: Rc) -> Option<&'static dyn BlockDevice>; - - fn data(&self) -> Option>; -} diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs deleted file mode 100644 index 06ad5ac4..00000000 --- a/lib/vfs/src/ioctx.rs +++ /dev/null @@ -1,274 +0,0 @@ -use log::trace; -use yggdrasil_abi::{ - error::Error, - io::{FileMode, OpenOptions}, - path, -}; - -use crate::{file::FileRef, node::VnodeRef}; - -pub struct IoContext { - root: VnodeRef, - cwd: VnodeRef, -} - -impl IoContext { - pub fn new(root: VnodeRef) -> Self { - Self { - cwd: root.clone(), - root, - } - } - - fn _find( - mut at: VnodeRef, - path: &str, - follow: bool, - follow_mount: bool, - ) -> Result { - let mut element; - let mut rest = path; - - loop { - (element, rest) = path::split_left(rest); - - if !at.is_directory() { - todo!(); - } - - match element { - path::PARENT_NAME => { - at = at.parent(); - } - path::SELF_NAME => {} - _ => break, - } - } - - if follow || follow_mount { - while let Some(target) = at.target() { - assert!(at.is_mountpoint()); - if at.is_mountpoint() && !follow_mount { - break; - } - - trace!("resolve parent: {:?} -> {:?}", at, target); - at = target; - } - } - - if !at.is_directory() { - return Err(Error::NotADirectory); - } - - if element.is_empty() && rest.is_empty() { - return Ok(at); - } - - let mut node = at.lookup_or_load(element)?; - - if follow || follow_mount { - while let Some(target) = node.target() { - assert!(node.is_mountpoint()); - if node.is_mountpoint() && !follow_mount { - break; - } - - trace!("resolve node: {:?} -> {:?}", node, target); - node = target; - } - } - - if rest.is_empty() { - Ok(node) - } else { - Self::_find(node, rest, follow, follow_mount) - } - } - - pub fn find( - &self, - at: Option, - mut path: &str, - follow: bool, - follow_mount: bool, - ) -> Result { - trace!("find {:?} in {:?}", path, at); - - let at = if path.starts_with('/') { - path = path.trim_start_matches('/'); - self.root.clone() - } else if let Some(at) = at { - at - } else { - self.cwd.clone() - }; - - Self::_find(at, path, follow, follow_mount) - } - - pub fn open( - &self, - at: Option, - path: &str, - opts: OpenOptions, - _mode: FileMode, - ) -> Result { - let node = match self.find(at.clone(), path, true, true) { - Err(Error::DoesNotExist) => { - // TODO check for create option - return Err(Error::DoesNotExist); - } - o => o, - }?; - - node.open(opts) - } - - pub fn root(&self) -> &VnodeRef { - &self.root - } -} - -#[cfg(test)] -mod tests { - use yggdrasil_abi::error::Error; - - use crate::{node::VnodeRef, IoContext}; - use std::fmt; - - macro_rules! node { - ($name:literal) => {{ - $crate::node::Vnode::new($name, $crate::node::VnodeKind::Regular) - }}; - - ($name:literal [ $($child:expr),* ]) => {{ - let _node = $crate::node::Vnode::new($name, $crate::node::VnodeKind::Directory); - - $( - _node.add_child($child); - )* - - _node - }}; - } - - struct DumpNode<'a> { - node: &'a VnodeRef, - } - - impl fmt::Debug for DumpNode<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.node.dump(f, 0, true) - } - } - - #[test] - fn test_vnode_find() { - let t = node! { - "" [ - node!("file1.txt"), - node!("file2.txt"), - node! { - "dir1" [ - node!("file3.txt") - ] - } - ] - }; - - let ctx = IoContext::new(t); - - // Absolute lookups - assert_eq!( - ctx.find(None, "/file1.txt", false, false).unwrap().name(), - "file1.txt" - ); - assert_eq!( - ctx.find(None, "/file3.txt", false, false).unwrap_err(), - Error::DoesNotExist - ); - assert_eq!( - ctx.find(None, "/dir1/file3.txt", false, false) - .unwrap() - .name(), - "file3.txt" - ); - - // Non-absolute lookups from root - assert_eq!( - ctx.find(None, "file1.txt", false, false).unwrap().name(), - "file1.txt" - ); - assert_eq!( - ctx.find(None, "dir1/file3.txt", false, false) - .unwrap() - .name(), - "file3.txt" - ); - - // Absolute lookups from non-root - let cwd = ctx.find(None, "/dir1", false, false).unwrap(); - - assert_eq!( - ctx.find(Some(cwd.clone()), "/file1.txt", false, false) - .unwrap() - .name(), - "file1.txt" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "/dir1/file3.txt", false, false) - .unwrap() - .name(), - "file3.txt" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "/file3.txt", false, false) - .unwrap_err(), - Error::DoesNotExist - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "/dir2", false, false) - .unwrap_err(), - Error::DoesNotExist - ); - - // Non-absolute lookups in non-root - assert_eq!( - ctx.find(Some(cwd.clone()), "file3.txt", false, false) - .unwrap() - .name(), - "file3.txt" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "././file3.txt", false, false) - .unwrap() - .name(), - "file3.txt" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "../dir1/file3.txt", false, false) - .unwrap() - .name(), - "file3.txt" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), ".", false, false) - .unwrap() - .name(), - "dir1" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "..", false, false) - .unwrap() - .name(), - "" - ); - assert_eq!( - ctx.find(Some(cwd.clone()), "../..", false, false) - .unwrap() - .name(), - "" - ); - } -} diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 75855bd4..0c9ac1ac 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,69 +1 @@ #![no_std] - -use core::mem::MaybeUninit; - -use yggdrasil_abi::error::Error; -use yggdrasil_abi::io::{DirectoryEntry, SeekFrom}; - -extern crate alloc; - -#[cfg(test)] -extern crate std; - -pub(crate) mod block; -pub(crate) mod char; -pub(crate) mod file; -pub(crate) mod fs; -pub(crate) mod ioctx; -pub(crate) mod node; - -pub use self::block::BlockDevice; -pub use self::char::{CharDevice, CharDeviceWrapper}; -pub use file::{File, FileFlags, FileRef}; -pub use fs::Filesystem; -pub use ioctx::IoContext; -pub use node::{CreateInfo, Vnode, VnodeDump, VnodeImpl, VnodeKind, VnodeRef, VnodeWeak}; - -pub const DIR_POSITION_FROM_CACHE: u64 = u64::MAX; - -pub trait Write { - fn write(&mut self, data: &[u8]) -> Result; -} - -pub trait Read { - fn read(&mut self, data: &mut [u8]) -> Result; - - fn read_exact(&mut self, data: &mut [u8]) -> Result<(), Error> { - default_read_exact(self, data) - } -} - -pub trait Seek { - fn seek(&mut self, pos: SeekFrom) -> Result; -} - -pub trait ReadDirectory { - fn read_dir_entries( - &mut self, - entries: &mut [MaybeUninit], - ) -> Result; -} - -fn default_read_exact(this: &mut R, mut buf: &mut [u8]) -> Result<(), Error> { - while !buf.is_empty() { - match this.read(buf) { - Ok(0) => break, - Ok(n) => { - let tmp = buf; - buf = &mut tmp[n..]; - } - Err(e) => todo!("default_read_exact: {:?}", e), - } - } - - if !buf.is_empty() { - todo!("default_read_exact unexpected eof") - } else { - Ok(()) - } -} diff --git a/lib/vfs/src/node.rs b/lib/vfs/src/node.rs deleted file mode 100644 index 547d19ee..00000000 --- a/lib/vfs/src/node.rs +++ /dev/null @@ -1,519 +0,0 @@ -use core::{ - any::Any, - cell::{Ref, RefCell, RefMut}, - fmt, -}; - -use alloc::{ - boxed::Box, - rc::{Rc, Weak}, - string::String, - vec::Vec, -}; -use kernel_util::util::OneTimeInit; -use yggdrasil_abi::{ - error::Error, - io::{DeviceRequest, FileAttr, FileMode, FileType, OpenOptions}, -}; - -use crate::{ - file::{File, FileFlags, FileRef}, - fs::Filesystem, - DIR_POSITION_FROM_CACHE, -}; - -pub type VnodeRef = Rc; -pub type VnodeWeak = Weak; - -pub struct VnodeDump { - node: VnodeRef, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum VnodeKind { - Directory, - Regular, - Char, - Block, -} - -pub(crate) struct TreeNode { - parent: Option, - children: Vec, -} - -pub struct Vnode { - name: String, - tree: RefCell, - kind: VnodeKind, - data: OneTimeInit>, - fs: RefCell>>, - target: RefCell>, -} - -pub struct CreateInfo<'a> { - pub name: &'a str, - pub kind: VnodeKind, - pub mode: FileMode, - pub uid: u32, - pub gid: u32, -} - -#[allow(unused_variables)] -pub trait VnodeImpl { - fn create(&self, at: &VnodeRef, info: &CreateInfo) -> Result { - Err(Error::NotImplemented) - } - - fn remove(&self, at: &VnodeRef, name: &str) -> Result<(), Error> { - Err(Error::NotImplemented) - } - - fn open( - &self, - node: &VnodeRef, - opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - Err(Error::NotImplemented) - } - - fn close(&self, node: &VnodeRef) -> Result<(), Error> { - Err(Error::NotImplemented) - } - - fn read( - &self, - node: &VnodeRef, - pos: u64, - inner: Option<&mut Box>, - data: &mut [u8], - ) -> Result { - Err(Error::NotImplemented) - } - - fn write(&self, node: &VnodeRef, pos: u64, data: &[u8]) -> Result { - Err(Error::NotImplemented) - } - - fn size(&self, node: &VnodeRef) -> Result { - Err(Error::NotImplemented) - } - - fn metadata(&self, node: &VnodeRef) -> Result { - Err(Error::NotImplemented) - } - - fn device_request(&self, node: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { - Err(Error::NotImplemented) - } -} - -impl Vnode { - pub fn new>(name: S, kind: VnodeKind) -> VnodeRef { - Rc::new(Self { - name: name.into(), - tree: RefCell::new(TreeNode { - parent: None, - children: Vec::new(), - }), - kind, - data: OneTimeInit::new(), - fs: RefCell::new(None), - target: RefCell::new(None), - }) - } - - #[inline] - pub fn name(&self) -> &str { - &self.name - } - - #[inline] - pub fn kind(&self) -> VnodeKind { - self.kind - } - - #[inline] - pub fn target(&self) -> Option { - self.target.borrow().clone() - } - - #[inline] - pub fn data(&self) -> Option<&dyn VnodeImpl> { - self.data.try_get().map(Box::as_ref) - } - // #[inline] - // pub fn data(&self) -> RefMut>> { - // match self.data.try_borrow_mut() { - // Ok(r) => r, - // Err(e) => { - // panic!("{:?} on {:?}", e, self.name()) - // } - // } - // } - - #[inline] - pub fn fs(&self) -> Option> { - self.fs.borrow().clone() - } - - #[inline] - pub fn set_target(&self, target: Option) { - self.target.replace(target); - } - - pub fn parent(self: &VnodeRef) -> VnodeRef { - match &self.tree.borrow().parent { - Some(parent) => parent.upgrade().unwrap(), - None => self.clone(), - } - } - - #[inline] - pub fn is_root(&self) -> bool { - self.tree.borrow().parent.is_none() - } - - pub fn set_data(&self, data: Box) { - self.data.init(data); - // self.data.borrow_mut().replace(data); - } - - pub fn set_fs(&self, data: Rc) { - self.fs.replace(Some(data)); - } - - #[inline] - pub fn is_directory(&self) -> bool { - self.kind == VnodeKind::Directory - } - - #[inline] - pub fn is_mountpoint(&self) -> bool { - self.is_directory() && self.target.borrow().is_some() - } - - // Cache tree operations - pub fn add_child(self: &VnodeRef, child: VnodeRef) { - let parent_weak = Rc::downgrade(self); - let mut parent_borrow = self.tree.borrow_mut(); - - assert!(child - .tree - .borrow_mut() - .parent - .replace(parent_weak) - .is_none()); - parent_borrow.children.push(child); - } - - pub fn remove_child(self: &VnodeRef, name: &str) { - self.children_mut().retain(|node| node.name() != name); - } - - pub fn child_at(self: &VnodeRef, index: usize) -> Option { - let tree = self.tree.borrow(); - tree.children.get(index).cloned() - } - - pub fn children(&self) -> Ref> { - let tree = self.tree.borrow(); - Ref::map(tree, |t| &t.children) - } - - pub fn children_mut(&self) -> RefMut> { - let tree = self.tree.borrow_mut(); - RefMut::map(tree, |t| &mut t.children) - } - - pub fn dump(&self, f: &mut fmt::Formatter<'_>, depth: usize, indent: bool) -> fmt::Result { - if indent { - for _ in 0..depth { - f.write_str(" ")?; - } - } - - write!(f, "{:?}", self.name)?; - - if self.is_directory() { - let tree = self.tree.borrow(); - let target = self.target(); - - if let Some(target) = target { - f.write_str(" -> ")?; - return target.dump(f, depth, false); - } - - if tree.children.is_empty() { - f.write_str(" []")?; - } else { - f.write_str(" [\n")?; - for child in tree.children.iter() { - child.dump(f, depth + 1, true)?; - f.write_str("\n")?; - } - for _ in 0..depth { - f.write_str(" ")?; - } - f.write_str("]")?; - } - } - - Ok(()) - } - - pub fn lookup(self: &VnodeRef, name: &str) -> Option { - assert!(self.is_directory()); - self.tree - .borrow() - .children - .iter() - .find(|e| e.name == name) - .cloned() - } - - // - pub fn lookup_or_load(self: &VnodeRef, name: &str) -> Result { - // Lookup in cache - if let Some(node) = self.lookup(name) { - return Ok(node); - } - - // TODO load from FS - Err(Error::DoesNotExist) - } - - // Node operations - pub fn open(self: &VnodeRef, flags: OpenOptions) -> Result { - let mut open_flags = FileFlags::empty(); - - if flags.contains(OpenOptions::READ) { - open_flags |= FileFlags::READ; - } - if flags.contains(OpenOptions::WRITE) { - open_flags |= FileFlags::WRITE; - } - - if self.kind == VnodeKind::Directory { - return Err(Error::IsADirectory); - } - - if let Some(data) = self.data() { - let (pos, inner) = data.open(self, flags)?; - Ok(File::normal(self.clone(), pos, inner, open_flags)) - } else { - todo!() - } - } - - pub fn open_directory(self: &VnodeRef) -> Result { - if !self.is_directory() { - return Err(Error::IsADirectory); - } - - if let Some(data) = self.data() { - // TODO don't discard directory's Box - let (pos, _) = data.open(self, OpenOptions::READ)?; - Ok(File::directory(self.clone(), pos)) - } else { - // TODO: some options here? - Ok(File::directory(self.clone(), DIR_POSITION_FROM_CACHE)) - } - } - - pub fn close(self: &VnodeRef) -> Result<(), Error> { - if let Some(data) = self.data() { - data.close(self) - } else { - Ok(()) - } - } - - pub fn create(self: &VnodeRef, info: &CreateInfo) -> Result { - if self.kind != VnodeKind::Directory { - todo!(); - } - if info.name.contains('/') { - return Err(Error::InvalidArgument); - } - - match self.lookup_or_load(info.name) { - Err(Error::DoesNotExist) => {} - Ok(_) => return Err(Error::AlreadyExists), - e => return e, - }; - - if let Some(data) = self.data() { - let vnode = data.create(self, info)?; - if let Some(fs) = self.fs() { - vnode.set_fs(fs); - } - self.add_child(vnode.clone()); - Ok(vnode) - } else { - Err(Error::NotImplemented) - } - } - - pub fn remove(self: &VnodeRef, node: VnodeRef, recurse: bool) -> Result<(), Error> { - let name = node.name(); - - if self.kind != VnodeKind::Directory { - todo!(); - } - if name.contains('/') { - todo!(); - } - - if node.kind() == VnodeKind::Directory { - if recurse { - todo!(); - } - - // Check if remove target is not empty - if node.size()? != 0 { - return Err(Error::DirectoryNotEmpty); - } - } - - if let Some(data) = self.data() { - data.remove(self, name)?; - } - - // Unlink node from cache - self.remove_child(name); - - Ok(()) - } - - pub fn write(self: &VnodeRef, pos: u64, buf: &[u8]) -> Result { - if self.kind == VnodeKind::Directory { - todo!(); - } - - if let Some(data) = self.data() { - data.write(self, pos, buf) - } else { - todo!() - } - } - - pub fn read( - self: &VnodeRef, - pos: u64, - inner: Option<&mut Box>, - buf: &mut [u8], - ) -> Result { - if self.kind == VnodeKind::Directory { - todo!(); - } - - if let Some(data) = self.data() { - data.read(self, pos, inner, buf) - } else { - todo!() - } - } - - pub fn size(self: &VnodeRef) -> Result { - if let Some(data) = self.data() { - data.size(self) - } else { - todo!(); - } - } - - pub fn mount(self: &VnodeRef, fs_root: VnodeRef) -> Result<(), Error> { - if !self.is_directory() { - return Err(Error::NotADirectory); - } - if !fs_root.is_directory() { - todo!("Filesystem root is not a directory"); - } - if self.target.borrow().is_some() { - todo!("Target mountpoint is busy"); - } - - { - let mut child_borrow = fs_root.tree.borrow_mut(); - if child_borrow.parent.is_some() { - todo!("Filesystem is already mounted somewhere else"); - } - child_borrow.parent = Some(Rc::downgrade(self)); - } - self.target.replace(Some(fs_root)); - - Ok(()) - } - - pub fn unmount_target(self: &VnodeRef) -> Result<(), Error> { - if !self.is_directory() { - return Err(Error::NotADirectory); - } - let Some(fs_root) = self.target.take() else { - todo!(); - }; - - { - let mut target_borrow = fs_root.tree.borrow_mut(); - let Some(parent) = target_borrow.parent.take() else { - todo!() - }; - assert!(Rc::ptr_eq(self, &parent.upgrade().unwrap())); - } - - Ok(()) - } - - pub fn metadata(self: &VnodeRef) -> Result { - if let Some(data) = self.data() { - data.metadata(self) - } else { - todo!() - } - } - - pub fn device_request(self: &VnodeRef, req: &mut DeviceRequest) -> Result<(), Error> { - if let Some(data) = self.data() { - data.device_request(self, req) - } else { - todo!() - } - } -} - -impl fmt::Debug for Vnode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let prefix = match self.kind { - VnodeKind::Directory => "DIR ", - VnodeKind::Regular => "REG ", - VnodeKind::Char => "CHR ", - VnodeKind::Block => "BLK ", - }; - - write!(f, "[{} {}]", prefix, self.name) - } -} - -impl VnodeDump { - pub fn new(node: VnodeRef) -> Self { - Self { node } - } -} - -impl fmt::Debug for VnodeDump { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.node.dump(f, 0, true) - } -} - -impl From for FileType { - fn from(value: VnodeKind) -> Self { - match value { - VnodeKind::Regular => Self::File, - VnodeKind::Directory => Self::Directory, - VnodeKind::Block => Self::Block, - VnodeKind::Char => Self::Char, - } - } -} diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 63f6fab3..ad97337e 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -12,7 +12,6 @@ use super::{ }; use crate::{ arch::{aarch64::mem::table::L3, Architecture}, - fs::devfs, kernel_main, kernel_secondary_main, mem::{address::IntoRaw, phys, table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET}, task::runtime, @@ -103,7 +102,8 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { // // Setup initrd // super::setup_initrd(); - devfs::init(); + // XXX + // devfs::init(); runtime::init_task_queue(); diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 1163da49..90490168 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -27,7 +27,6 @@ use crate::{ devtree::{self, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}, power::arm_psci::Psci, }, - fs::{Initrd, INITRD_DATA}, mem::{ address::{FromRaw, IntoRaw}, device::RawDeviceMemoryMapping, @@ -315,11 +314,12 @@ impl AArch64 { let data = unsafe { PhysicalRef::map_slice(initrd_start, len) }; self.initrd.init(data); - INITRD_DATA.init(Initrd { - phys_page_start: aligned_start, - phys_page_len: aligned_end - aligned_start, - data: self.initrd.get().as_ref(), - }); + // XXX + // INITRD_DATA.init(Initrd { + // phys_page_start: aligned_start, + // phys_page_len: aligned_end - aligned_start, + // data: self.initrd.get().as_ref(), + // }); } Ok(()) diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 22136890..5b75b017 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -8,7 +8,6 @@ use tock_registers::{ register_bitfields, register_structs, registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use vfs::CharDevice; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, @@ -19,7 +18,6 @@ use crate::{ tty::{TtyContext, TtyDevice}, }, device_tree_driver, - fs::devfs::{self, CharDeviceType}, mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, sync::IrqSafeSpinlock, task::process::ProcessId, @@ -109,37 +107,37 @@ impl TtyDevice for Pl011 { } } -impl CharDevice for Pl011 { - fn write(&self, blocking: bool, data: &[u8]) -> Result { - assert!(blocking); - self.line_write(data) - } - - fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { - assert!(blocking); - match block! { - self.line_read(data).await - } { - Ok(res) => res, - Err(err) => Err(err), - } - } - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - match req { - &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(ProcessId::from(id)); - Ok(()) - } - DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), - DeviceRequest::GetTerminalOptions(config) => { - config.write(self.context.config()); - Ok(()) - } - _ => Err(Error::InvalidArgument), - } - } -} +// impl CharDevice for Pl011 { +// fn write(&self, blocking: bool, data: &[u8]) -> Result { +// assert!(blocking); +// self.line_write(data) +// } +// +// fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { +// assert!(blocking); +// match block! { +// self.line_read(data).await +// } { +// Ok(res) => res, +// Err(err) => Err(err), +// } +// } +// +// fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { +// match req { +// &mut DeviceRequest::SetTerminalGroup(id) => { +// self.set_signal_group(ProcessId::from(id)); +// Ok(()) +// } +// DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), +// DeviceRequest::GetTerminalOptions(config) => { +// config.write(self.context.config()); +// Ok(()) +// } +// _ => Err(Error::InvalidArgument), +// } +// } +// } impl SerialDevice for Pl011 { fn send(&self, byte: u8) -> Result<(), Error> { @@ -193,7 +191,7 @@ impl Device for Pl011 { self.inner.init(IrqSafeSpinlock::new(inner)); debug::add_sink(self, LogLevel::Debug); - devfs::add_char_device(self, CharDeviceType::TtySerial)?; + // devfs::add_char_device(self, CharDeviceType::TtySerial)?; Ok(()) } diff --git a/src/device/tty.rs b/src/device/tty.rs index 6f75a59d..06f00a6d 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -21,7 +21,7 @@ pub mod combined { io::{DeviceRequest, TerminalSize}, }; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; - use vfs::CharDevice; + // use vfs::CharDevice; use crate::device::{ display::{console::DisplayConsole, fb_console::FramebufferConsole}, @@ -73,43 +73,43 @@ pub mod combined { } } - impl CharDevice for CombinedTerminal { - fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { - assert!(blocking); - match block! { - self.line_read(data).await - } { - Ok(res) => res, - Err(err) => Err(err), - } - // self.line_read(data) - } + // impl CharDevice for CombinedTerminal { + // fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { + // assert!(blocking); + // match block! { + // self.line_read(data).await + // } { + // Ok(res) => res, + // Err(err) => Err(err), + // } + // // self.line_read(data) + // } - fn write(&self, blocking: bool, data: &[u8]) -> Result { - assert!(blocking); - self.line_write(data) - } + // fn write(&self, blocking: bool, data: &[u8]) -> Result { + // assert!(blocking); + // self.line_write(data) + // } - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - match req { - &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(ProcessId::from(id)); - Ok(()) - } - DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), - DeviceRequest::GetTerminalOptions(config) => { - config.write(self.context.config()); - Ok(()) - } - DeviceRequest::GetTerminalSize(out) => { - let (rows, columns) = self.output.text_dimensions(); - out.write(TerminalSize { rows, columns }); - Ok(()) - } - _ => Err(Error::InvalidArgument), - } - } - } + // fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + // match req { + // &mut DeviceRequest::SetTerminalGroup(id) => { + // self.set_signal_group(ProcessId::from(id)); + // Ok(()) + // } + // DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), + // DeviceRequest::GetTerminalOptions(config) => { + // config.write(self.context.config()); + // Ok(()) + // } + // DeviceRequest::GetTerminalSize(out) => { + // let (rows, columns) = self.output.text_dimensions(); + // out.write(TerminalSize { rows, columns }); + // Ok(()) + // } + // _ => Err(Error::InvalidArgument), + // } + // } + // } } #[cfg(feature = "fb_console")] diff --git a/src/init.rs b/src/init.rs index 37bc02cc..efa38b4e 100644 --- a/src/init.rs +++ b/src/init.rs @@ -3,19 +3,8 @@ use abi::{ error::Error, io::{OpenOptions, RawFd}, }; -use memfs::MemoryFilesystem; -use vfs::{Filesystem, IoContext, VnodeRef}; -use crate::{ - fs::{devfs, FileBlockAllocator, INITRD_DATA}, - proc, -}; - -fn setup_root() -> Result { - let initrd_data = INITRD_DATA.get(); - let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); - fs.root() -} +use crate::proc; /// Kernel's "main" process function. /// @@ -25,46 +14,47 @@ fn setup_root() -> Result { /// initialization has finished. pub fn kinit() -> Result<(), Error> { infoln!("In main"); + loop {} - #[cfg(feature = "fb_console")] - { - use crate::{device::display::console::update_consoles_task, task::runtime}; + // #[cfg(feature = "fb_console")] + // { + // use crate::{device::display::console::update_consoles_task, task::runtime}; - runtime::spawn(async move { - update_consoles_task().await; - })?; - } + // runtime::spawn(async move { + // update_consoles_task().await; + // })?; + // } - let root = setup_root()?; + // let root = setup_root()?; - let ioctx = IoContext::new(root); - let node = ioctx.find(None, "/init", true, true)?; - let file = node.open(OpenOptions::READ)?; + // let ioctx = IoContext::new(root); + // let node = ioctx.find(None, "/init", true, true)?; + // let file = node.open(OpenOptions::READ)?; - let devfs = devfs::root(); - #[cfg(target_arch = "x86_64")] - let console = ioctx.find(Some(devfs.clone()), "tty0", true, true)?; - #[cfg(target_arch = "aarch64")] - let console = ioctx.find(Some(devfs.clone()), "ttyS0", true, true)?; - let stdin = console.open(OpenOptions::READ)?; - let stdout = console.open(OpenOptions::WRITE)?; - let stderr = stdout.clone(); + // let devfs = devfs::root(); + // #[cfg(target_arch = "x86_64")] + // let console = ioctx.find(Some(devfs.clone()), "tty0", true, true)?; + // #[cfg(target_arch = "aarch64")] + // let console = ioctx.find(Some(devfs.clone()), "ttyS0", true, true)?; + // let stdin = console.open(OpenOptions::READ)?; + // let stdout = console.open(OpenOptions::WRITE)?; + // let stderr = stdout.clone(); - { - // XXX - let (user_init, user_init_main) = - proc::exec::load_elf("init", file, &["/init", "xxx"], &[])?; - let mut io = user_init.io.lock(); - io.set_ioctx(ioctx); - io.set_file(RawFd::STDIN, stdin)?; - io.set_file(RawFd::STDOUT, stdout)?; - io.set_file(RawFd::STDERR, stderr)?; - drop(io); + // { + // // XXX + // let (user_init, user_init_main) = + // proc::exec::load_elf("init", file, &["/init", "xxx"], &[])?; + // let mut io = user_init.io.lock(); + // io.set_ioctx(ioctx); + // io.set_file(RawFd::STDIN, stdin)?; + // io.set_file(RawFd::STDOUT, stdout)?; + // io.set_file(RawFd::STDERR, stderr)?; + // drop(io); - user_init.set_session_terminal(console); + // user_init.set_session_terminal(console); - user_init_main.enqueue_somewhere(); - } + // user_init_main.enqueue_somewhere(); + // } - Ok(()) + // Ok(()) } diff --git a/src/main.rs b/src/main.rs index c4d7df49..56d2851f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,7 +34,6 @@ use arch::Architecture; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, - fs::sysfs, mem::heap, sync::SpinFence, task::{spawn_kernel_closure, Cpu}, @@ -51,7 +50,7 @@ pub mod debug; pub mod arch; pub mod device; -pub mod fs; +// pub mod fs; pub mod init; pub mod mem; pub mod panic; @@ -89,7 +88,7 @@ pub fn kernel_main() -> ! { debugln!("Heap: {:#x?}", heap::heap_range()); // Setup the sysfs - sysfs::init(); + // sysfs::init(); unsafe { ARCHITECTURE.start_application_processors(); diff --git a/src/proc/exec.rs b/src/proc/exec.rs index f1566591..d42b0821 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -7,7 +7,6 @@ use abi::{ process::ProgramArgumentInner, }; use alloc::{string::String, sync::Arc}; -use vfs::FileRef; use crate::{ mem::{ @@ -146,7 +145,8 @@ fn setup_binary>( } } - let tls_address = proc::elf::clone_tls(&space, &image)?; + // XXX + let tls_address = 0; // proc::elf::clone_tls(&space, &image)?; let context = TaskContext::user( image.entry, @@ -165,15 +165,15 @@ fn setup_binary>( // Ok(Process::new_with_context(name, Some(space), context)) } -/// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory -pub fn load_elf>( - name: S, - file: FileRef, - args: &[&str], - envs: &[&str], -) -> Result<(Arc, Arc), Error> { - let space = ProcessAddressSpace::new()?; - let image = proc::elf::load_elf_from_file(&space, file)?; - - setup_binary(name, space, image, args, envs) -} +// /// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory +// pub fn load_elf>( +// name: S, +// file: FileRef, +// args: &[&str], +// envs: &[&str], +// ) -> Result<(Arc, Arc), Error> { +// let space = ProcessAddressSpace::new()?; +// let image = proc::elf::load_elf_from_file(&space, file)?; +// +// setup_binary(name, space, image, args, envs) +// } diff --git a/src/proc/io.rs b/src/proc/io.rs index e91a5915..9c42f970 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -1,79 +1,12 @@ //! Process I/O management -use abi::{error::Error, io::RawFd}; -use alloc::collections::{btree_map::Entry, BTreeMap}; -use vfs::{FileRef, IoContext}; /// I/O context of a process, contains information like root, current directory and file /// descriptor table -pub struct ProcessIo { - ioctx: Option, - files: BTreeMap, -} +pub struct ProcessIo {} impl ProcessIo { /// Constructs an uninitialized I/O context pub fn new() -> Self { - Self { - ioctx: None, - files: BTreeMap::new(), - } - } - - /// Returns a file given descriptor refers to - pub fn file(&self, fd: RawFd) -> Result { - self.files.get(&fd).cloned().ok_or(Error::InvalidFile) - } - - /// Iterates over the file descriptors in [ProcessIo] and removes them if predicate is `false` - pub fn retain bool>(&mut self, f: F) { - self.files.retain(f) - } - - /// Sets the inner I/O context - pub fn set_ioctx(&mut self, ioctx: IoContext) { - self.ioctx.replace(ioctx); - } - - /// Inserts a file into the descriptor table. Returns error if the file is already present for - /// given descriptor. - pub fn set_file(&mut self, fd: RawFd, file: FileRef) -> Result<(), Error> { - if self.files.contains_key(&fd) { - todo!(); - } - - self.files.insert(fd, file); - Ok(()) - } - - /// Allocates a slot for a file and returns it - pub fn place_file(&mut self, file: FileRef) -> Result { - for idx in 0..64 { - let fd = RawFd(idx); - - if let Entry::Vacant(e) = self.files.entry(fd) { - e.insert(file); - return Ok(fd); - } - } - todo!(); - } - - /// Closes the file and removes it from the table - pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { - let file = self.files.remove(&fd); - if file.is_none() { - todo!(); - } - Ok(()) - } - - /// Returns the inner I/O context reference - pub fn ioctx(&mut self) -> &mut IoContext { - self.ioctx.as_mut().unwrap() - } - - /// Cleans up I/O context after the process exits - pub fn handle_exit(&mut self) { - self.files.clear(); + todo!() } } diff --git a/src/proc/mod.rs b/src/proc/mod.rs index aa280e3b..c460b3ac 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -1,5 +1,5 @@ //! Internal management for processes -pub mod elf; +// pub mod elf; pub mod exec; pub mod io; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index d26de8cb..0e0fd087 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -8,7 +8,7 @@ use abi::{ syscall::SyscallFunction, }; use alloc::rc::Rc; -use vfs::{IoContext, Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; +// use vfs::{IoContext, Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; use yggdrasil_abi::{ error::SyscallResult, io::{MountOptions, UnmountOptions}, @@ -17,7 +17,6 @@ use yggdrasil_abi::{ use crate::{ block, debug::LogLevel, - fs, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo}, sync::IrqSafeSpinlockGuard, @@ -31,26 +30,26 @@ use crate::{ mod arg; use arg::*; -fn run_with_io) -> T>(proc: &Process, f: F) -> T { - let io = proc.io.lock(); - f(io) -} - -fn run_with_io_at< - T, - F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, ->( - proc: &Process, - at: Option, - f: F, -) -> Result { - let io = proc.io.lock(); - let at = at - .map(|fd| io.file(fd).and_then(|f| f.borrow().node())) - .transpose()?; - - f(at, io) -} +// fn run_with_io) -> T>(proc: &Process, f: F) -> T { +// let io = proc.io.lock(); +// f(io) +// } +// +// fn run_with_io_at< +// T, +// F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, +// >( +// proc: &Process, +// at: Option, +// f: F, +// ) -> Result { +// let io = proc.io.lock(); +// let at = at +// .map(|fd| io.file(fd).and_then(|f| f.borrow().node())) +// .transpose()?; +// +// f(at, io) +// } fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { let thread = Thread::current(); @@ -119,247 +118,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } // I/O - SyscallFunction::Write => { - let fd = RawFd(args[0] as u32); - let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - - run_with_io(&process, |io| { - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); - - file_borrow.write(data) - }) - } - SyscallFunction::Read => { - let fd = RawFd(args[0] as u32); - let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - - run_with_io(&process, |io| { - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); - - file_borrow.read(data) - }) - } - SyscallFunction::Open => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let opts = OpenOptions::from(args[3] as u32); - let mode = FileMode::from(args[4] as u32); - - run_with_io_at(&process, at, |at, mut io| { - let file = io.ioctx().open(at, path, opts, mode)?; - - // TODO NO_CTTY? - if process.session_terminal().is_none() && - let Ok(node) = file.borrow().node() && node.kind() == VnodeKind::Char { - debugln!("Session terminal set for #{}: {}", process.id(), path); - process.set_session_terminal(node); - } - - let fd = io.place_file(file)?; - Ok(fd.0 as usize) - }) - } - SyscallFunction::Close => { - let fd = RawFd(args[0] as u32); - - run_with_io(&process, |mut io| { - io.close_file(fd)?; - Ok(0) - }) - } - SyscallFunction::OpenDirectory => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - - run_with_io_at(&process, at, |at, mut io| { - let node = io.ioctx().find(at, path, true, true)?; - let file = node.open_directory()?; - let fd = io.place_file(file)?; - - Ok(fd.0 as usize) - }) - } - SyscallFunction::ReadDirectory => { - let fd = RawFd(args[0] as u32); - let buffer = arg_user_slice_mut::>( - args[1] as usize, - args[2] as usize, - )?; - - run_with_io(&process, |io| { - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); - - file_borrow.read_dir_entries(buffer) - }) - } - SyscallFunction::CreateDirectory => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let _mode = FileMode::from(args[3] as u32); - - run_with_io_at(&process, at, |at, mut io| { - let (parent, name) = abi::path::split_right(path); - let parent_node = io.ioctx().find(at, parent, true, true)?; - todo!(); - // parent_node.create(name, VnodeKind::Directory)?; - - Ok(0) - }) - } - SyscallFunction::Remove => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let recurse = args[3] != 0; - - run_with_io_at(&process, at, |at, mut io| { - let node = io.ioctx().find(at, path, false, false)?; - - if node.is_root() || Rc::ptr_eq(io.ioctx().root(), &node) { - todo!(); - } - - let parent = node.parent(); - - parent.remove(node, recurse)?; - - Ok(0) - }) - } - SyscallFunction::GetMetadata => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let buffer = arg_user_mut::>(args[3] as usize)?; - let follow = args[4] != 0; - - run_with_io_at(&process, at, |at, mut io| { - let node = if path.is_empty() { - at.ok_or(Error::InvalidArgument)? - } else { - io.ioctx().find(None, path, follow, true)? - }; - - let metadata = node.metadata()?; - buffer.write(metadata); - - Ok(0) - }) - } - SyscallFunction::Seek => { - let fd = RawFd(args[0] as u32); - let pos = SeekFrom::from(args[1]); - - run_with_io(&process, |io| { - let file = io.file(fd)?; - let mut file_borrow = file.borrow_mut(); - - file_borrow.seek(pos).map(|v| v as usize) - }) - } - SyscallFunction::Mount => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(&process, |mut io| { - let target_node = io.ioctx().find(None, options.target, true, false)?; - if !target_node.is_directory() { - return Err(Error::NotADirectory); - } - - let fs_root = fs::create_filesystem(options)?; - - target_node.mount(fs_root)?; - - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - - Ok(0) - }) - } - SyscallFunction::Unmount => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(&process, |mut io| { - let mountpoint = io.ioctx().find(None, options.mountpoint, true, false)?; - mountpoint.unmount_target()?; - - debugln!("{:?}", vfs::VnodeDump::new(io.ioctx().root().clone())); - - Ok(0) - }) - } - SyscallFunction::DeviceRequest => { - let fd = RawFd(args[0] as u32); - let req = arg_user_mut::(args[1] as usize)?; - - run_with_io(&process, |io| { - let file = io.file(fd)?; - let node = file.borrow().node()?; - - node.device_request(req)?; - - Ok(0) - }) - } // Process management - SyscallFunction::SpawnProcess => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(&process, |mut io| { - let node = io.ioctx().find(None, options.program, true, true)?; - - // Setup a new process from the file - let file = node.open(OpenOptions::READ)?; - let (child_process, child_main) = proc::exec::load_elf( - options.program, - file, - options.arguments, - options.environment, - )?; - let pid: u32 = child_process.id().into(); - - // Inherit group and session from the creator - child_process.inherit(&process)?; - - // Inherit root from the creator - let child_ioctx = IoContext::new(io.ioctx().root().clone()); - let mut child_io = child_process.io.lock(); - child_io.set_ioctx(child_ioctx); - - for opt in options.optional { - match opt { - &SpawnOption::InheritFile { source, child } => { - let src_file = io.file(source)?; - child_io.set_file(child, src_file)?; - } - &SpawnOption::SetProcessGroup(pgroup) => { - child_process.set_group_id(pgroup.into()); - } - _ => (), - } - } - - if let Some(fd) = options.optional.iter().find_map(|item| { - if let &SpawnOption::GainTerminal(fd) = item { - Some(fd) - } else { - None - } - }) { - debugln!("{} requested terminal {:?}", pid, fd); - let file = child_io.file(fd)?; - let node = file.borrow().node()?; - let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); - - node.device_request(&mut req)?; - } - - drop(child_io); - child_main.enqueue_somewhere(); - - Ok(pid as _) - }) - } SyscallFunction::SpawnThread => { let options = arg_user_ref::(args[0] as usize)?; let id = process.spawn_thread(options)?; @@ -421,26 +180,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result process.set_group_id(group_id); Ok(0) } - SyscallFunction::StartSession => { - let session_terminal = process.clear_session_terminal(); - - if let Some(ctty) = session_terminal { - // Drop all FDs referring to the old session terminal - run_with_io(&process, |mut io| { - io.retain(|_, f| { - f.borrow() - .node() - .map(|node| !Rc::ptr_eq(&node, &ctty)) - .unwrap_or(true) - }); - }); - } - - process.set_session_id(process.id()); - process.set_group_id(process.id()); - - Ok(0) - } // Waiting and polling SyscallFunction::WaitProcess => { let pid = ProcessId::from(args[0] as u32); @@ -458,6 +197,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } + + _ => { + todo!("System call: {:?}", func); + } } } diff --git a/src/task/process.rs b/src/task/process.rs index 4156bab2..da6903e9 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -20,7 +20,6 @@ use alloc::{ }; use futures_util::Future; use kernel_util::util::OneTimeInit; -use vfs::VnodeRef; use crate::{ mem::{ @@ -131,8 +130,7 @@ struct ProcessInner { session_id: ProcessId, group_id: ProcessId, - session_terminal: Option, - + // session_terminal: Option, threads: Vec>, mutexes: BTreeMap>, } @@ -173,7 +171,7 @@ impl Process { state: ProcessState::Running, session_id: id, group_id: id, - session_terminal: None, + // session_terminal: None, threads: Vec::new(), mutexes: BTreeMap::new(), }), @@ -199,7 +197,8 @@ impl Process { ); let tls_address = if let Some(image) = self.image.as_ref() { - proc::elf::clone_tls(&self.space, image)? + todo!() + // proc::elf::clone_tls(&self.space, image)? } else { 0 }; @@ -247,28 +246,28 @@ impl Process { } // Resources - pub fn session_terminal(&self) -> Option { - self.inner.lock().session_terminal.clone() - } + // pub fn session_terminal(&self) -> Option { + // self.inner.lock().session_terminal.clone() + // } - pub fn set_session_terminal(&self, node: VnodeRef) { - self.inner.lock().session_terminal.replace(node); - } + // pub fn set_session_terminal(&self, node: VnodeRef) { + // self.inner.lock().session_terminal.replace(node); + // } - pub fn clear_session_terminal(&self) -> Option { - self.inner.lock().session_terminal.take() - } + // pub fn clear_session_terminal(&self) -> Option { + // self.inner.lock().session_terminal.take() + // } - pub fn inherit(&self, parent: &Process) -> Result<(), Error> { - let mut our_inner = self.inner.lock(); - let their_inner = parent.inner.lock(); + // pub fn inherit(&self, parent: &Process) -> Result<(), Error> { + // let mut our_inner = self.inner.lock(); + // let their_inner = parent.inner.lock(); - our_inner.session_id = their_inner.session_id; - our_inner.group_id = their_inner.group_id; - our_inner.session_terminal = their_inner.session_terminal.clone(); + // our_inner.session_id = their_inner.session_id; + // our_inner.group_id = their_inner.group_id; + // our_inner.session_terminal = their_inner.session_terminal.clone(); - Ok(()) - } + // Ok(()) + // } // State pub fn get_exit_status(&self) -> Option { @@ -319,7 +318,8 @@ impl Process { inner.state = ProcessState::Terminated(code); - self.io.lock().handle_exit(); + // XXX + // self.io.lock().handle_exit(); drop(inner); From 2d7568f829daa2a7dcdf236bd80ce158e869a88f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 30 Nov 2023 11:00:51 +0200 Subject: [PATCH 096/211] refactor: move kernel::sync to kernel-util --- lib/kernel-util/src/api.rs | 7 + lib/kernel-util/src/lib.rs | 2 + lib/kernel-util/src/sync.rs | 207 ++++++++++++++++++++++++++++++ src/arch/aarch64/cpu.rs | 4 +- src/arch/aarch64/exception.rs | 2 +- src/arch/aarch64/gic/mod.rs | 3 +- src/arch/mod.rs | 20 +++ src/debug.rs | 6 +- src/device/display/console.rs | 4 +- src/device/display/fb_console.rs | 3 +- src/device/display/linear_fb.rs | 6 +- src/device/mod.rs | 3 +- src/device/serial/pl011.rs | 3 +- src/device/tty.rs | 2 +- src/main.rs | 3 +- src/mem/phys/mod.rs | 3 +- src/mem/process.rs | 3 +- src/panic.rs | 2 +- src/sync.rs | 211 ------------------------------- src/syscall/mod.rs | 2 +- src/task/mod.rs | 2 +- src/task/process.rs | 3 +- src/task/runtime/task.rs | 3 +- src/task/runtime/task_queue.rs | 3 +- src/task/runtime/waker.rs | 3 +- src/task/sched.rs | 6 +- src/task/thread.rs | 6 +- src/util/ring.rs | 3 +- 28 files changed, 274 insertions(+), 251 deletions(-) create mode 100644 lib/kernel-util/src/api.rs delete mode 100644 src/sync.rs diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs new file mode 100644 index 00000000..5ead83e6 --- /dev/null +++ b/lib/kernel-util/src/api.rs @@ -0,0 +1,7 @@ +extern "Rust" { + pub fn __acquire_irq_guard() -> bool; + pub fn __release_irq_guard(mask: bool); + + pub fn __suspend(); + pub fn __yield(); +} diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 9597c0aa..dc25946f 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -1,6 +1,8 @@ #![no_std] #![feature(maybe_uninit_slice)] +pub(crate) mod api; + pub mod sync; pub mod util; diff --git a/lib/kernel-util/src/sync.rs b/lib/kernel-util/src/sync.rs index e69de29b..c567357c 100644 --- a/lib/kernel-util/src/sync.rs +++ b/lib/kernel-util/src/sync.rs @@ -0,0 +1,207 @@ +//! Synchronization primitives +use core::{ + cell::UnsafeCell, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use crate::api; + +// use crate::arch::{Architecture, ArchitectureImpl}; + +static LOCK_HACK: AtomicBool = AtomicBool::new(false); + +/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. +/// +/// # Safety +/// +/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. +pub unsafe fn hack_locks() { + LOCK_HACK.store(true, Ordering::Release); +} + +/// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is +/// met. +pub struct SpinFence { + value: AtomicUsize, +} + +/// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation +/// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. +pub struct IrqGuard(bool); + +struct SpinlockInner { + value: UnsafeCell, + state: AtomicBool, +} + +struct SpinlockInnerGuard<'a, T> { + lock: &'a SpinlockInner, +} + +/// Spinlock implementation which prevents interrupts to avoid deadlocks when an interrupt handler +/// tries to acquire a lock taken before the IRQ fired. +pub struct IrqSafeSpinlock { + inner: SpinlockInner, +} + +/// Token type allowing safe access to the underlying data of the [IrqSafeSpinlock]. Resumes normal +/// IRQ operation (if enabled before acquiring) when the lifetime is over. +pub struct IrqSafeSpinlockGuard<'a, T> { + // Must come first to ensure the lock is dropped first and only then IRQs are re-enabled + inner: SpinlockInnerGuard<'a, T>, + _irq: IrqGuard, +} + +// Spinlock impls +impl SpinlockInner { + const fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + state: AtomicBool::new(false), + } + } + + fn lock(&self) -> SpinlockInnerGuard { + // Loop until the lock can be acquired + if LOCK_HACK.load(Ordering::Acquire) { + return SpinlockInnerGuard { lock: self }; + } + while self + .state + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } + + SpinlockInnerGuard { lock: self } + } +} + +impl<'a, T> Deref for SpinlockInnerGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.lock.value.get() } + } +} + +impl<'a, T> DerefMut for SpinlockInnerGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.lock.value.get() } + } +} + +impl<'a, T> Drop for SpinlockInnerGuard<'a, T> { + fn drop(&mut self) { + if !LOCK_HACK.load(Ordering::Acquire) { + self.lock + .state + .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) + .unwrap(); + } + } +} + +unsafe impl Sync for SpinlockInner {} +unsafe impl Send for SpinlockInner {} + +// IrqSafeSpinlock impls +impl IrqSafeSpinlock { + /// Wraps the value in a spinlock primitive + pub const fn new(value: T) -> Self { + Self { + inner: SpinlockInner::new(value), + } + } + + /// Attempts to acquire a lock. IRQs will be disabled until the lock is released. + pub fn lock(&self) -> IrqSafeSpinlockGuard { + // Disable IRQs to avoid IRQ handler trying to acquire the same lock + let irq_guard = IrqGuard::acquire(); + + // Acquire the inner lock + let inner = self.inner.lock(); + + IrqSafeSpinlockGuard { + inner, + _irq: irq_guard, + } + } + + /// Returns an unsafe reference to the inner value. + /// + /// # Safety + /// + /// Unsafe: explicitly ignores proper access sharing. + #[allow(clippy::mut_from_ref)] + pub unsafe fn grab(&self) -> &mut T { + unsafe { &mut *self.inner.value.get() } + } +} + +impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} + +// IrqGuard impls +impl IrqGuard { + /// Saves the current IRQ state and masks them + pub fn acquire() -> Self { + Self(unsafe { api::__acquire_irq_guard() }) + } +} + +impl Drop for IrqGuard { + fn drop(&mut self) { + unsafe { api::__release_irq_guard(self.0) } + } +} + +// SpinFence impls +impl SpinFence { + /// Constructs a new [SpinFence] + pub const fn new() -> Self { + Self { + value: AtomicUsize::new(0), + } + } + + /// Resets a fence back to its original state + pub fn reset(&self) { + self.value.store(0, Ordering::Release); + } + + /// "Signals" a fence, incrementing its internal counter by one + pub fn signal(&self) { + self.value.fetch_add(1, Ordering::SeqCst); + } + + /// Waits until the fence is signalled at least the amount of times specified + pub fn wait_all(&self, count: usize) { + while self.value.load(Ordering::Acquire) < count { + core::hint::spin_loop(); + } + } + + /// Waits until the fence is signalled at least once + pub fn wait_one(&self) { + self.wait_all(1); + } + + /// Returns `true` if the fence has been signalled at least the amount of times specified + pub fn try_wait_all(&self, count: usize) -> bool { + self.value.load(Ordering::Acquire) >= count + } +} diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 2ef75b42..177927d2 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -3,10 +3,10 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::interfaces::{Readable, Writeable}; -use crate::{arch::CpuMessage, panic, sync::IrqSafeSpinlock, task::sched::CpuQueue}; +use crate::{arch::CpuMessage, panic, task::sched::CpuQueue}; use super::smp::CPU_COUNT; diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 03c2dd36..5f0784d9 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -278,7 +278,7 @@ extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) { let iss = esr_el1 & 0x1FFFFFF; unsafe { - crate::sync::hack_locks(); + kernel_util::sync::hack_locks(); } dump_irrecoverable_exception(frame, ec, iss); diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 511e8792..f912f143 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -12,7 +12,7 @@ use device_api::{ }, Device, }; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::{ arch::{aarch64::IrqNumber, Architecture, CpuMessage}, @@ -23,7 +23,6 @@ use crate::{ device::{DeviceMemoryIo, RawDeviceMemoryMapping}, PhysicalAddress, }, - sync::IrqSafeSpinlock, }; use self::{gicc::Gicc, gicd::Gicd}; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 651ee988..2584b1a8 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -42,6 +42,26 @@ cfg_if! { } } +// External API for architecture specifics + +#[no_mangle] +fn __acquire_irq_guard() -> bool { + let mask = ArchitectureImpl::interrupt_mask(); + unsafe { + ArchitectureImpl::set_interrupt_mask(true); + } + mask +} + +#[no_mangle] +fn __release_irq_guard(mask: bool) { + unsafe { + ArchitectureImpl::set_interrupt_mask(mask); + } +} + +// Architecture interfaces + /// Describes messages sent from some CPU to others #[derive(Clone, Copy, PartialEq, Debug)] #[repr(u64)] diff --git a/src/debug.rs b/src/debug.rs index 34062f6c..e1021547 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -4,10 +4,12 @@ use core::fmt::{self, Arguments}; use abi::error::Error; use alloc::sync::Arc; use futures_util::Future; -use kernel_util::util::{OneTimeInit, StaticVector}; +use kernel_util::{ + sync::IrqSafeSpinlock, + util::{OneTimeInit, StaticVector}, +}; use crate::{ - sync::IrqSafeSpinlock, task::{process::Process, runtime::QueueWaker}, util::ring::RingBuffer, }; diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 31505c13..68548e22 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -5,9 +5,9 @@ use core::time::Duration; use abi::{error::Error, primitive_enum}; use alloc::{vec, vec::Vec}; use bitflags::bitflags; -use kernel_util::util::StaticVector; +use kernel_util::{sync::IrqSafeSpinlock, util::StaticVector}; -use crate::{debug::DebugSink, sync::IrqSafeSpinlock, task::runtime}; +use crate::{debug::DebugSink, task::runtime}; const CONSOLE_ROW_LEN: usize = 80; const MAX_CSI_ARGS: usize = 8; diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 53c4cb6b..9336b750 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -1,8 +1,9 @@ //! Framebuffer console driver use abi::error::Error; +use kernel_util::sync::IrqSafeSpinlock; -use crate::{debug::DebugSink, sync::IrqSafeSpinlock}; +use crate::debug::DebugSink; use super::{ console::{Attributes, ConsoleBuffer, ConsoleState, DisplayConsole}, diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 0d15b89b..843af4dd 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -4,11 +4,9 @@ use core::ops::{Index, IndexMut}; use abi::error::Error; use device_api::Device; +use kernel_util::sync::IrqSafeSpinlock; -use crate::{ - mem::{device::RawDeviceMemoryMapping, PhysicalAddress}, - sync::IrqSafeSpinlock, -}; +use crate::mem::{device::RawDeviceMemoryMapping, PhysicalAddress}; use super::{DisplayDevice, DisplayDimensions}; diff --git a/src/device/mod.rs b/src/device/mod.rs index 032006a3..7fa4d9e5 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,8 +1,7 @@ //! Device management and interfaces use device_api::{manager::DeviceManager, Device, DeviceId}; - -use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; +use kernel_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; #[cfg(target_arch = "aarch64")] pub mod devtree; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 5b75b017..53fc5225 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -2,7 +2,7 @@ use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -19,7 +19,6 @@ use crate::{ }, device_tree_driver, mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, - sync::IrqSafeSpinlock, task::process::ProcessId, }; diff --git a/src/device/tty.rs b/src/device/tty.rs index 06f00a6d..dfc80398 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -5,9 +5,9 @@ use abi::{ process::Signal, }; use device_api::serial::SerialDevice; +use kernel_util::sync::IrqSafeSpinlock; use crate::{ - sync::IrqSafeSpinlock, task::process::{Process, ProcessId}, util::ring::AsyncRing, }; diff --git a/src/main.rs b/src/main.rs index 56d2851f..34e110b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,11 +31,11 @@ #![no_main] use arch::Architecture; +use kernel_util::sync::SpinFence; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, mem::heap, - sync::SpinFence, task::{spawn_kernel_closure, Cpu}, }; @@ -55,7 +55,6 @@ pub mod init; pub mod mem; pub mod panic; pub mod proc; -pub mod sync; pub mod syscall; pub mod task; pub mod util; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 0c0a19b6..b645fe58 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,12 +1,11 @@ use core::ops::Range; use abi::error::Error; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::{ arch::{Architecture, ARCHITECTURE}, mem::{address::IntoRaw, phys::reserved::is_reserved}, - sync::IrqSafeSpinlock, }; use self::{ diff --git a/src/mem/process.rs b/src/mem/process.rs index cac37a1c..ef5e23dd 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,8 +1,9 @@ use abi::error::Error; use cfg_if::cfg_if; +use kernel_util::sync::IrqSafeSpinlock; use vmalloc::VirtualMemoryAllocator; -use crate::{mem::phys, sync::IrqSafeSpinlock}; +use crate::mem::phys; use super::{table::MapAttributes, PhysicalAddress}; diff --git a/src/panic.rs b/src/panic.rs index 312b31db..14939ecf 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,12 +2,12 @@ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use device_api::interrupt::IpiDeliveryTarget; +use kernel_util::sync::{hack_locks, SpinFence}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuMessage, ARCHITECTURE}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, - sync::{hack_locks, SpinFence}, task::{sched::CpuQueue, Cpu}, }; diff --git a/src/sync.rs b/src/sync.rs deleted file mode 100644 index dc8debf9..00000000 --- a/src/sync.rs +++ /dev/null @@ -1,211 +0,0 @@ -//! Synchronization primitives -use core::{ - cell::UnsafeCell, - ops::{Deref, DerefMut}, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, -}; - -use crate::arch::{Architecture, ArchitectureImpl}; - -static LOCK_HACK: AtomicBool = AtomicBool::new(false); - -/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. -/// -/// # Safety -/// -/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. -pub unsafe fn hack_locks() { - LOCK_HACK.store(true, Ordering::Release); -} - -/// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is -/// met. -pub struct SpinFence { - value: AtomicUsize, -} - -/// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation -/// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. -pub struct IrqGuard(bool); - -struct SpinlockInner { - value: UnsafeCell, - state: AtomicBool, -} - -struct SpinlockInnerGuard<'a, T> { - lock: &'a SpinlockInner, -} - -/// Spinlock implementation which prevents interrupts to avoid deadlocks when an interrupt handler -/// tries to acquire a lock taken before the IRQ fired. -pub struct IrqSafeSpinlock { - inner: SpinlockInner, -} - -/// Token type allowing safe access to the underlying data of the [IrqSafeSpinlock]. Resumes normal -/// IRQ operation (if enabled before acquiring) when the lifetime is over. -pub struct IrqSafeSpinlockGuard<'a, T> { - // Must come first to ensure the lock is dropped first and only then IRQs are re-enabled - inner: SpinlockInnerGuard<'a, T>, - _irq: IrqGuard, -} - -// Spinlock impls -impl SpinlockInner { - const fn new(value: T) -> Self { - Self { - value: UnsafeCell::new(value), - state: AtomicBool::new(false), - } - } - - fn lock(&self) -> SpinlockInnerGuard { - // Loop until the lock can be acquired - if LOCK_HACK.load(Ordering::Acquire) { - return SpinlockInnerGuard { lock: self }; - } - while self - .state - .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - core::hint::spin_loop(); - } - - SpinlockInnerGuard { lock: self } - } -} - -impl<'a, T> Deref for SpinlockInnerGuard<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - unsafe { &*self.lock.value.get() } - } -} - -impl<'a, T> DerefMut for SpinlockInnerGuard<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { &mut *self.lock.value.get() } - } -} - -impl<'a, T> Drop for SpinlockInnerGuard<'a, T> { - fn drop(&mut self) { - if !LOCK_HACK.load(Ordering::Acquire) { - self.lock - .state - .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) - .unwrap(); - } - } -} - -unsafe impl Sync for SpinlockInner {} -unsafe impl Send for SpinlockInner {} - -// IrqSafeSpinlock impls -impl IrqSafeSpinlock { - /// Wraps the value in a spinlock primitive - pub const fn new(value: T) -> Self { - Self { - inner: SpinlockInner::new(value), - } - } - - /// Attempts to acquire a lock. IRQs will be disabled until the lock is released. - pub fn lock(&self) -> IrqSafeSpinlockGuard { - // Disable IRQs to avoid IRQ handler trying to acquire the same lock - let irq_guard = IrqGuard::acquire(); - - // Acquire the inner lock - let inner = self.inner.lock(); - - IrqSafeSpinlockGuard { - inner, - _irq: irq_guard, - } - } - - /// Returns an unsafe reference to the inner value. - /// - /// # Safety - /// - /// Unsafe: explicitly ignores proper access sharing. - #[allow(clippy::mut_from_ref)] - pub unsafe fn grab(&self) -> &mut T { - unsafe { &mut *self.inner.value.get() } - } -} - -impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.inner.deref() - } -} - -impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.inner.deref_mut() - } -} - -// IrqGuard impls -impl IrqGuard { - /// Saves the current IRQ state and masks them - pub fn acquire() -> Self { - let mask = ArchitectureImpl::interrupt_mask(); - unsafe { - ArchitectureImpl::set_interrupt_mask(true); - } - Self(mask) - } -} - -impl Drop for IrqGuard { - fn drop(&mut self) { - unsafe { - ArchitectureImpl::set_interrupt_mask(self.0); - } - } -} - -// SpinFence impls -impl SpinFence { - /// Constructs a new [SpinFence] - pub const fn new() -> Self { - Self { - value: AtomicUsize::new(0), - } - } - - /// Resets a fence back to its original state - pub fn reset(&self) { - self.value.store(0, Ordering::Release); - } - - /// "Signals" a fence, incrementing its internal counter by one - pub fn signal(&self) { - self.value.fetch_add(1, Ordering::SeqCst); - } - - /// Waits until the fence is signalled at least the amount of times specified - pub fn wait_all(&self, count: usize) { - while self.value.load(Ordering::Acquire) < count { - core::hint::spin_loop(); - } - } - - /// Waits until the fence is signalled at least once - pub fn wait_one(&self) { - self.wait_all(1); - } - - /// Returns `true` if the fence has been signalled at least the amount of times specified - pub fn try_wait_all(&self, count: usize) -> bool { - self.value.load(Ordering::Acquire) >= count - } -} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 0e0fd087..bf401418 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -8,6 +8,7 @@ use abi::{ syscall::SyscallFunction, }; use alloc::rc::Rc; +use kernel_util::sync::IrqSafeSpinlockGuard; // use vfs::{IoContext, Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; use yggdrasil_abi::{ error::SyscallResult, @@ -19,7 +20,6 @@ use crate::{ debug::LogLevel, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo}, - sync::IrqSafeSpinlockGuard, task::{ process::{Process, ProcessId}, runtime, diff --git a/src/task/mod.rs b/src/task/mod.rs index 1f7658aa..f3c3c8c5 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -4,10 +4,10 @@ use abi::error::Error; use alloc::{string::String, sync::Arc, vec::Vec}; +use kernel_util::sync::{IrqSafeSpinlock, SpinFence}; use crate::{ arch::{Architecture, ArchitectureImpl}, - sync::{IrqSafeSpinlock, SpinFence}, task::{sched::CpuQueue, thread::Thread}, }; diff --git a/src/task/process.rs b/src/task/process.rs index da6903e9..341649b7 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -19,7 +19,7 @@ use alloc::{ vec::Vec, }; use futures_util::Future; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::{ mem::{ @@ -29,7 +29,6 @@ use crate::{ table::MapAttributes, }, proc::{self, io::ProcessIo}, - sync::IrqSafeSpinlock, task::context::TaskContextImpl, }; diff --git a/src/task/runtime/task.rs b/src/task/runtime/task.rs index c4b5b253..7a064c32 100644 --- a/src/task/runtime/task.rs +++ b/src/task/runtime/task.rs @@ -1,7 +1,6 @@ use alloc::sync::Arc; use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt}; - -use crate::sync::IrqSafeSpinlock; +use kernel_util::sync::IrqSafeSpinlock; use super::executor; diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs index 0fc0715d..43970baf 100644 --- a/src/task/runtime/task_queue.rs +++ b/src/task/runtime/task_queue.rs @@ -1,11 +1,10 @@ use abi::error::Error; use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqGuard, util::OneTimeInit}; use crate::{ arch::{Architecture, ArchitectureImpl}, - sync::IrqGuard, task::thread::Thread, }; diff --git a/src/task/runtime/waker.rs b/src/task/runtime/waker.rs index 2ca9b9f4..2f20f992 100644 --- a/src/task/runtime/waker.rs +++ b/src/task/runtime/waker.rs @@ -1,8 +1,7 @@ use core::task::Waker; use alloc::collections::VecDeque; - -use crate::sync::IrqSafeSpinlock; +use kernel_util::sync::IrqSafeSpinlock; pub struct QueueWaker { queue: IrqSafeSpinlock>, diff --git a/src/task/sched.rs b/src/task/sched.rs index f36b12b4..9de56d65 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -9,12 +9,14 @@ use alloc::{ vec::Vec, }; use cfg_if::cfg_if; -use kernel_util::util::OneTimeInit; +use kernel_util::{ + sync::{IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard}, + util::OneTimeInit, +}; use crate::{ // arch::aarch64::{context::TaskContext, cpu::Cpu}, arch::{Architecture, ArchitectureImpl}, - sync::{IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard}, task::thread::ThreadState, }; diff --git a/src/task/thread.rs b/src/task/thread.rs index 1719db56..84b91cd0 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -18,12 +18,14 @@ use alloc::{ }; use atomic_enum::atomic_enum; use futures_util::{task::ArcWake, Future}; -use kernel_util::util::OneTimeInit; +use kernel_util::{ + sync::{IrqGuard, IrqSafeSpinlock}, + util::OneTimeInit, +}; use crate::{ block, mem::{process::ProcessAddressSpace, ForeignPointer}, - sync::{IrqGuard, IrqSafeSpinlock}, task::{context::TaskContextImpl, Cpu}, }; diff --git a/src/util/ring.rs b/src/util/ring.rs index 41205e47..56e06d2c 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -6,8 +6,9 @@ use core::{ use abi::error::Error; use alloc::sync::Arc; use futures_util::Future; +use kernel_util::sync::IrqSafeSpinlock; -use crate::{sync::IrqSafeSpinlock, task::runtime::QueueWaker}; +use crate::task::runtime::QueueWaker; pub struct RingBuffer { rd: usize, From aadd97fc8e850ffc0123a59cc39674c6961bc4c0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 2 Dec 2023 20:28:43 +0200 Subject: [PATCH 097/211] fs: initial rework --- lib/hosted-tests/Cargo.toml | 8 + lib/hosted-tests/src/lib.rs | 9 + lib/kernel-util/src/sync.rs | 13 + lib/vfs/Cargo.toml | 7 + lib/vfs/src/device.rs | 80 ++++++ lib/vfs/src/file/device.rs | 84 ++++++ lib/vfs/src/file/directory.rs | 106 +++++++ lib/vfs/src/file/mod.rs | 500 ++++++++++++++++++++++++++++++++++ lib/vfs/src/file/regular.rs | 85 ++++++ lib/vfs/src/lib.rs | 18 +- lib/vfs/src/node/impls.rs | 294 ++++++++++++++++++++ lib/vfs/src/node/mod.rs | 196 +++++++++++++ lib/vfs/src/node/ops.rs | 47 ++++ lib/vfs/src/node/traits.rs | 88 ++++++ lib/vfs/src/node/tree.rs | 39 +++ lib/vfs/src/traits.rs | 17 ++ 16 files changed, 1590 insertions(+), 1 deletion(-) create mode 100644 lib/hosted-tests/Cargo.toml create mode 100644 lib/hosted-tests/src/lib.rs create mode 100644 lib/vfs/src/device.rs create mode 100644 lib/vfs/src/file/device.rs create mode 100644 lib/vfs/src/file/directory.rs create mode 100644 lib/vfs/src/file/mod.rs create mode 100644 lib/vfs/src/file/regular.rs create mode 100644 lib/vfs/src/node/impls.rs create mode 100644 lib/vfs/src/node/mod.rs create mode 100644 lib/vfs/src/node/ops.rs create mode 100644 lib/vfs/src/node/traits.rs create mode 100644 lib/vfs/src/node/tree.rs create mode 100644 lib/vfs/src/traits.rs diff --git a/lib/hosted-tests/Cargo.toml b/lib/hosted-tests/Cargo.toml new file mode 100644 index 00000000..2699c8a0 --- /dev/null +++ b/lib/hosted-tests/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hosted-tests" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/lib/hosted-tests/src/lib.rs b/lib/hosted-tests/src/lib.rs new file mode 100644 index 00000000..aa7fc159 --- /dev/null +++ b/lib/hosted-tests/src/lib.rs @@ -0,0 +1,9 @@ +#![no_std] + +#[no_mangle] +fn __acquire_irq_guard() -> bool { + false +} + +#[no_mangle] +fn __release_irq_guard(_: bool) {} diff --git a/lib/kernel-util/src/sync.rs b/lib/kernel-util/src/sync.rs index c567357c..146d8613 100644 --- a/lib/kernel-util/src/sync.rs +++ b/lib/kernel-util/src/sync.rs @@ -1,6 +1,7 @@ //! Synchronization primitives use core::{ cell::UnsafeCell, + mem, ops::{Deref, DerefMut}, sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; @@ -116,6 +117,12 @@ impl IrqSafeSpinlock { } } + #[inline] + pub fn replace(&self, value: T) -> T { + let mut lock = self.lock(); + mem::replace(&mut lock, value) + } + /// Attempts to acquire a lock. IRQs will be disabled until the lock is released. pub fn lock(&self) -> IrqSafeSpinlockGuard { // Disable IRQs to avoid IRQ handler trying to acquire the same lock @@ -141,6 +148,12 @@ impl IrqSafeSpinlock { } } +impl IrqSafeSpinlock { + pub fn cloned(&self) -> T { + self.lock().clone() + } +} + impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { type Target = T; diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 6d86b5ea..e089a224 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -6,3 +6,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../kernel-util" } + +log = "0.4.20" + +[dev-dependencies] +hosted-tests = { path = "../hosted-tests" } diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs new file mode 100644 index 00000000..0fbc1643 --- /dev/null +++ b/lib/vfs/src/device.rs @@ -0,0 +1,80 @@ +use yggdrasil_abi::error::Error; + +use crate::node::{CommonImpl, NodeRef}; + +pub trait BlockDevice { + fn read(&self, pos: u64, buf: &mut [u8]) -> Result { + Err(Error::NotImplemented) + } + fn write(&self, pos: u64, buf: &[u8]) -> Result { + Err(Error::NotImplemented) + } + + fn is_readable(&self) -> bool { + true + } + fn is_writable(&self) -> bool { + true + } + fn size(&self) -> Result; +} + +pub trait CharDevice { + fn read(&self, buf: &mut [u8]) -> Result { + Err(Error::NotImplemented) + } + fn write(&self, buf: &[u8]) -> Result { + Err(Error::NotImplemented) + } + + fn is_readable(&self) -> bool { + true + } + fn is_writable(&self) -> bool { + true + } + fn is_terminal(&self) -> bool { + false + } +} + +#[derive(Clone)] +pub struct BlockDeviceWrapper(pub(crate) &'static dyn BlockDevice); +#[derive(Clone)] +pub struct CharDeviceWrapper(pub(crate) &'static dyn CharDevice); + +impl BlockDeviceWrapper { + pub fn is_readable(&self) -> bool { + self.0.is_readable() + } + + pub fn is_writable(&self) -> bool { + self.0.is_writable() + } +} + +impl CommonImpl for BlockDeviceWrapper { + fn size(&self, _node: &NodeRef) -> Result { + self.0.size() + } +} + +impl CharDeviceWrapper { + pub fn is_terminal(&self) -> bool { + self.0.is_terminal() + } + + pub fn is_readable(&self) -> bool { + self.0.is_readable() + } + + pub fn is_writable(&self) -> bool { + self.0.is_writable() + } +} + +impl CommonImpl for CharDeviceWrapper { + fn size(&self, _node: &NodeRef) -> Result { + Ok(0) + } +} diff --git a/lib/vfs/src/file/device.rs b/lib/vfs/src/file/device.rs new file mode 100644 index 00000000..58a73826 --- /dev/null +++ b/lib/vfs/src/file/device.rs @@ -0,0 +1,84 @@ +use core::cell::Cell; + +use yggdrasil_abi::{error::Error, io::SeekFrom}; + +use crate::{ + device::{BlockDeviceWrapper, CharDeviceWrapper}, + node::NodeRef, +}; + +pub struct BlockFile { + pub(super) device: BlockDeviceWrapper, + pub(super) node: NodeRef, + pub(super) position: Cell, + pub(super) read: bool, + pub(super) write: bool, +} + +pub struct CharFile { + pub(super) device: CharDeviceWrapper, + pub(super) node: NodeRef, + pub(super) read: bool, + pub(super) write: bool, +} + +impl BlockFile { + pub fn read(&self, buf: &mut [u8]) -> Result { + let pos = self.position.get(); + let count = self.device.0.read(pos, buf)?; + self.position.set(pos + count as u64); + Ok(count) + } + + pub fn write(&self, buf: &[u8]) -> Result { + let pos = self.position.get(); + let count = self.device.0.write(pos, buf)?; + self.position.set(pos + count as u64); + Ok(count) + } + + pub fn seek(&self, from: SeekFrom) -> Result { + let pos = self.position.get(); + let newpos = match from { + SeekFrom::Current(off) => { + let newpos = i64::try_from(pos).unwrap() + off; + if newpos < 0 { + return Err(Error::InvalidArgument); + } + newpos as u64 + } + SeekFrom::Start(pos) => pos, + SeekFrom::End(off) => { + let size = i64::try_from(self.device.0.size()?).unwrap(); + let newpos = size + off; + + if newpos < 0 { + return Err(Error::InvalidArgument); + } + + newpos as u64 + } + }; + + self.position.set(newpos); + Ok(newpos) + } +} + +impl CharFile { + pub fn read(&self, buf: &mut [u8]) -> Result { + if self.read { + self.device.0.read(buf) + } else { + Err(Error::InvalidOperation) + } + } + + pub fn write(&self, buf: &[u8]) -> Result { + if self.write { + self.device.0.write(buf) + } else { + Err(Error::ReadOnly) + } + } +} diff --git a/lib/vfs/src/file/directory.rs b/lib/vfs/src/file/directory.rs new file mode 100644 index 00000000..1043a8c5 --- /dev/null +++ b/lib/vfs/src/file/directory.rs @@ -0,0 +1,106 @@ +use core::{cell::Cell, mem::MaybeUninit, str::FromStr}; + +use yggdrasil_abi::{error::Error, io::DirectoryEntry, util::FixedString}; + +use crate::node::NodeRef; + +use super::DirectoryOpenPosition; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum DirectoryCachePosition { + Dot, + DotDot, + Index(usize), +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum DirectoryPosition { + Cache(DirectoryCachePosition), + Physical(u64), +} + +pub struct DirectoryFile { + pub(super) node: NodeRef, + pub(super) position: Cell, +} + +impl DirectoryFile { + pub(super) fn read_cached( + node: &NodeRef, + mut pos: DirectoryCachePosition, + entries: &mut [MaybeUninit], + ) -> Result<(usize, DirectoryCachePosition), Error> { + let directory = node.as_directory()?; + let children = directory.children.lock(); + let mut rem = entries.len(); + let mut off = 0; + + while rem != 0 { + let entry = match pos { + DirectoryCachePosition::Dot => { + pos = DirectoryCachePosition::DotDot; + Some((FixedString::from_str(".").unwrap(), node.clone())) + } + DirectoryCachePosition::DotDot => { + pos = DirectoryCachePosition::Index(0); + Some((FixedString::from_str("..").unwrap(), node.parent())) + } + DirectoryCachePosition::Index(index) if let Some((name, node)) = children.get(index) => { + pos = DirectoryCachePosition::Index(index + 1); + Some((FixedString::from_str(name)?, node.clone())) + } + DirectoryCachePosition::Index(_) => None + }; + + let Some((name, node)) = entry else { + break; + }; + + let ty = node.ty(); + + entries[off].write(DirectoryEntry { name, ty }); + + off += 1; + rem -= 1; + } + + Ok((off, pos)) + } + + pub(super) fn read_physical( + node: &NodeRef, + pos: u64, + entries: &mut [MaybeUninit], + ) -> Result<(usize, u64), Error> { + node.read_directory(pos, entries) + } + + pub(super) fn read_entries( + &self, + entries: &mut [MaybeUninit], + ) -> Result { + let pos = self.position.get(); + let (count, pos) = match pos { + DirectoryPosition::Cache(pos) => { + let (count, pos) = DirectoryFile::read_cached(&self.node, pos, entries)?; + (count, DirectoryPosition::Cache(pos)) + } + DirectoryPosition::Physical(off) => { + let (count, pos) = DirectoryFile::read_physical(&self.node, off, entries)?; + (count, DirectoryPosition::Physical(pos)) + } + }; + self.position.set(pos); + + Ok(count) + } +} + +impl From for DirectoryPosition { + fn from(value: DirectoryOpenPosition) -> Self { + match value { + DirectoryOpenPosition::FromCache => Self::Cache(DirectoryCachePosition::Dot), + DirectoryOpenPosition::FromPhysical(off) => Self::Physical(off), + } + } +} diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs new file mode 100644 index 00000000..d7de4f45 --- /dev/null +++ b/lib/vfs/src/file/mod.rs @@ -0,0 +1,500 @@ +use core::{any::Any, cell::Cell, fmt, mem::MaybeUninit}; + +use alloc::{boxed::Box, sync::Arc}; +use yggdrasil_abi::{ + error::Error, + io::{DirectoryEntry, OpenOptions, SeekFrom}, +}; + +use crate::{ + device::{BlockDeviceWrapper, CharDeviceWrapper}, + node::NodeRef, + traits::{Read, Seek, Write}, +}; + +use self::{ + device::{BlockFile, CharFile}, + directory::DirectoryFile, + regular::RegularFile, +}; + +mod device; +mod directory; +mod regular; + +pub enum DirectoryOpenPosition { + FromPhysical(u64), + FromCache, +} + +pub type FileRef = Arc; + +// TODO some kind of a mutex instead? +pub enum File { + Directory(DirectoryFile), + Regular(RegularFile), + Block(BlockFile), + Char(CharFile), +} + +impl File { + pub fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { + let position = Cell::new(position.into()); + Arc::new(Self::Directory(DirectoryFile { node, position })) + } + + pub fn regular( + node: NodeRef, + position: u64, + instance_data: Option>, + opts: OpenOptions, + ) -> Arc { + let read = opts.contains(OpenOptions::READ); + let write = opts.contains(OpenOptions::WRITE); + + Arc::new(Self::Regular(RegularFile { + node, + read, + write, + instance_data, + position: Cell::new(position), + })) + } + + pub fn block( + device: BlockDeviceWrapper, + node: NodeRef, + opts: OpenOptions, + ) -> Result, Error> { + let read = opts.contains(OpenOptions::READ); + let write = opts.contains(OpenOptions::WRITE); + + if read && !device.is_readable() { + return Err(Error::InvalidOperation); + } + if write && !device.is_writable() { + return Err(Error::ReadOnly); + } + + Ok(Arc::new(Self::Block(BlockFile { + device, + node, + position: Cell::new(0), + read, + write, + }))) + } + + pub fn char( + device: CharDeviceWrapper, + node: NodeRef, + opts: OpenOptions, + ) -> Result, Error> { + let read = opts.contains(OpenOptions::READ); + let write = opts.contains(OpenOptions::WRITE); + + if read && !device.is_readable() { + return Err(Error::InvalidOperation); + } + if write && !device.is_writable() { + return Err(Error::ReadOnly); + } + + Ok(Arc::new(Self::Char(CharFile { + device, + node, + read, + write, + }))) + } + + pub fn read_dir(&self, entries: &mut [MaybeUninit]) -> Result { + match self { + Self::Directory(dir) => dir.read_entries(entries), + _ => Err(Error::NotADirectory), + } + } +} + +impl Read for File { + fn read(&self, buf: &mut [u8]) -> Result { + match self { + Self::Regular(file) => file.read(buf), + Self::Block(file) => file.read(buf), + Self::Char(file) => file.read(buf), + Self::Directory(_) => Err(Error::IsADirectory), + } + } +} + +impl Write for File { + fn write(&self, buf: &[u8]) -> Result { + match self { + Self::Regular(file) => file.write(buf), + Self::Block(file) => file.write(buf), + Self::Char(file) => file.write(buf), + Self::Directory(_) => Err(Error::IsADirectory), + } + } +} + +impl Seek for File { + fn tell(&self) -> Result { + match self { + Self::Regular(file) => Ok(file.position.get()), + Self::Block(file) => Ok(file.position.get()), + Self::Char(_) => Err(Error::InvalidOperation), + Self::Directory(_) => Err(Error::IsADirectory), + } + } + + fn seek(&self, from: SeekFrom) -> Result { + match self { + Self::Regular(file) => file.seek(from), + Self::Block(file) => file.seek(from), + Self::Char(_) => Err(Error::InvalidOperation), + Self::Directory(_) => Err(Error::IsADirectory), + } + } +} + +impl fmt::Debug for File { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Regular(file) => f + .debug_struct("RegularFile") + .field("position", &file.position.get()) + .field("read", &file.read) + .field("write", &file.write) + .finish_non_exhaustive(), + Self::Block(file) => f + .debug_struct("BlockFile") + .field("position", &file.position.get()) + .field("read", &file.read) + .field("write", &file.write) + .finish_non_exhaustive(), + Self::Char(file) => f + .debug_struct("CharFile") + .field("read", &file.read) + .field("write", &file.write) + .finish_non_exhaustive(), + Self::Directory(_) => f.debug_struct("DirectoryFile").finish_non_exhaustive(), + } + } +} + +#[cfg(test)] +mod tests { + use core::{any::Any, cell::RefCell, mem::MaybeUninit, str::FromStr}; + use std::sync::Arc; + + use kernel_util::sync::IrqSafeSpinlock; + use yggdrasil_abi::{ + error::Error, + io::{DirectoryEntry, FileType, OpenOptions, SeekFrom}, + util::FixedString, + }; + + use crate::{ + device::{BlockDevice, CharDevice}, + file::DirectoryOpenPosition, + node::{CommonImpl, DirectoryImpl, Node, NodeRef, RegularImpl}, + traits::{Read, Seek, Write}, + }; + + #[test] + fn physical_dir_read() { + struct D { + entries: Vec<(String, NodeRef)>, + } + struct F; + + impl CommonImpl for D {} + impl DirectoryImpl for D { + fn open(&self, _node: &NodeRef) -> Result { + Ok(DirectoryOpenPosition::FromPhysical(0)) + } + + fn read_entries( + &self, + _node: &NodeRef, + pos: u64, + entries: &mut [MaybeUninit], + ) -> Result<(usize, u64), Error> { + let pos = pos as usize; + if pos == self.entries.len() { + return Ok((0, pos as u64)); + } + + let count = core::cmp::min(entries.len(), self.entries.len() - pos); + for i in 0..count { + let (name, node) = &self.entries[i]; + let entry = DirectoryEntry { + name: FixedString::from_str(name)?, + ty: node.ty(), + }; + + entries[i].write(entry); + } + + Ok((count, (pos + count) as u64)) + } + } + + impl CommonImpl for F {} + impl RegularImpl for F {} + + let d = Node::directory(D { + entries: Vec::from_iter([ + ("f1".to_owned(), Node::regular(F)), + ("f2".to_owned(), Node::regular(F)), + ("f3".to_owned(), Node::regular(F)), + ]), + }); + + let f = d.open_directory().unwrap(); + + let mut entries = [MaybeUninit::uninit(); 16]; + let count = f.read_dir(&mut entries).unwrap(); + assert_eq!(count, 3); + + unsafe { + assert_eq!( + MaybeUninit::slice_assume_init_ref(&entries[..count]), + &[ + DirectoryEntry { + name: FixedString::from_str("f1").unwrap(), + ty: FileType::File, + }, + DirectoryEntry { + name: FixedString::from_str("f2").unwrap(), + ty: FileType::File, + }, + DirectoryEntry { + name: FixedString::from_str("f3").unwrap(), + ty: FileType::File + } + ] + ); + } + + let count = f.read_dir(&mut entries).unwrap(); + assert_eq!(count, 0); + } + + #[test] + fn cache_dir_read() { + struct D; + + impl CommonImpl for D {} + impl DirectoryImpl for D { + fn open(&self, _node: &NodeRef) -> Result { + Ok(DirectoryOpenPosition::FromCache) + } + } + + let d = Node::directory(D); + let child = Node::directory(D); + + d.add_child("child1", child).unwrap(); + + let f = d.open_directory().unwrap(); + + let mut entries = [MaybeUninit::uninit(); 16]; + let count = f.read_dir(&mut entries).unwrap(); + assert_eq!(count, 3); + unsafe { + assert_eq!( + MaybeUninit::slice_assume_init_ref(&entries[..count]), + &[ + DirectoryEntry { + name: FixedString::from_str(".").unwrap(), + ty: FileType::Directory + }, + DirectoryEntry { + name: FixedString::from_str("..").unwrap(), + ty: FileType::Directory + }, + DirectoryEntry { + name: FixedString::from_str("child1").unwrap(), + ty: FileType::Directory + } + ] + ); + } + + let count = f.read_dir(&mut entries).unwrap(); + assert_eq!(count, 0); + } + + #[test] + fn file_read_write() { + struct F { + data: Arc>>, + } + + impl CommonImpl for F { + fn size(&self, _node: &NodeRef) -> Result { + Ok(self.data.borrow().len()) + } + } + impl RegularImpl for F { + fn open( + &self, + _node: &NodeRef, + _opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Ok((0, None)) + } + + fn read( + &self, + _node: &NodeRef, + _instance: Option<&Box>, + pos: u64, + buf: &mut [u8], + ) -> Result { + let pos = pos as usize; + let data = self.data.borrow(); + if pos >= data.len() { + return Ok(0); + } + let count = core::cmp::min(data.len() - pos, buf.len()); + buf[..count].copy_from_slice(&data[pos..pos + count]); + Ok(count) + } + + fn write( + &self, + _node: &NodeRef, + _instance: Option<&Box>, + pos: u64, + buf: &[u8], + ) -> Result { + let pos = pos as usize; + let mut data = self.data.borrow_mut(); + data.resize(pos + buf.len(), 0); + data[pos..pos + buf.len()].copy_from_slice(buf); + Ok(buf.len()) + } + } + + let data = Arc::new(RefCell::new(vec![])); + let node = Node::regular(F { data: data.clone() }); + let file = node.open(OpenOptions::READ | OpenOptions::WRITE).unwrap(); + let mut buf = [0; 512]; + + assert_eq!(&*data.borrow(), &[]); + assert_eq!(file.tell().unwrap(), 0); + + assert_eq!(file.write(b"Hello").unwrap(), 5); + assert_eq!(file.tell().unwrap(), 5); + assert_eq!(&*data.borrow(), b"Hello"); + + assert_eq!(file.seek(SeekFrom::End(-2)).unwrap(), 3); + assert_eq!(file.tell().unwrap(), 3); + + assert_eq!(file.write(b"123456").unwrap(), 6); + assert_eq!(file.tell().unwrap(), 9); + assert_eq!(&*data.borrow(), b"Hel123456"); + + assert_eq!(file.seek(SeekFrom::Start(2)).unwrap(), 2); + assert_eq!(file.read(&mut buf).unwrap(), 7); + assert_eq!(file.tell().unwrap(), 9); + assert_eq!(&buf[..7], b"l123456"); + } + + #[test] + fn block_device() { + struct B { + data: Arc>>, + } + + impl BlockDevice for B { + fn read(&self, pos: u64, buf: &mut [u8]) -> Result { + let data = self.data.lock(); + let pos = pos as usize; + if pos >= data.len() { + return Ok(0); + } + + let count = core::cmp::min(data.len() - pos, buf.len()); + buf[..count].copy_from_slice(&data[pos..pos + count]); + Ok(count) + } + + fn write(&self, pos: u64, buf: &[u8]) -> Result { + let mut data = self.data.lock(); + let pos = pos as usize; + if pos >= data.len() { + return Ok(0); + } + + let count = core::cmp::min(data.len() - pos, buf.len()); + data[pos..pos + count].copy_from_slice(&buf[..count]); + Ok(count) + } + + fn size(&self) -> Result { + Ok(self.data.lock().len()) + } + } + + let vec = vec![0; 1024]; + let state = Arc::new(IrqSafeSpinlock::new(vec)); + let data = state.clone(); + let dev = Box::leak(Box::new(B { data })); + let mut buf = [0; 512]; + + let node = Node::block(dev); + + let file = node.open(OpenOptions::READ | OpenOptions::WRITE).unwrap(); + + assert_eq!(file.seek(SeekFrom::End(0)).unwrap(), 1024); + assert_eq!(file.write(b"12345").unwrap(), 0); + assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); + assert_eq!(file.write(b"12345").unwrap(), 5); + assert_eq!(&state.lock()[..6], b"12345\0"); + + assert_eq!(file.read(&mut buf).unwrap(), 512); + assert_eq!(buf, [0; 512]); + assert_eq!(file.seek(SeekFrom::Start(2)).unwrap(), 2); + assert_eq!(file.read(&mut buf[..8]).unwrap(), 8); + assert_eq!(&buf[..8], b"345\0\0\0\0\0"); + } + + #[test] + fn char_device() { + struct C; + + impl CharDevice for C { + fn read(&self, buf: &mut [u8]) -> Result { + buf.fill(b'@'); + Ok(buf.len()) + } + + fn is_writable(&self) -> bool { + false + } + } + + static DEV: C = C; + + let node = Node::char(&DEV); + let mut buf = [0; 512]; + + let err = node.open(OpenOptions::WRITE).unwrap_err(); + assert_eq!(err, Error::ReadOnly); + + let file = node.open(OpenOptions::READ).unwrap(); + assert_eq!(file.tell().unwrap_err(), Error::InvalidOperation); + assert_eq!( + file.seek(SeekFrom::Start(10)).unwrap_err(), + Error::InvalidOperation + ); + assert_eq!(file.read(&mut buf).unwrap(), 512); + assert_eq!(buf, [b'@'; 512]); + + assert_eq!(file.write(b"1234").unwrap_err(), Error::ReadOnly); + } +} diff --git a/lib/vfs/src/file/regular.rs b/lib/vfs/src/file/regular.rs new file mode 100644 index 00000000..b5e64798 --- /dev/null +++ b/lib/vfs/src/file/regular.rs @@ -0,0 +1,85 @@ +use alloc::boxed::Box; +use core::{any::Any, cell::Cell}; +use yggdrasil_abi::{error::Error, io::SeekFrom}; + +use crate::node::NodeRef; + +pub struct RegularFile { + pub(super) node: NodeRef, + pub(super) read: bool, + pub(super) write: bool, + pub(super) instance_data: Option>, + pub(super) position: Cell, +} + +impl RegularFile { + pub fn read(&self, buf: &mut [u8]) -> Result { + if !self.read { + return Err(Error::InvalidFile); + } + let reg = self.node.as_regular()?; + let pos = self.position.get(); + let count = reg.read(&self.node, self.instance_data.as_ref(), pos, buf)?; + self.position.set(pos + count as u64); + Ok(count) + } + + pub fn write(&self, buf: &[u8]) -> Result { + if !self.write { + return Err(Error::InvalidFile); + } + let reg = self.node.as_regular()?; + let pos = self.position.get(); + let count = reg.write(&self.node, self.instance_data.as_ref(), pos, buf)?; + self.position.set(pos + count as u64); + Ok(count) + } + + // TODO should seek beyond the end of a read-only file be allowed? + pub fn seek(&self, from: SeekFrom) -> Result { + let pos = self.position.get(); + let newpos = match from { + SeekFrom::Current(off) => { + let newpos = i64::try_from(pos).unwrap() + off; + if newpos < 0 { + return Err(Error::InvalidArgument); + } + newpos as u64 + } + SeekFrom::Start(pos) => pos, + SeekFrom::End(off) => { + let reg = self.node.as_regular()?; + let size = i64::try_from(reg.size(&self.node)?).unwrap(); + let newpos = size + off; + + if newpos < 0 { + return Err(Error::InvalidArgument); + } + + newpos as u64 + } + }; + + self.position.set(newpos); + Ok(newpos) + } +} + +impl Drop for RegularFile { + fn drop(&mut self) { + let reg = match self.node.as_regular() { + Ok(reg) => reg, + Err(err) => { + log::warn!( + "RegularFile::Drop: self.node.as_regular() failed: {:?}", + err + ); + return; + } + }; + + if let Err(err) = reg.close(&self.node, self.instance_data.as_ref()) { + log::warn!("RegularFile::Drop: close() failed: {:?}", err); + } + } +} diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 0c9ac1ac..1d0d95dc 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1 +1,17 @@ -#![no_std] +#![cfg_attr(not(test), no_std)] +#![feature(trait_upcasting, if_let_guard, maybe_uninit_slice, trait_alias)] + +#[cfg(test)] +extern crate hosted_tests; + +extern crate alloc; + +pub(crate) mod device; +pub(crate) mod file; +pub(crate) mod node; +pub(crate) mod traits; + +pub use device::{BlockDevice, CharDevice}; +pub use file::{DirectoryOpenPosition, File, FileRef}; +pub use node::{impls, Node, NodeRef}; +pub use traits::{Read, Seek, Write}; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs new file mode 100644 index 00000000..e3a2f6d2 --- /dev/null +++ b/lib/vfs/src/node/impls.rs @@ -0,0 +1,294 @@ +use core::{any::Any, cell::RefCell, marker::PhantomData, str::FromStr}; + +use alloc::{boxed::Box, string::ToString, sync::Arc, vec::Vec}; + +use yggdrasil_abi::{error::Error, io::OpenOptions}; + +use super::{CommonImpl, Node, NodeRef, RegularImpl}; + +pub trait SliceRead { + fn read_slice(&self, offset: usize, buf: &mut [u8]) -> usize; +} + +pub trait SliceWrite { + fn write_slice(&mut self, offset: usize, buf: &[u8]) -> usize; +} + +trait IntoInstanceData { + fn into_instance_data(&self) -> Vec; +} + +impl> SliceRead for T { + fn read_slice(&self, pos: usize, buf: &mut [u8]) -> usize { + let value = self.as_ref(); + if pos >= value.len() { + return 0; + } + let count = core::cmp::min(value.len() - pos, buf.len()); + buf[..count].copy_from_slice(&value[pos..pos + count]); + count + } +} + +impl SliceWrite for Vec { + fn write_slice(&mut self, offset: usize, buf: &[u8]) -> usize { + if offset + buf.len() > self.len() { + self.resize(offset + buf.len(), 0); + } + self[offset..offset + buf.len()].copy_from_slice(buf); + buf.len() + } +} + +impl IntoInstanceData for T { + fn into_instance_data(&self) -> Vec { + self.to_string().as_bytes().to_vec() + } +} + +pub trait ReadFn = Fn() -> Result; +pub trait WriteFn = Fn(T) -> Result<(), Error>; + +enum FnNodeData { + Read(Vec), + Write(RefCell>), +} + +pub struct ReadOnlyFnNode> { + read: R, + _pd: PhantomData, +} + +pub struct FnNode, W: WriteFn> { + read: R, + write: W, + _pd: PhantomData, +} + +impl + 'static> ReadOnlyFnNode { + pub fn new(read: R) -> NodeRef { + Node::regular(Self::new_impl(read)) + } + + pub const fn new_impl(read: R) -> Self { + Self { + read, + _pd: PhantomData, + } + } +} + +impl> CommonImpl for ReadOnlyFnNode { + fn size(&self, _node: &NodeRef) -> Result { + Ok(0) + } +} + +impl> RegularImpl for ReadOnlyFnNode { + fn open( + &self, + _node: &NodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + if opts.contains(OpenOptions::WRITE) { + return Err(Error::ReadOnly); + } + let t = (self.read)()?; + Ok((0, Some(Box::new(t.into_instance_data())))) + } + + fn read( + &self, + _node: &NodeRef, + instance: Option<&Box>, + pos: u64, + buf: &mut [u8], + ) -> Result { + let value = instance.unwrap().downcast_ref::>().unwrap(); + Ok(value.read_slice(pos as usize, buf)) + } + + fn write( + &self, + _node: &NodeRef, + _instance: Option<&Box>, + _pos: u64, + _buf: &[u8], + ) -> Result { + Err(Error::ReadOnly) + } +} + +// Read-write FnNode + +impl FnNodeData { + fn write() -> Self { + Self::Write(RefCell::new(Vec::new())) + } + + fn read(value: T) -> Self { + Self::Read(value.into_instance_data()) + } + + fn as_read(&self) -> Result<&Vec, Error> { + match self { + Self::Read(r) => Ok(r), + Self::Write(_) => Err(Error::InvalidOperation), + } + } + + fn as_write(&self) -> Result<&RefCell>, Error> { + match self { + Self::Write(w) => Ok(w), + Self::Read(_) => Err(Error::InvalidOperation), + } + } +} + +impl + 'static, W: WriteFn + 'static> + FnNode +{ + pub fn new(read: R, write: W) -> NodeRef { + Node::regular(Self::new_impl(read, write)) + } + + pub const fn new_impl(read: R, write: W) -> Self { + Self { + read, + write, + _pd: PhantomData, + } + } +} + +impl, W: WriteFn> CommonImpl for FnNode { + fn size(&self, _node: &NodeRef) -> Result { + Ok(0) + } +} + +impl, W: WriteFn> RegularImpl for FnNode { + fn open( + &self, + _node: &NodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + if opts.contains(OpenOptions::READ | OpenOptions::WRITE) { + Err(Error::InvalidOperation) + } else if opts.contains(OpenOptions::WRITE) { + Ok((0, Some(Box::new(FnNodeData::write())))) + } else if opts.contains(OpenOptions::READ) { + let t = (self.read)()?; + Ok((0, Some(Box::new(FnNodeData::read(t))))) + } else { + Err(Error::InvalidOperation) + } + } + + fn close(&self, _node: &NodeRef, instance: Option<&Box>) -> Result<(), Error> { + if let Ok(write) = instance + .unwrap() + .downcast_ref::() + .unwrap() + .as_write() + { + let write = write.borrow(); + // Flush write + let str = core::str::from_utf8(write.as_ref()) + .map_err(|_| Error::InvalidArgument)? + .trim(); + let t = T::from_str(str).map_err(|_| Error::InvalidArgument)?; + + (self.write)(t)?; + } + Ok(()) + } + + fn read( + &self, + _node: &NodeRef, + instance: Option<&Box>, + pos: u64, + buf: &mut [u8], + ) -> Result { + let instance = instance.unwrap().downcast_ref::().unwrap(); + Ok(instance.as_read()?.read_slice(pos as usize, buf)) + } + + fn write( + &self, + _node: &NodeRef, + instance: Option<&Box>, + pos: u64, + buf: &[u8], + ) -> Result { + let instance = instance.unwrap().downcast_ref::().unwrap(); + Ok(instance.as_write()?.borrow_mut().write_slice(pos as _, buf)) + } +} + +pub fn const_value_node(value: T) -> NodeRef { + ReadOnlyFnNode::new(move || Ok(value.clone())) +} + +pub fn value_node(value: T) -> NodeRef { + let rd_state = Arc::new(RefCell::new(value)); + let wr_state = rd_state.clone(); + + FnNode::new( + move || Ok(rd_state.borrow().clone()), + move |t| { + *wr_state.borrow_mut() = t; + Ok(()) + }, + ) +} + +#[cfg(test)] +mod tests { + use yggdrasil_abi::io::OpenOptions; + + use crate::{ + node::impls::{const_value_node, value_node}, + traits::{Read, Seek, Write}, + }; + + #[test] + fn read_only_fn_node() { + let node = const_value_node("abcdef"); + let file = node.open(OpenOptions::READ).unwrap(); + let mut buf = [0; 512]; + + assert_eq!(file.tell().unwrap(), 0); + + assert_eq!(file.read(&mut buf[..3]).unwrap(), 3); + assert_eq!(&buf[..3], b"abc"); + + assert_eq!(file.read(&mut buf).unwrap(), 3); + assert_eq!(&buf[..3], b"def"); + } + + #[test] + fn fn_node() { + let node = value_node(1234); + let mut buf = [0; 512]; + + // Try read the value + let file = node.open(OpenOptions::READ).unwrap(); + assert_eq!(file.tell().unwrap(), 0); + assert_eq!(file.read(&mut buf).unwrap(), 4); + assert_eq!(&buf[..4], b"1234"); + + // Try write the value + let file = node.open(OpenOptions::WRITE).unwrap(); + assert_eq!(file.tell().unwrap(), 0); + assert_eq!(file.write(b"654321").unwrap(), 6); + drop(file); + + // Try read the value again + let file = node.open(OpenOptions::READ).unwrap(); + assert_eq!(file.tell().unwrap(), 0); + assert_eq!(file.read(&mut buf).unwrap(), 6); + assert_eq!(&buf[..6], b"654321"); + } +} diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs new file mode 100644 index 00000000..6f39003a --- /dev/null +++ b/lib/vfs/src/node/mod.rs @@ -0,0 +1,196 @@ +use core::any::Any; + +use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; +use kernel_util::sync::IrqSafeSpinlock; +use yggdrasil_abi::{error::Error, io::FileType}; + +pub mod impls; +mod traits; + +// Node is implemented in the following modules +mod ops; +mod tree; + +pub use traits::{CommonImpl, DirectoryImpl, RegularImpl}; + +use crate::device::{BlockDevice, BlockDeviceWrapper, CharDevice, CharDeviceWrapper}; + +pub type NodeRef = Arc; + +pub(crate) struct DirectoryData { + pub(crate) imp: Box, + pub(crate) children: IrqSafeSpinlock>, +} + +enum NodeImpl { + Regular(Box), + Directory(DirectoryData), + Block(BlockDeviceWrapper), + Char(CharDeviceWrapper), +} + +pub struct Node { + data: NodeImpl, + parent: IrqSafeSpinlock>, +} + +impl Node { + pub fn directory(data: T) -> NodeRef { + let data = NodeImpl::Directory(DirectoryData { + imp: Box::new(data), + children: IrqSafeSpinlock::new(Vec::new()), + }); + Arc::new(Self { + data, + parent: IrqSafeSpinlock::new(None), + }) + } + + pub fn regular(data: T) -> NodeRef { + Arc::new(Self { + data: NodeImpl::Regular(Box::new(data)), + parent: IrqSafeSpinlock::new(None), + }) + } + + pub fn block(device: &'static dyn BlockDevice) -> NodeRef { + Arc::new(Self { + data: NodeImpl::Block(BlockDeviceWrapper(device)), + parent: IrqSafeSpinlock::new(None), + }) + } + + pub fn char(device: &'static dyn CharDevice) -> NodeRef { + Arc::new(Self { + data: NodeImpl::Char(CharDeviceWrapper(device)), + parent: IrqSafeSpinlock::new(None), + }) + } + + pub fn data_as_any(&self) -> &dyn Any { + match &self.data { + NodeImpl::Directory(dir) => dir.imp.as_any(), + NodeImpl::Regular(imp) => imp.as_any(), + NodeImpl::Block(w) => w.as_any(), + NodeImpl::Char(w) => w.as_any(), + } + } + + pub fn data_as_common(&self) -> &dyn CommonImpl { + match &self.data { + NodeImpl::Directory(dir) => dir.imp.as_ref(), + NodeImpl::Regular(imp) => imp.as_ref(), + NodeImpl::Block(w) => w, + NodeImpl::Char(w) => w, + } + } + + pub fn data_as_ref(&self) -> &T { + self.data_as_any().downcast_ref().unwrap() + } + + pub fn ty(&self) -> FileType { + match &self.data { + NodeImpl::Regular(_) => FileType::File, + NodeImpl::Directory(_) => FileType::Directory, + NodeImpl::Block(_) => FileType::Block, + NodeImpl::Char(_) => FileType::Char, + } + } + + pub fn is_directory(&self) -> bool { + matches!(&self.data, NodeImpl::Directory(_)) + } + + pub fn is_terminal(&self) -> bool { + if let NodeImpl::Char(dev) = &self.data { + dev.is_terminal() + } else { + false + } + } + + pub(crate) fn as_directory(&self) -> Result<&DirectoryData, Error> { + match &self.data { + NodeImpl::Directory(dir) => Ok(dir), + _ => Err(Error::InvalidFile), + } + } + + pub(crate) fn as_regular(&self) -> Result<&dyn RegularImpl, Error> { + match &self.data { + NodeImpl::Regular(imp) => Ok(imp.as_ref()), + _ => Err(Error::InvalidFile), + } + } +} + +#[cfg(test)] +mod tests { + use core::any::Any; + use std::sync::Arc; + + use super::{CommonImpl, DirectoryImpl, Node, RegularImpl}; + + struct DummyDirectory; + struct DummyFile; + + impl CommonImpl for DummyDirectory {} + impl DirectoryImpl for DummyDirectory {} + + impl CommonImpl for DummyFile {} + impl RegularImpl for DummyFile {} + + #[test] + fn dir_cache_add() { + let d1 = Node::directory(DummyDirectory); + let d2 = Node::directory(DummyDirectory); + let f1 = Node::regular(DummyFile); + + assert!(Arc::ptr_eq(&f1.parent(), &f1)); + assert_eq!(d1.children_len().unwrap(), 0); + + d1.add_child("f1", f1.clone()).unwrap(); + assert!(Arc::ptr_eq(&f1.parent(), &d1)); + assert_eq!(d1.children_len().unwrap(), 1); + + assert!(Arc::ptr_eq(&d2.parent(), &d2)); + d2.add_child("d1", d1.clone()).unwrap(); + assert!(Arc::ptr_eq(&f1.parent(), &d1)); + assert!(Arc::ptr_eq(&d1.parent(), &d2)); + assert_eq!(d1.children_len().unwrap(), 1); + assert_eq!(d2.children_len().unwrap(), 1); + } + + #[test] + fn in_mem_dir_size_coherence() { + let d = Node::directory(DummyDirectory); + + for i in 0..10 { + let name = format!("f{}", i); + let node = Node::regular(DummyFile); + + d.add_child(name, node).unwrap(); + assert_eq!(d.size().unwrap(), d.children_len().unwrap()); + } + } + + #[test] + fn data_any() { + struct AnyData { + value: u32, + } + + impl CommonImpl for AnyData { + fn as_any(&self) -> &dyn Any { + self + } + } + impl DirectoryImpl for AnyData {} + + let d = Node::directory(AnyData { value: 1234 }); + let r = d.data_as_ref::(); + + assert_eq!(r.value, 1234); + } +} diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs new file mode 100644 index 00000000..934ad043 --- /dev/null +++ b/lib/vfs/src/node/ops.rs @@ -0,0 +1,47 @@ +use core::mem::MaybeUninit; + +use yggdrasil_abi::{ + error::Error, + io::{DirectoryEntry, OpenOptions}, +}; + +use crate::file::{File, FileRef}; + +use super::{Node, NodeImpl, NodeRef}; + +impl Node { + pub fn open(self: &NodeRef, opts: OpenOptions) -> Result { + match &self.data { + NodeImpl::Regular(imp) => { + let (pos, instance) = imp.open(self, opts)?; + Ok(File::regular(self.clone(), pos, instance, opts)) + } + NodeImpl::Block(dev) => File::block(dev.clone(), self.clone(), opts), + NodeImpl::Char(dev) => File::char(dev.clone(), self.clone(), opts), + // TODO: maybe merge open_directory and open? + NodeImpl::Directory(_) => todo!(), + } + } + + // Directory + + pub fn open_directory(self: &NodeRef) -> Result { + let dir = self.as_directory()?; + let pos = dir.imp.open(self)?; + Ok(File::directory(self.clone(), pos)) + } + + pub fn read_directory( + self: &NodeRef, + pos: u64, + entries: &mut [MaybeUninit], + ) -> Result<(usize, u64), Error> { + self.as_directory()?.imp.read_entries(self, pos, entries) + } + + // Common + + pub fn size(self: &NodeRef) -> Result { + self.data_as_common().size(self) + } +} diff --git a/lib/vfs/src/node/traits.rs b/lib/vfs/src/node/traits.rs new file mode 100644 index 00000000..7081f49a --- /dev/null +++ b/lib/vfs/src/node/traits.rs @@ -0,0 +1,88 @@ +use alloc::boxed::Box; +use core::{any::Any, mem::MaybeUninit}; + +use yggdrasil_abi::{ + error::Error, + io::{DirectoryEntry, FileAttr, OpenOptions}, +}; + +use crate::file::DirectoryOpenPosition; + +use super::NodeRef; + +#[allow(unused)] +pub trait CommonImpl { + fn as_any(&self) -> &dyn Any { + unimplemented!(); + } + + fn metadata(&self, node: &NodeRef) -> Result { + Err(Error::NotImplemented) + } + fn size(&self, node: &NodeRef) -> Result { + // TODO this only works for dirs + if node.is_directory() { + node.children_len() + } else { + Err(Error::NotImplemented) + } + } +} + +#[allow(unused)] +pub trait RegularImpl: CommonImpl { + fn open( + &self, + node: &NodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + Err(Error::NotImplemented) + } + + fn close(&self, node: &NodeRef, instance: Option<&Box>) -> Result<(), Error> { + Ok(()) + } + + fn read( + &self, + node: &NodeRef, + instance: Option<&Box>, + pos: u64, + buf: &mut [u8], + ) -> Result { + Err(Error::NotImplemented) + } + fn write( + &self, + node: &NodeRef, + instance: Option<&Box>, + pos: u64, + buf: &[u8], + ) -> Result { + Err(Error::NotImplemented) + } +} + +#[allow(unused)] +pub trait DirectoryImpl: CommonImpl { + fn open(&self, node: &NodeRef) -> Result { + Err(Error::NotImplemented) + } + + fn read_entries( + &self, + node: &NodeRef, + pos: u64, + entries: &mut [MaybeUninit], + ) -> Result<(usize, u64), Error> { + Err(Error::NotImplemented) + } + + fn lookup(&self, node: &NodeRef, name: &str) -> Result { + Err(Error::NotImplemented) + } + + fn len(&self, node: &NodeRef) -> Result { + Err(Error::NotImplemented) + } +} diff --git a/lib/vfs/src/node/tree.rs b/lib/vfs/src/node/tree.rs new file mode 100644 index 00000000..693585e8 --- /dev/null +++ b/lib/vfs/src/node/tree.rs @@ -0,0 +1,39 @@ +use yggdrasil_abi::error::Error; + +use alloc::string::String; + +use super::{Node, NodeRef}; + +impl Node { + pub fn parent(self: &NodeRef) -> NodeRef { + self.parent.lock().as_ref().unwrap_or(self).clone() + } + + pub fn children_len(&self) -> Result { + let directory = self.as_directory()?; + Ok(directory.children.lock().len()) + } + + pub fn add_child>( + self: &NodeRef, + name: S, + child: NodeRef, + ) -> Result<(), Error> { + let name = name.into(); + let directory = self.as_directory()?; + let mut children = directory.children.lock(); + + // TODO check if an entry already exists with such name + + // if children.contains_key(&name) { + // log::warn!("Directory cache already contains an entry: {:?}", name); + // return Err(Error::AlreadyExists); + // } + + assert!(child.parent.replace(Some(self.clone())).is_none()); + children.push((name, child)); + // children.insert(name, child); + + Ok(()) + } +} diff --git a/lib/vfs/src/traits.rs b/lib/vfs/src/traits.rs new file mode 100644 index 00000000..f5a7525a --- /dev/null +++ b/lib/vfs/src/traits.rs @@ -0,0 +1,17 @@ +use yggdrasil_abi::{error::Error, io::SeekFrom}; + +pub trait Read { + fn read(&self, buf: &mut [u8]) -> Result; +} + +pub trait Write { + fn write(&self, buf: &[u8]) -> Result; +} + +pub trait Seek { + fn seek(&self, from: SeekFrom) -> Result; + + fn tell(&self) -> Result { + self.seek(SeekFrom::Current(0)) + } +} From fbe05c1d4d68d098e3d92a978abed9c7219f27f0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 3 Dec 2023 14:52:42 +0200 Subject: [PATCH 098/211] fs: path abi, ioctx impl --- lib/vfs/Cargo.toml | 2 +- lib/vfs/src/device.rs | 6 +- lib/vfs/src/file/mod.rs | 46 +++-- lib/vfs/src/ioctx.rs | 390 +++++++++++++++++++++++++++++++++++++ lib/vfs/src/lib.rs | 10 +- lib/vfs/src/node/impls.rs | 69 ++++++- lib/vfs/src/node/mod.rs | 179 ++++++++++++++--- lib/vfs/src/node/ops.rs | 57 +++++- lib/vfs/src/node/traits.rs | 43 ++-- lib/vfs/src/node/tree.rs | 1 - lib/vfs/src/path.rs | 0 11 files changed, 729 insertions(+), 74 deletions(-) create mode 100644 lib/vfs/src/ioctx.rs create mode 100644 lib/vfs/src/path.rs diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index e089a224..e4d187d8 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } kernel-util = { path = "../kernel-util" } log = "0.4.20" diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs index 0fbc1643..218a24f3 100644 --- a/lib/vfs/src/device.rs +++ b/lib/vfs/src/device.rs @@ -16,7 +16,7 @@ pub trait BlockDevice { fn is_writable(&self) -> bool { true } - fn size(&self) -> Result; + fn size(&self) -> Result; } pub trait CharDevice { @@ -54,7 +54,7 @@ impl BlockDeviceWrapper { } impl CommonImpl for BlockDeviceWrapper { - fn size(&self, _node: &NodeRef) -> Result { + fn size(&self, _node: &NodeRef) -> Result { self.0.size() } } @@ -74,7 +74,7 @@ impl CharDeviceWrapper { } impl CommonImpl for CharDeviceWrapper { - fn size(&self, _node: &NodeRef) -> Result { + fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index d7de4f45..36d9282d 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -114,6 +114,15 @@ impl File { _ => Err(Error::NotADirectory), } } + + pub fn node(&self) -> Option<&NodeRef> { + match self { + Self::Directory(file) => Some(&file.node), + Self::Regular(file) => Some(&file.node), + Self::Block(file) => Some(&file.node), + Self::Char(file) => Some(&file.node), + } + } } impl Read for File { @@ -198,7 +207,7 @@ mod tests { use crate::{ device::{BlockDevice, CharDevice}, file::DirectoryOpenPosition, - node::{CommonImpl, DirectoryImpl, Node, NodeRef, RegularImpl}, + node::{CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl}, traits::{Read, Seek, Write}, }; @@ -244,13 +253,16 @@ mod tests { impl CommonImpl for F {} impl RegularImpl for F {} - let d = Node::directory(D { - entries: Vec::from_iter([ - ("f1".to_owned(), Node::regular(F)), - ("f2".to_owned(), Node::regular(F)), - ("f3".to_owned(), Node::regular(F)), - ]), - }); + let d = Node::directory( + D { + entries: Vec::from_iter([ + ("f1".to_owned(), Node::regular(F, NodeFlags::empty())), + ("f2".to_owned(), Node::regular(F, NodeFlags::empty())), + ("f3".to_owned(), Node::regular(F, NodeFlags::empty())), + ]), + }, + NodeFlags::empty(), + ); let f = d.open_directory().unwrap(); @@ -293,8 +305,8 @@ mod tests { } } - let d = Node::directory(D); - let child = Node::directory(D); + let d = Node::directory(D, NodeFlags::empty()); + let child = Node::directory(D, NodeFlags::empty()); d.add_child("child1", child).unwrap(); @@ -334,8 +346,8 @@ mod tests { } impl CommonImpl for F { - fn size(&self, _node: &NodeRef) -> Result { - Ok(self.data.borrow().len()) + fn size(&self, _node: &NodeRef) -> Result { + Ok(self.data.borrow().len() as _) } } impl RegularImpl for F { @@ -380,7 +392,7 @@ mod tests { } let data = Arc::new(RefCell::new(vec![])); - let node = Node::regular(F { data: data.clone() }); + let node = Node::regular(F { data: data.clone() }, NodeFlags::empty()); let file = node.open(OpenOptions::READ | OpenOptions::WRITE).unwrap(); let mut buf = [0; 512]; @@ -435,8 +447,8 @@ mod tests { Ok(count) } - fn size(&self) -> Result { - Ok(self.data.lock().len()) + fn size(&self) -> Result { + Ok(self.data.lock().len() as _) } } @@ -446,7 +458,7 @@ mod tests { let dev = Box::leak(Box::new(B { data })); let mut buf = [0; 512]; - let node = Node::block(dev); + let node = Node::block(dev, NodeFlags::empty()); let file = node.open(OpenOptions::READ | OpenOptions::WRITE).unwrap(); @@ -480,7 +492,7 @@ mod tests { static DEV: C = C; - let node = Node::char(&DEV); + let node = Node::char(&DEV, NodeFlags::empty()); let mut buf = [0; 512]; let err = node.open(OpenOptions::WRITE).unwrap_err(); diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs new file mode 100644 index 00000000..dec75333 --- /dev/null +++ b/lib/vfs/src/ioctx.rs @@ -0,0 +1,390 @@ +use alloc::borrow::ToOwned; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, FileType, GroupId, OpenOptions, UserId}, + path::{Path, PathBuf}, +}; + +use crate::{ + node::{CreateInfo, SymlinkData}, + FileRef, NodeRef, +}; + +pub struct IoContext { + uid: UserId, + gid: GroupId, + umask: FileMode, + cwd_node: NodeRef, + cwd_path: PathBuf, + root: NodeRef, +} + +impl IoContext { + pub fn new(root: NodeRef) -> Self { + Self { + uid: UserId::root(), + gid: GroupId::root(), + umask: FileMode::new(0o022), + cwd_node: root.clone(), + cwd_path: PathBuf::new(), + root, + } + } + + pub fn root(&self) -> &NodeRef { + &self.root + } + + pub fn cwd(&self) -> &NodeRef { + &self.cwd_node + } + + pub fn set_cwd>(&mut self, path: P) -> Result<(), Error> { + let path = path.as_ref(); + if !path.is_absolute() { + todo!(); + } + let node = Self::_find(self.root.clone(), path.trim_start_separators(), true, true)?; + if !node.is_directory() { + return Err(Error::NotADirectory); + } + self.cwd_node = node; + self.cwd_path = path.to_owned(); + Ok(()) + } + + pub fn cwd_path(&self) -> &Path { + self.cwd_path.as_ref() + } + + pub fn mount>(&mut self, target: P, fs_root: NodeRef) -> Result<(), Error> { + let target = target.as_ref(); + if !target.is_absolute() { + todo!(); + } + let target_node = Self::_find( + self.root.clone(), + target.trim_start_separators(), + true, + false, + )?; + + target_node.set_mountpoint_target(fs_root) + } + + pub fn open>( + &mut self, + at: Option, + path: P, + opts: OpenOptions, + mode: FileMode, + ) -> Result { + let path = path.as_ref(); + let node = match self.find(at.clone(), path, true, true) { + Ok(node) => node, + Err(Error::DoesNotExist) if opts.contains(OpenOptions::CREATE) => { + // let create_mode = mode & !self.umask; + let (parent, name) = path.split_right(); + let parent = self.find(at, parent, true, true)?; + let create_info = CreateInfo { + name: name.into(), + mode: mode & !self.umask, + uid: self.uid, + gid: self.gid, + ty: FileType::File, + }; + parent.create(&create_info)? + } + Err(err) => return Err(err), + }; + + node.open(opts) + } + + pub fn find>( + &mut self, + at: Option, + path: P, + follow_links: bool, + follow_mount: bool, + ) -> Result { + let mut path = path.as_ref(); + let at = if path.is_absolute() { + path = path.trim_start_separators(); + self.root.clone() + } else if let Some(at) = at { + at + } else { + self.cwd_node.clone() + }; + + Self::_find(at, path, follow_links, follow_mount) + } + + fn _resolve_link(at: &NodeRef, link: &SymlinkData) -> Result { + // If the filesystem itself implements direct target resolution, use that + match link.imp.target(at) { + Err(Error::NotImplemented) => {} + other => return other, + } + + // If that call wasn't implemented, check if the target was cached + if let Some((_, target)) = link.target.lock().as_ref() { + return Ok(target.clone()); + } + + // If no cache is present, read the link's data and resolve the path + let _path = link.imp.read_to_string()?; + todo!() + } + + fn _resolve(mut at: NodeRef, follow_links: bool, follow_mount: bool) -> Result { + loop { + if follow_mount && let Some(target) = at.mountpoint_target() { + at = target.clone(); + continue; + } + + if follow_links && let Ok(link) = at.as_symlink() { + // Resolve the link + at = Self::_resolve_link(&at, link)?; + continue; + } + + break Ok(at); + } + } + + fn _find( + mut at: NodeRef, + path: &Path, + follow_links: bool, + follow_mount: bool, + ) -> Result { + let mut element; + let mut rest = path; + + loop { + (element, rest) = rest.split_left(); + + if !at.is_directory() { + return Err(Error::NotADirectory); + } + + match element { + Path::PARENT_NAME => { + at = at.parent(); + } + Path::SELF_NAME => {} + _ => break, + } + } + + at = Self::_resolve(at, follow_links, follow_mount)?; + + if !at.is_directory() { + return Err(Error::NotADirectory); + } + + if element.is_empty() && rest.is_empty() { + return Ok(at); + } + + let node = at.lookup_or_load(element)?; + let node = Self::_resolve(node, follow_links, follow_mount)?; + + if rest.is_empty() { + Ok(node) + } else { + Self::_find(node, rest, follow_links, follow_mount) + } + } +} + +#[cfg(test)] +mod tests { + use alloc::sync::Arc; + use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, + path::Path, + }; + + use crate::{ + impls::{const_value_node, f_symlink, mdir}, + Read, + }; + + use super::IoContext; + + #[test] + fn cwd() { + let d1_f1 = const_value_node("dir1-file1"); + let d1 = mdir([("f1", d1_f1.clone())]); + let f1 = const_value_node("file1"); + let root = mdir([("f1", f1.clone()), ("d1", d1.clone())]); + + let mut ioctx = IoContext::new(root.clone()); + + assert_eq!(ioctx.cwd_path(), Path::empty()); + + let node = ioctx.find(None, "f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &f1)); + + let node = ioctx.find(None, "d1/f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &d1_f1)); + + let node = ioctx.find(None, "d1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &d1)); + + ioctx.set_cwd("/d1").unwrap(); + assert_eq!(ioctx.cwd_path(), Path::from_str("/d1")); + + let err = ioctx.find(None, "d1", true, true).unwrap_err(); + assert_eq!(err, Error::DoesNotExist); + + let node = ioctx.find(None, "f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &d1_f1)); + + let node = ioctx.find(None, "../d1/f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &d1_f1)); + + let node = ioctx.find(None, "/f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &f1)); + + let node = ioctx.find(None, "../f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &f1)); + + let node = ioctx.find(None, "..", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &root)); + } + + #[test] + fn mount_resolution() { + let root2_f1 = const_value_node("root2-file1"); + let root2 = mdir([("f1", root2_f1.clone())]); + let root1_f1 = const_value_node("root1-file1"); + let root1 = mdir([("f1", root1_f1.clone())]); + + let mut ioctx = IoContext::new(root1.clone()); + + let node = ioctx.find(None, "/f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &root1_f1)); + + ioctx.mount("/", root2.clone()).unwrap(); + + let node = ioctx.find(None, "/f1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &root2_f1)); + + let node = ioctx.find(None, "/f1", true, false).unwrap(); + assert!(Arc::ptr_eq(&node, &root1_f1)); + + let node = ioctx.find(None, "/", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &root2)); + + let node = ioctx.find(None, "/", true, false).unwrap(); + assert!(Arc::ptr_eq(&node, &root1)); + } + + #[test] + fn symlink_resolution() { + let f1 = const_value_node("file1"); + let l1 = f_symlink(f1.clone()); + let root = mdir([("l1", l1.clone())]); + + let mut ioctx = IoContext::new(root.clone()); + + // No follow + let node = ioctx.find(None, "/l1", false, true).unwrap(); + assert!(Arc::ptr_eq(&node, &l1)); + + // Follow + let node = ioctx.find(None, "/l1", true, true).unwrap(); + assert!(Arc::ptr_eq(&node, &f1)); + } + + #[test] + fn open_or_create() { + let root_d1_f1 = const_value_node("dir1/file1"); + let root_f1 = const_value_node("file1"); + let root_d1 = mdir([("f1", root_d1_f1.clone())]); + let root = mdir([("f1", root_f1.clone()), ("d1", root_d1.clone())]); + + let mut ioctx = IoContext::new(root.clone()); + let mut buf = [0; 512]; + + let file = ioctx + .open(None, "/d1/f1", OpenOptions::READ, FileMode::empty()) + .unwrap(); + assert!(Arc::ptr_eq(&file.node().unwrap(), &root_d1_f1)); + assert_eq!(file.read(&mut buf).unwrap(), 10); + assert_eq!(&buf[..10], b"dir1/file1"); + drop(file); + + let file = ioctx + .open( + None, + "/d1/f1", + OpenOptions::READ | OpenOptions::CREATE, + FileMode::empty(), + ) + .unwrap(); + assert!(Arc::ptr_eq(&file.node().unwrap(), &root_d1_f1)); + assert_eq!(file.read(&mut buf).unwrap(), 10); + assert_eq!(&buf[..10], b"dir1/file1"); + drop(file); + + let err = ioctx + .open( + None, + "/d1/f2", + OpenOptions::WRITE | OpenOptions::CREATE, + FileMode::empty(), + ) + .unwrap_err(); + assert_eq!(err, Error::ReadOnly); + } + + #[test] + fn find_tests() { + let root_d1_f1 = const_value_node("dir1/file1"); + let root_f1 = const_value_node("file1"); + let root_d1 = mdir([("f1", root_d1_f1.clone())]); + let root = mdir([("f1", root_f1.clone()), ("d1", root_d1.clone())]); + + let mut ioctx = IoContext::new(root.clone()); + + // Ok paths + let node = ioctx.find(None, Path::empty(), false, false).unwrap(); + assert!(Arc::ptr_eq(&node, &root)); + let node = ioctx.find(None, Path::from_str("/"), false, false).unwrap(); + assert!(Arc::ptr_eq(&node, &root)); + + // Weird paths + let node = ioctx + .find(None, Path::from_str("////.////./"), false, false) + .unwrap(); + assert!(Arc::ptr_eq(&node, &root)); + let node = ioctx + .find(None, Path::from_str("/../.././//."), false, false) + .unwrap(); + assert!(Arc::ptr_eq(&node, &root)); + + let node = ioctx + .find(None, Path::from_str("/f1"), false, false) + .unwrap(); + assert!(Arc::ptr_eq(&node, &root_f1)); + let node = ioctx + .find(None, Path::from_str("/d1/f1"), false, false) + .unwrap(); + assert!(Arc::ptr_eq(&node, &root_d1_f1)); + let node = ioctx + .find(None, Path::from_str("/d1/../d1/./f1"), false, false) + .unwrap(); + assert!(Arc::ptr_eq(&node, &root_d1_f1)); + let node = ioctx + .find(None, Path::from_str("/d1/.."), false, false) + .unwrap(); + assert!(Arc::ptr_eq(&node, &root)); + } +} diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 1d0d95dc..c1c6bb7c 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,5 +1,11 @@ #![cfg_attr(not(test), no_std)] -#![feature(trait_upcasting, if_let_guard, maybe_uninit_slice, trait_alias)] +#![feature( + trait_upcasting, + if_let_guard, + maybe_uninit_slice, + trait_alias, + let_chains +)] #[cfg(test)] extern crate hosted_tests; @@ -8,7 +14,9 @@ extern crate alloc; pub(crate) mod device; pub(crate) mod file; +pub(crate) mod ioctx; pub(crate) mod node; +pub(crate) mod path; pub(crate) mod traits; pub use device::{BlockDevice, CharDevice}; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index e3a2f6d2..8fdba6ba 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -1,10 +1,19 @@ use core::{any::Any, cell::RefCell, marker::PhantomData, str::FromStr}; -use alloc::{boxed::Box, string::ToString, sync::Arc, vec::Vec}; +use alloc::{ + boxed::Box, + string::{String, ToString}, + sync::Arc, + vec::Vec, +}; use yggdrasil_abi::{error::Error, io::OpenOptions}; -use super::{CommonImpl, Node, NodeRef, RegularImpl}; +use crate::DirectoryOpenPosition; + +use super::{ + CommonImpl, CreateInfo, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl, SymlinkImpl, +}; pub trait SliceRead { fn read_slice(&self, offset: usize, buf: &mut [u8]) -> usize; @@ -65,9 +74,17 @@ pub struct FnNode, W: WriteFn> { _pd: PhantomData, } +pub struct MemoryDirectory; +pub struct FixedSymlink { + target: NodeRef, +} + impl + 'static> ReadOnlyFnNode { pub fn new(read: R) -> NodeRef { - Node::regular(Self::new_impl(read)) + Node::regular( + Self::new_impl(read), + NodeFlags::IN_MEMORY_PROPS | NodeFlags::IN_MEMORY_SIZE, + ) } pub const fn new_impl(read: R) -> Self { @@ -79,7 +96,7 @@ impl + 'static> ReadOnlyFnNode { } impl> CommonImpl for ReadOnlyFnNode { - fn size(&self, _node: &NodeRef) -> Result { + fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } @@ -149,7 +166,10 @@ impl + 'static, W: WriteFn + 's FnNode { pub fn new(read: R, write: W) -> NodeRef { - Node::regular(Self::new_impl(read, write)) + Node::regular( + Self::new_impl(read, write), + NodeFlags::IN_MEMORY_PROPS | NodeFlags::IN_MEMORY_SIZE, + ) } pub const fn new_impl(read: R, write: W) -> Self { @@ -162,7 +182,7 @@ impl + 'static, W: WriteFn + 's } impl, W: WriteFn> CommonImpl for FnNode { - fn size(&self, _node: &NodeRef) -> Result { + fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } @@ -227,6 +247,28 @@ impl, W: WriteFn> RegularImpl for FnNode< } } +// In-memory directory + +impl CommonImpl for MemoryDirectory {} +impl DirectoryImpl for MemoryDirectory { + fn open(&self, _node: &NodeRef) -> Result { + Ok(DirectoryOpenPosition::FromCache) + } + + fn create(&self, _parent: &NodeRef, _info: &CreateInfo) -> Result { + Err(Error::ReadOnly) + } +} + +// In-memory fixed symlink + +impl CommonImpl for FixedSymlink {} +impl SymlinkImpl for FixedSymlink { + fn target(&self, _node: &NodeRef) -> Result { + Ok(self.target.clone()) + } +} + pub fn const_value_node(value: T) -> NodeRef { ReadOnlyFnNode::new(move || Ok(value.clone())) } @@ -244,6 +286,21 @@ pub fn value_node(value: T) -> NodeRef ) } +pub fn mdir, I: IntoIterator>(it: I) -> NodeRef { + let dir = Node::directory( + MemoryDirectory, + NodeFlags::IN_MEMORY_PROPS | NodeFlags::IN_MEMORY_SIZE, + ); + for (name, node) in it { + dir.add_child(name, node).unwrap(); + } + dir +} + +pub fn f_symlink(target: NodeRef) -> NodeRef { + Node::symlink(FixedSymlink { target }, NodeFlags::empty()) +} + #[cfg(test)] mod tests { use yggdrasil_abi::io::OpenOptions; diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index 6f39003a..c57c6cb5 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -1,8 +1,16 @@ -use core::any::Any; +use core::{ + any::Any, + cell::{Cell, RefCell}, + fmt, +}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use kernel_util::sync::IrqSafeSpinlock; -use yggdrasil_abi::{error::Error, io::FileType}; +use yggdrasil_abi::{ + bitflags, + error::Error, + io::{FileAttr, FileMode, FileType, GroupId, UserId}, +}; pub mod impls; mod traits; @@ -11,60 +19,117 @@ mod traits; mod ops; mod tree; -pub use traits::{CommonImpl, DirectoryImpl, RegularImpl}; +pub use traits::{CommonImpl, DirectoryImpl, RegularImpl, SymlinkImpl}; use crate::device::{BlockDevice, BlockDeviceWrapper, CharDevice, CharDeviceWrapper}; pub type NodeRef = Arc; +bitflags! { + pub struct NodeFlags: u32 { + const IN_MEMORY_PROPS: bit 0; + const IN_MEMORY_SIZE: bit 1; + } +} + +#[derive(Debug, Clone)] +pub struct CreateInfo { + pub name: String, + pub uid: UserId, + pub gid: GroupId, + pub mode: FileMode, + pub ty: FileType, +} + pub(crate) struct DirectoryData { pub(crate) imp: Box, + pub(crate) mountpoint: IrqSafeSpinlock>, pub(crate) children: IrqSafeSpinlock>, } +pub(crate) struct SymlinkData { + // Cached symlink target with the literal path + pub(crate) target: IrqSafeSpinlock>, + pub(crate) imp: Box, +} + enum NodeImpl { Regular(Box), Directory(DirectoryData), Block(BlockDeviceWrapper), Char(CharDeviceWrapper), + Symlink(SymlinkData), +} + +#[derive(Clone, Copy)] +pub struct Metadata { + pub uid: UserId, + pub gid: GroupId, + pub mode: FileMode, +} + +#[derive(Default)] +struct PropertyCache { + metadata: Metadata, + size: Option, } pub struct Node { data: NodeImpl, + flags: NodeFlags, + props: IrqSafeSpinlock, parent: IrqSafeSpinlock>, } +impl Default for Metadata { + fn default() -> Metadata { + Metadata { + uid: UserId::root(), + gid: GroupId::root(), + mode: FileMode::new(0), + } + } +} + impl Node { - pub fn directory(data: T) -> NodeRef { - let data = NodeImpl::Directory(DirectoryData { - imp: Box::new(data), - children: IrqSafeSpinlock::new(Vec::new()), - }); + fn new(data: NodeImpl, flags: NodeFlags) -> NodeRef { Arc::new(Self { data, + flags, + props: IrqSafeSpinlock::new(PropertyCache::default()), parent: IrqSafeSpinlock::new(None), }) } - pub fn regular(data: T) -> NodeRef { - Arc::new(Self { - data: NodeImpl::Regular(Box::new(data)), - parent: IrqSafeSpinlock::new(None), - }) + pub fn directory(data: T, flags: NodeFlags) -> NodeRef { + let data = NodeImpl::Directory(DirectoryData { + imp: Box::new(data), + mountpoint: IrqSafeSpinlock::new(None), + children: IrqSafeSpinlock::new(Vec::new()), + }); + Self::new(data, flags) } - pub fn block(device: &'static dyn BlockDevice) -> NodeRef { - Arc::new(Self { - data: NodeImpl::Block(BlockDeviceWrapper(device)), - parent: IrqSafeSpinlock::new(None), - }) + pub fn regular(data: T, flags: NodeFlags) -> NodeRef { + Self::new(NodeImpl::Regular(Box::new(data)), flags) } - pub fn char(device: &'static dyn CharDevice) -> NodeRef { - Arc::new(Self { - data: NodeImpl::Char(CharDeviceWrapper(device)), - parent: IrqSafeSpinlock::new(None), - }) + pub fn block(device: &'static dyn BlockDevice, flags: NodeFlags) -> NodeRef { + Self::new(NodeImpl::Block(BlockDeviceWrapper(device)), flags) + } + + pub fn char(device: &'static dyn CharDevice, flags: NodeFlags) -> NodeRef { + Self::new(NodeImpl::Char(CharDeviceWrapper(device)), flags) + } + + pub fn symlink(data: T, flags: NodeFlags) -> NodeRef { + Self::new( + NodeImpl::Symlink(SymlinkData { + target: IrqSafeSpinlock::new(None), + imp: Box::new(data), + }), + flags, + ) } pub fn data_as_any(&self) -> &dyn Any { @@ -73,6 +138,7 @@ impl Node { NodeImpl::Regular(imp) => imp.as_any(), NodeImpl::Block(w) => w.as_any(), NodeImpl::Char(w) => w.as_any(), + NodeImpl::Symlink(w) => w.imp.as_any(), } } @@ -82,6 +148,7 @@ impl Node { NodeImpl::Regular(imp) => imp.as_ref(), NodeImpl::Block(w) => w, NodeImpl::Char(w) => w, + NodeImpl::Symlink(w) => w.imp.as_ref(), } } @@ -95,6 +162,7 @@ impl Node { NodeImpl::Directory(_) => FileType::Directory, NodeImpl::Block(_) => FileType::Block, NodeImpl::Char(_) => FileType::Char, + NodeImpl::Symlink(_) => FileType::Symlink, } } @@ -110,6 +178,36 @@ impl Node { } } + pub(crate) fn mountpoint_target(&self) -> Option { + match &self.data { + NodeImpl::Directory(dir) => dir.mountpoint.lock().clone(), + _ => None, + } + } + + pub(crate) fn set_mountpoint_target(self: &NodeRef, target: NodeRef) -> Result<(), Error> { + let directory = self.as_directory()?; + let mut mountpoint = directory.mountpoint.lock(); + let mut target_parent_lock = target.parent.lock(); + + if mountpoint.is_some() { + // TODO Busy + todo!(); + } + if target_parent_lock.is_some() { + // TODO mount a filesystem more than once? + return Err(Error::AlreadyExists); + } + if !target.is_directory() { + return Err(Error::NotADirectory); + } + + mountpoint.replace(target.clone()); + target_parent_lock.replace(self.clone()); + + Ok(()) + } + pub(crate) fn as_directory(&self) -> Result<&DirectoryData, Error> { match &self.data { NodeImpl::Directory(dir) => Ok(dir), @@ -123,6 +221,25 @@ impl Node { _ => Err(Error::InvalidFile), } } + + pub(crate) fn as_symlink(&self) -> Result<&SymlinkData, Error> { + match &self.data { + NodeImpl::Symlink(imp) => Ok(imp), + _ => Err(Error::InvalidFile), + } + } +} + +impl fmt::Debug for Node { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.data { + NodeImpl::Directory(_) => f.debug_struct("DirectoryNode").finish_non_exhaustive(), + NodeImpl::Regular(_) => f.debug_struct("RegularNode").finish_non_exhaustive(), + NodeImpl::Char(_) => f.debug_struct("CharNode").finish_non_exhaustive(), + NodeImpl::Block(_) => f.debug_struct("BlockNode").finish_non_exhaustive(), + NodeImpl::Symlink(_) => f.debug_struct("SymlinkNode").finish_non_exhaustive(), + } + } } #[cfg(test)] @@ -130,6 +247,8 @@ mod tests { use core::any::Any; use std::sync::Arc; + use crate::node::NodeFlags; + use super::{CommonImpl, DirectoryImpl, Node, RegularImpl}; struct DummyDirectory; @@ -143,9 +262,9 @@ mod tests { #[test] fn dir_cache_add() { - let d1 = Node::directory(DummyDirectory); - let d2 = Node::directory(DummyDirectory); - let f1 = Node::regular(DummyFile); + let d1 = Node::directory(DummyDirectory, NodeFlags::empty()); + let d2 = Node::directory(DummyDirectory, NodeFlags::empty()); + let f1 = Node::regular(DummyFile, NodeFlags::empty()); assert!(Arc::ptr_eq(&f1.parent(), &f1)); assert_eq!(d1.children_len().unwrap(), 0); @@ -164,14 +283,14 @@ mod tests { #[test] fn in_mem_dir_size_coherence() { - let d = Node::directory(DummyDirectory); + let d = Node::directory(DummyDirectory, NodeFlags::IN_MEMORY_SIZE); for i in 0..10 { let name = format!("f{}", i); - let node = Node::regular(DummyFile); + let node = Node::regular(DummyFile, NodeFlags::empty()); d.add_child(name, node).unwrap(); - assert_eq!(d.size().unwrap(), d.children_len().unwrap()); + assert_eq!(d.size().unwrap(), d.children_len().unwrap() as u64); } } @@ -188,7 +307,7 @@ mod tests { } impl DirectoryImpl for AnyData {} - let d = Node::directory(AnyData { value: 1234 }); + let d = Node::directory(AnyData { value: 1234 }, NodeFlags::empty()); let r = d.data_as_ref::(); assert_eq!(r.value, 1234); diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs index 934ad043..735c31a1 100644 --- a/lib/vfs/src/node/ops.rs +++ b/lib/vfs/src/node/ops.rs @@ -7,7 +7,7 @@ use yggdrasil_abi::{ use crate::file::{File, FileRef}; -use super::{Node, NodeImpl, NodeRef}; +use super::{CreateInfo, Metadata, Node, NodeFlags, NodeImpl, NodeRef}; impl Node { pub fn open(self: &NodeRef, opts: OpenOptions) -> Result { @@ -19,7 +19,7 @@ impl Node { NodeImpl::Block(dev) => File::block(dev.clone(), self.clone(), opts), NodeImpl::Char(dev) => File::char(dev.clone(), self.clone(), opts), // TODO: maybe merge open_directory and open? - NodeImpl::Directory(_) => todo!(), + NodeImpl::Directory(_) | NodeImpl::Symlink(_) => todo!(), } } @@ -39,9 +39,58 @@ impl Node { self.as_directory()?.imp.read_entries(self, pos, entries) } + pub fn lookup_or_load(self: &NodeRef, name: &str) -> Result { + let dir = self.as_directory()?; + let children = dir.children.lock(); + + if let Some((_, node)) = children.iter().find(|(name_, _)| name_ == name) { + return Ok(node.clone()); + } + + // TODO lookup in real FS + + Err(Error::DoesNotExist) + } + + pub fn create(self: &NodeRef, info: &CreateInfo) -> Result { + let directory = self.as_directory()?; + let node = directory.imp.create(self, info)?; + // Attach the created node to the directory in memory cache + self.add_child(&info.name, node.clone())?; + Ok(node) + } + // Common - pub fn size(self: &NodeRef) -> Result { - self.data_as_common().size(self) + pub fn metadata(self: &NodeRef) -> Result { + if self.flags.contains(NodeFlags::IN_MEMORY_PROPS) { + let props = self.props.lock(); + return Ok(props.metadata); + } + + self.data_as_common().metadata(self) + } + + pub fn size(self: &NodeRef) -> Result { + // Try to fetch the size from the cache + let mut props = self.props.lock(); + + if let Some(size) = props.size { + return Ok(size); + } + + if self.flags.contains(NodeFlags::IN_MEMORY_SIZE) { + if let Ok(dir) = self.as_directory() { + return Ok(dir.children.lock().len() as _); + } + + Err(Error::NotImplemented) + } else { + // Fetch the size from the node + let size = self.data_as_common().size(self)?; + props.size = Some(size); + + Ok(size) + } } } diff --git a/lib/vfs/src/node/traits.rs b/lib/vfs/src/node/traits.rs index 7081f49a..21a94f13 100644 --- a/lib/vfs/src/node/traits.rs +++ b/lib/vfs/src/node/traits.rs @@ -1,14 +1,14 @@ -use alloc::boxed::Box; +use alloc::{boxed::Box, string::String}; use core::{any::Any, mem::MaybeUninit}; use yggdrasil_abi::{ error::Error, - io::{DirectoryEntry, FileAttr, OpenOptions}, + io::{DirectoryEntry, OpenOptions}, }; use crate::file::DirectoryOpenPosition; -use super::NodeRef; +use super::{CreateInfo, Metadata, NodeRef}; #[allow(unused)] pub trait CommonImpl { @@ -16,16 +16,12 @@ pub trait CommonImpl { unimplemented!(); } - fn metadata(&self, node: &NodeRef) -> Result { + fn metadata(&self, node: &NodeRef) -> Result { Err(Error::NotImplemented) } - fn size(&self, node: &NodeRef) -> Result { - // TODO this only works for dirs - if node.is_directory() { - node.children_len() - } else { - Err(Error::NotImplemented) - } + + fn size(&self, node: &NodeRef) -> Result { + Err(Error::NotImplemented) } } @@ -78,6 +74,10 @@ pub trait DirectoryImpl: CommonImpl { Err(Error::NotImplemented) } + fn create(&self, parent: &NodeRef, info: &CreateInfo) -> Result { + Err(Error::NotImplemented) + } + fn lookup(&self, node: &NodeRef, name: &str) -> Result { Err(Error::NotImplemented) } @@ -86,3 +86,24 @@ pub trait DirectoryImpl: CommonImpl { Err(Error::NotImplemented) } } + +#[allow(unused)] +pub trait SymlinkImpl: CommonImpl { + fn target(&self, node: &NodeRef) -> Result { + Err(Error::NotImplemented) + } + + fn read_to_string(&self) -> Result { + let mut data = Box::new([0; 512]); + let len = self.read_link(data.as_mut())?; + if len == data.len() { + return Err(Error::InvalidFile); + } + let str = core::str::from_utf8(&data[..len]).map_err(|_| Error::InvalidFile)?; + Ok(String::from(str)) + } + + fn read_link(&self, buf: &mut [u8]) -> Result { + Err(Error::NotImplemented) + } +} diff --git a/lib/vfs/src/node/tree.rs b/lib/vfs/src/node/tree.rs index 693585e8..04e2f541 100644 --- a/lib/vfs/src/node/tree.rs +++ b/lib/vfs/src/node/tree.rs @@ -32,7 +32,6 @@ impl Node { assert!(child.parent.replace(Some(self.clone())).is_none()); children.push((name, child)); - // children.insert(name, child); Ok(()) } diff --git a/lib/vfs/src/path.rs b/lib/vfs/src/path.rs new file mode 100644 index 00000000..e69de29b From e431d49ffba299fb796d45cee166398a0f3fd38b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 5 Dec 2023 10:27:56 +0200 Subject: [PATCH 099/211] fs: integrate the new vfs with the kernel --- lib/memfs/Cargo.toml | 2 + lib/memfs/src/dir.rs | 89 ++-- lib/memfs/src/file.rs | 71 ++-- lib/memfs/src/lib.rs | 536 +++++++++---------------- lib/memfs/src/tar.rs | 9 +- lib/vfs/src/device.rs | 43 +- lib/vfs/src/file/mod.rs | 44 +- lib/vfs/src/ioctx.rs | 278 ++++++++++++- lib/vfs/src/lib.rs | 9 +- lib/vfs/src/node/access.rs | 78 ++++ lib/vfs/src/node/impls.rs | 160 ++++++-- lib/vfs/src/node/mod.rs | 80 +++- lib/vfs/src/node/ops.rs | 111 ++++- lib/vfs/src/node/traits.rs | 36 +- lib/vfs/src/node/tree.rs | 8 + lib/vfs/src/traits.rs | 20 + src/arch/aarch64/boot/mod.rs | 7 +- src/arch/aarch64/mod.rs | 12 +- src/arch/x86_64/acpi.rs | 3 +- src/arch/x86_64/apic/ioapic.rs | 2 +- src/arch/x86_64/cpu.rs | 3 +- src/arch/x86_64/mod.rs | 3 +- src/arch/x86_64/peripherals/i8253.rs | 2 +- src/arch/x86_64/peripherals/ps2/mod.rs | 14 +- src/arch/x86_64/peripherals/serial.rs | 2 +- src/device/bus/pci/mod.rs | 6 +- src/device/serial/pl011.rs | 48 ++- src/device/tty.rs | 68 ++-- src/fs/devfs.rs | 120 ++++-- src/fs/mod.rs | 4 +- src/fs/sysfs.rs | 175 +------- src/init.rs | 84 ++-- src/main.rs | 6 +- src/proc/elf.rs | 18 +- src/proc/exec.rs | 28 +- src/proc/io.rs | 59 ++- src/proc/mod.rs | 2 +- src/syscall/mod.rs | 270 +++++++++++-- src/task/process.rs | 39 +- 39 files changed, 1620 insertions(+), 929 deletions(-) create mode 100644 lib/vfs/src/node/access.rs diff --git a/lib/memfs/Cargo.toml b/lib/memfs/Cargo.toml index 4369a818..ccfe38f0 100644 --- a/lib/memfs/Cargo.toml +++ b/lib/memfs/Cargo.toml @@ -8,7 +8,9 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "../vfs" } + static_assertions = "1.1.0" +log = "0.4.20" [features] default = [] diff --git a/lib/memfs/src/dir.rs b/lib/memfs/src/dir.rs index 421a7908..ead383a0 100644 --- a/lib/memfs/src/dir.rs +++ b/lib/memfs/src/dir.rs @@ -1,76 +1,35 @@ -use core::{any::Any, cell::RefCell, marker::PhantomData}; +use core::marker::PhantomData; -use alloc::boxed::Box; +use vfs::{CommonImpl, DirectoryImpl, DirectoryOpenPosition, Node, NodeFlags, NodeRef}; +use yggdrasil_abi::{error::Error, io::FileType}; -use vfs::{CreateInfo, Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; -use yggdrasil_abi::{ - error::Error, - io::{FileAttr, FileMode, FileType, OpenOptions}, -}; - -use crate::{block::BlockAllocator, bvec::BVec, file::FileNode}; +use crate::{block::BlockAllocator, file::FileNode}; pub(crate) struct DirectoryNode { - uid: u32, - gid: u32, - mode: FileMode, _pd: PhantomData, } -impl VnodeImpl for DirectoryNode { - fn create(&self, at: &VnodeRef, info: &CreateInfo) -> Result { - let child = Vnode::new(info.name, info.kind); - match info.kind { - VnodeKind::Directory => child.set_data(Box::new(Self { - uid: info.uid, - gid: info.gid, - mode: info.mode, - _pd: PhantomData, - })), - VnodeKind::Regular => child.set_data(Box::new(FileNode { - data: RefCell::new(BVec::::new()), - uid: info.uid, - gid: info.gid, - mode: info.mode, - })), +impl DirectoryNode { + pub fn new() -> NodeRef { + Node::directory( + Self { _pd: PhantomData }, + NodeFlags::IN_MEMORY_SIZE | NodeFlags::IN_MEMORY_PROPS, + ) + } +} + +impl CommonImpl for DirectoryNode {} + +impl DirectoryImpl for DirectoryNode { + fn open(&self, _node: &NodeRef) -> Result { + Ok(DirectoryOpenPosition::FromCache) + } + + fn create_node(&self, _parent: &NodeRef, ty: FileType) -> Result { + match ty { + FileType::File => Ok(FileNode::::new()), + FileType::Directory => Ok(DirectoryNode::::new()), _ => todo!(), } - child.set_fs(at.fs().unwrap()); - Ok(child) - } - - fn open( - &self, - _node: &VnodeRef, - _opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - Ok((DIR_POSITION_FROM_CACHE, None)) - } - - fn remove(&self, _at: &VnodeRef, _name: &str) -> Result<(), Error> { - Ok(()) - } - - fn size(&self, node: &VnodeRef) -> Result { - Ok(node.children().len() as u64) - } - - fn metadata(&self, _node: &VnodeRef) -> Result { - Ok(FileAttr { - size: 0, - mode: self.mode, - ty: FileType::Directory, - }) - } -} - -impl DirectoryNode { - pub fn new(uid: u32, gid: u32, mode: FileMode) -> Self { - Self { - uid, - gid, - mode, - _pd: PhantomData, - } } } diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs index 8fe15a13..c1b7bd27 100644 --- a/lib/memfs/src/file.rs +++ b/lib/memfs/src/file.rs @@ -1,27 +1,44 @@ use core::{any::Any, cell::RefCell}; use alloc::boxed::Box; -use vfs::{VnodeImpl, VnodeRef}; -use yggdrasil_abi::{ - error::Error, - io::{FileAttr, FileMode, FileType, OpenOptions}, -}; +use vfs::{CommonImpl, Node, NodeFlags, NodeRef, RegularImpl}; +use yggdrasil_abi::{error::Error, io::OpenOptions}; use crate::{block::BlockAllocator, bvec::BVec}; pub(crate) struct FileNode { + // TODO IrqSafeSpinlock pub(crate) data: RefCell>, - pub(crate) uid: u32, - pub(crate) gid: u32, - pub(crate) mode: FileMode, } -impl VnodeImpl for FileNode { +impl FileNode { + pub fn new() -> NodeRef { + Node::regular( + Self { + data: RefCell::new(BVec::new()), + }, + NodeFlags::IN_MEMORY_PROPS, + ) + } +} + +impl CommonImpl for FileNode { + fn as_any(&self) -> &dyn Any { + self + } + + fn size(&self, _node: &NodeRef) -> Result { + Ok(self.data.borrow().size() as u64) + } +} + +impl RegularImpl for FileNode { fn open( &self, - _node: &VnodeRef, + _node: &NodeRef, opts: OpenOptions, ) -> Result<(u64, Option>), Error> { + // TODO provide APPEND by vfs driver instead if opts.contains(OpenOptions::APPEND) { Ok((self.data.borrow().size() as u64, None)) } else { @@ -29,33 +46,27 @@ impl VnodeImpl for FileNode { } } - fn close(&self, _node: &VnodeRef) -> Result<(), Error> { - Ok(()) - } - fn read( &self, - _node: &VnodeRef, + _node: &NodeRef, + _instance: Option<&Box>, pos: u64, - _inner: Option<&mut Box>, - data: &mut [u8], + buf: &mut [u8], ) -> Result { - self.data.borrow().read(pos, data) + self.data.borrow().read(pos, buf) } - fn write(&self, _node: &VnodeRef, pos: u64, data: &[u8]) -> Result { - self.data.borrow_mut().write(pos, data) + fn write( + &self, + _node: &NodeRef, + _instance: Option<&Box>, + pos: u64, + buf: &[u8], + ) -> Result { + self.data.borrow_mut().write(pos, buf) } - fn size(&self, _node: &VnodeRef) -> Result { - Ok(self.data.borrow().size() as u64) - } - - fn metadata(&self, _node: &VnodeRef) -> Result { - Ok(FileAttr { - size: self.data.borrow().size() as u64, - mode: self.mode, - ty: FileType::File, - }) + fn close(&self, _node: &NodeRef, _instance: Option<&Box>) -> Result<(), Error> { + Ok(()) } } diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index 10aec7fc..c08b0653 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -1,348 +1,192 @@ //! In-memory filesystem driver #![no_std] -// #![warn(missing_docs)] -// #![allow(clippy::new_without_default)] -// #![feature( -// const_mut_refs, -// maybe_uninit_uninit_array, -// const_maybe_uninit_uninit_array, -// maybe_uninit_array_assume_init -// )] -// -// use core::{ -// any::Any, -// cell::{Ref, RefCell}, -// marker::PhantomData, -// }; -// -// use alloc::{boxed::Box, rc::Rc}; -// use block::BlockAllocator; -// use vfs::{BlockDevice, CreateInfo, Filesystem, Vnode, VnodeKind, VnodeRef}; -// use yggdrasil_abi::{error::Error, io::FileMode, path}; -// -// use crate::{bvec::BVec, dir::DirectoryNode, file::FileNode, tar::TarIterator}; -// -// #[cfg(test)] -// extern crate std; -// -// extern crate alloc; -// -// #[cfg(test)] -// macro_rules! test_allocator_with_counter { -// ($counter:ident, $allocator:ident) => { -// static $counter: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); -// -// struct $allocator; -// -// unsafe impl $crate::block::BlockAllocator for $allocator { -// fn alloc() -> Result, yggdrasil_abi::error::Error> { -// let b = std::boxed::Box::into_raw(std::boxed::Box::new([0; $crate::block::SIZE])); -// $counter.fetch_add(1, core::sync::atomic::Ordering::Release); -// Ok(unsafe { core::ptr::NonNull::new_unchecked(b as _) }) -// } -// -// unsafe fn dealloc(block: core::ptr::NonNull) { -// $counter.fetch_sub(1, core::sync::atomic::Ordering::Release); -// drop(std::boxed::Box::from_raw( -// block.as_ptr() as *mut [u8; $crate::block::SIZE] -// )); -// } -// } -// }; -// } -// -// pub mod block; -// pub mod bvec; -// -// mod dir; -// mod file; -// mod tar; -// -// const DEFAULT_FILE_MODE: FileMode = FileMode::new(0o644); -// const DEFAULT_DIR_MODE: FileMode = FileMode::new(0o755); -// -// /// In-memory read/write filesystem -// pub struct MemoryFilesystem { -// root: RefCell>, -// _pd: PhantomData, -// } -// -// impl Filesystem for MemoryFilesystem { -// fn dev(self: Rc) -> Option<&'static dyn BlockDevice> { -// todo!() -// } -// -// fn data(&self) -> Option> { -// todo!() -// } -// -// fn root(self: Rc) -> Result { -// Ok(self.root.borrow().clone().unwrap()) -// } -// } -// -// impl MemoryFilesystem { -// fn make_path( -// self: &Rc, -// at: &VnodeRef, -// path: &str, -// kind: VnodeKind, -// create: bool, -// ) -> Result { -// if path.is_empty() { -// return Ok(at.clone()); -// } -// let (element, rest) = path::split_left(path); -// assert!(!element.is_empty()); -// -// let node = at.lookup(element); -// let node = match node { -// Some(node) => node, -// None => { -// if !create { -// return Err(Error::DoesNotExist); -// } -// -// let node = self.create_node_initial(element, kind); -// // TODO require .tar's to have all the directories present to extract their metadata? -// if kind == VnodeKind::Directory { -// node.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); -// } -// at.add_child(node.clone()); -// -// node -// } -// }; -// -// if rest.is_empty() { -// Ok(node) -// } else { -// assert!(node.is_directory()); -// self.make_path(&node, rest, kind, create) -// } -// } -// -// fn create_node_initial(self: &Rc, name: &str, kind: VnodeKind) -> VnodeRef { -// assert!(!name.is_empty()); -// assert!(!name.contains('/')); -// -// let node = Vnode::new(name, kind); -// node.set_fs(self.clone()); -// -// // match kind { -// // VnodeKind::Directory => node.set_data(Box::new(DirectoryNode::::new( -// // info.uid, info.gid, info.mode, -// // ))), -// // VnodeKind::Regular => {} -// // _ => todo!(), -// // } -// -// node -// } -// -// fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { -// let root = Vnode::new("", VnodeKind::Directory); -// root.set_fs(self.clone()); -// root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); -// -// // 1. Create paths in tar -// for item in TarIterator::new(tar_data) { -// let Ok((hdr, _)) = item else { -// return Err(Error::InvalidArgument); -// }; -// -// let path = hdr.name.as_str()?.trim_matches('/'); -// let (dirname, filename) = path::split_right(path); -// let parent = self.make_path(&root, dirname, VnodeKind::Directory, true)?; -// let node = self.create_node_initial(filename, hdr.node_kind()); -// -// parent.add_child(node); -// } -// -// // 2. Associate files with their data -// for item in TarIterator::new(tar_data) { -// let Ok((hdr, data)) = item else { -// panic!("Unreachable"); -// }; -// -// let path = hdr.name.as_str()?.trim_matches('/'); -// let node = self.make_path(&root, path, VnodeKind::Directory, false)?; -// assert_eq!(node.kind(), hdr.node_kind()); -// -// if hdr.node_kind() == VnodeKind::Regular { -// let uid = usize::from(&hdr.uid).try_into().unwrap(); -// let gid = usize::from(&hdr.gid).try_into().unwrap(); -// let mode = convert_mode(usize::from(&hdr.mode))?; -// -// let data = data.unwrap(); -// let bvec = BVec::::try_from(data)?; -// assert_eq!(bvec.size(), data.len()); -// node.set_data(Box::new(FileNode { -// uid, -// gid, -// mode, -// data: RefCell::new(bvec), -// })); -// } -// } -// -// Ok(root) -// } -// -// /// Constructs a filesystem tree from a tar image in memory -// pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { -// let fs = Rc::new(Self { -// root: RefCell::new(None), -// _pd: PhantomData, -// }); -// let root = fs.from_slice_internal(tar_data)?; -// fs.root.replace(Some(root)); -// -// Ok(fs) -// } -// -// /// Constructs an empty memory filesystem -// pub fn empty() -> Rc { -// let fs = Rc::new(Self { -// root: RefCell::new(None), -// _pd: PhantomData, -// }); -// let root = Vnode::new("", VnodeKind::Directory); -// root.set_data(Box::new(DirectoryNode::::new(0, 0, DEFAULT_DIR_MODE))); -// root.set_fs(fs.clone()); -// fs.root.replace(Some(root)); -// fs -// } -// } -// -// fn convert_mode(mode: usize) -> Result { -// Ok(FileMode::new(mode as u32 & 0o777)) -// } -// -// #[cfg(test)] -// mod tests { -// use core::sync::atomic::Ordering; -// use std::rc::Rc; -// use vfs::{Filesystem, IoContext, Read, Seek, VnodeKind, Write}; -// use yggdrasil_abi::io::{FileMode, OpenOptions, SeekFrom}; -// -// use crate::MemoryFilesystem; -// -// #[repr(C)] -// struct AlignedTo { -// _align: [Align; 0], -// bytes: Bytes, -// } -// -// static TEST_IMAGE: &'static AlignedTo = &AlignedTo { -// _align: [], -// bytes: *include_bytes!("../test/test_image.tar"), -// }; -// -// #[test] -// fn test_memfs_construction() { -// fn check_file(ioctx: &IoContext, path: &str, expected_data: &str) { -// let node = ioctx.find(None, path, false, false).unwrap(); -// -// assert_eq!(node.kind(), VnodeKind::Regular); -// assert_eq!(node.size().unwrap(), expected_data.len() as u64); -// -// let file = node.open(OpenOptions::READ).unwrap(); -// let mut buf = [0; 512]; -// -// assert_eq!( -// file.borrow_mut().read(&mut buf).unwrap(), -// expected_data.len() -// ); -// -// assert_eq!(&buf[..expected_data.len()], expected_data.as_bytes()); -// } -// -// test_allocator_with_counter!(A_COUNTER, A); -// -// let fs = MemoryFilesystem::::from_slice(&TEST_IMAGE.bytes).unwrap(); -// let root = fs.root().unwrap(); -// -// // Files are small, so no indirect blocks allocated -// assert_eq!(A_COUNTER.load(Ordering::Acquire), 0); -// -// let ioctx = IoContext::new(root.clone()); -// -// assert!(Rc::ptr_eq( -// &root, -// &ioctx.find(None, "/", false, false).unwrap() -// )); -// -// let old_data = include_str!("../test/test1.txt"); -// check_file(&ioctx, "/test1.txt", old_data); -// -// // Write to the file -// { -// let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); -// let file = node.open(OpenOptions::WRITE).unwrap(); -// -// assert_eq!(file.borrow_mut().write(b"Hello").unwrap(), 5); -// } -// -// assert_eq!(A_COUNTER.load(Ordering::Acquire), 1); -// -// // Read back -// { -// let node = ioctx.find(None, "/test1.txt", false, false).unwrap(); -// let file = node.open(OpenOptions::READ).unwrap(); -// -// let mut buf = [0; 512]; -// assert_eq!(file.borrow_mut().read(&mut buf).unwrap(), old_data.len()); -// -// assert_eq!(&buf[..5], b"Hello"); -// assert_eq!(&buf[5..old_data.len()], &old_data.as_bytes()[5..]); -// } -// } -// -// // #[test] -// // fn test_memfs_create_and_write() { -// // test_allocator_with_counter!(A_COUNTER, A); -// -// // let fs = MemoryFilesystem::::empty(); -// // let root = fs.root().unwrap(); -// -// // let ioctx = IoContext::new(root.clone()); -// -// // // Create, write, seek and read file -// // { -// // // TODO CREATE option handling -// // root.create("test1.txt", VnodeKind::Regular).unwrap(); -// -// // let file = ioctx -// // .open( -// // None, -// // "/test1.txt", -// // OpenOptions::WRITE | OpenOptions::READ, -// // FileMode::empty(), -// // ) -// // .unwrap(); -// -// // let write_data = [1, 2, 3, 4]; -// // let mut read_data = [0; 512]; -// -// // let mut file = file.borrow_mut(); -// // assert_eq!(file.write(&write_data).unwrap(), write_data.len()); -// // assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); -// // assert_eq!(file.read(&mut read_data).unwrap(), write_data.len()); -// // assert_eq!(&read_data[..write_data.len()], &write_data[..]); -// // } -// -// // // Create a directory -// // { -// // // TODO read directory -// // root.create("dir1", VnodeKind::Directory).unwrap(); -// -// // let dir1 = ioctx.find(None, "/dir1", false, false).unwrap(); -// // let node = dir1.create("file1.txt", VnodeKind::Regular).unwrap(); -// // assert!(Rc::ptr_eq( -// // &ioctx.find(None, "/dir1/file1.txt", false, false).unwrap(), -// // &node -// // )); -// // } -// // } -// } +#![deny(missing_docs)] +#![allow(clippy::new_without_default)] +#![feature( + const_mut_refs, + maybe_uninit_uninit_array, + const_maybe_uninit_uninit_array, + maybe_uninit_array_assume_init +)] + +use core::{cell::RefCell, marker::PhantomData}; + +use alloc::rc::Rc; +use block::BlockAllocator; +use dir::DirectoryNode; +use file::FileNode; +use vfs::{AccessToken, NodeRef}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, FileType, GroupId, UserId}, + path::Path, +}; + +use crate::tar::TarIterator; + +#[cfg(test)] +extern crate std; + +extern crate alloc; + +#[cfg(test)] +macro_rules! test_allocator_with_counter { + ($counter:ident, $allocator:ident) => { + static $counter: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + + struct $allocator; + + unsafe impl $crate::block::BlockAllocator for $allocator { + fn alloc() -> Result, yggdrasil_abi::error::Error> { + let b = std::boxed::Box::into_raw(std::boxed::Box::new([0; $crate::block::SIZE])); + $counter.fetch_add(1, core::sync::atomic::Ordering::Release); + Ok(unsafe { core::ptr::NonNull::new_unchecked(b as _) }) + } + + unsafe fn dealloc(block: core::ptr::NonNull) { + $counter.fetch_sub(1, core::sync::atomic::Ordering::Release); + drop(std::boxed::Box::from_raw( + block.as_ptr() as *mut [u8; $crate::block::SIZE] + )); + } + } + }; +} + +pub mod block; +pub mod bvec; + +mod dir; +mod file; +mod tar; + +/// In-memory read/write filesystem +pub struct MemoryFilesystem { + root: RefCell>, + _pd: PhantomData, +} + +impl MemoryFilesystem { + fn make_path( + self: &Rc, + at: &NodeRef, + path: &Path, + kind: FileType, + create: bool, + ) -> Result { + let access = unsafe { AccessToken::authorized() }; + if path.is_empty() { + return Ok(at.clone()); + } + let (element, rest) = path.split_left(); + // let (element, rest) = path::split_left(path); + assert!(!element.is_empty()); + assert!(!element.contains('/')); + + // let node = at.lookup(element); + let node = at.lookup_or_load(element, access); + let node = match node { + Ok(node) => node, + Err(Error::DoesNotExist) => { + if !create { + return Err(Error::DoesNotExist); + } + + let node = self.create_node_initial(kind); + at.add_child(element, node.clone())?; + + node + } + Err(err) => { + log::warn!("{:?}: lookup failed: {:?}", path, err); + return Err(err); + } + }; + + if rest.is_empty() { + Ok(node) + } else { + assert!(node.is_directory()); + self.make_path(&node, rest, kind, create) + } + } + + fn create_node_initial(self: &Rc, kind: FileType) -> NodeRef { + match kind { + FileType::File => FileNode::::new(), + FileType::Directory => DirectoryNode::::new(), + _ => todo!(), + } + } + + fn from_slice_internal(self: &Rc, tar_data: &'static [u8]) -> Result { + let root = DirectoryNode::::new(); + + // 1. Create paths in tar + for item in TarIterator::new(tar_data) { + let Ok((hdr, _)) = item else { + return Err(Error::InvalidArgument); + }; + + let path = Path::from_str(hdr.name.as_str()?.trim_matches('/')); + log::debug!("Make path {:?}", path); + + let (dirname, filename) = path.split_right(); + let parent = self.make_path(&root, dirname, FileType::Directory, true)?; + let node = self.create_node_initial(hdr.node_kind()); + + parent.add_child(filename, node)?; + } + + // 2. Associate files with their data + for item in TarIterator::new(tar_data) { + let Ok((hdr, data)) = item else { + panic!("Unreachable"); + }; + + let path = Path::from_str(hdr.name.as_str()?.trim_matches('/')); + let node = self.make_path(&root, path, FileType::Directory, false)?; + assert_eq!(node.ty(), hdr.node_kind()); + + let uid = UserId::from(usize::from(&hdr.uid) as u32); + let gid = GroupId::from(usize::from(&hdr.gid) as u32); + let mode = convert_mode(usize::from(&hdr.mode))?; + + let access = unsafe { AccessToken::authorized() }; + node.set_access(Some(uid), Some(gid), Some(mode), access)?; + + if hdr.node_kind() == FileType::File { + let data = data.unwrap(); + let node_data = node.data_as_ref::>(); + let mut bvec = node_data.data.borrow_mut(); + + bvec.init_with_cow(data)?; + assert_eq!(bvec.size(), data.len()); + } + } + + Ok(root) + } + + /// Constructs a filesystem tree from a tar image in memory + pub fn from_slice(tar_data: &'static [u8]) -> Result, Error> { + let fs = Rc::new(Self { + root: RefCell::new(None), + _pd: PhantomData, + }); + let root = fs.from_slice_internal(tar_data)?; + fs.root.replace(Some(root)); + + Ok(fs) + } + + // TODO Filesystem trait? + /// Returns the root node of the memory filesystem + pub fn root(&self) -> Result { + Ok(self.root.borrow().clone().unwrap()) + } +} + +fn convert_mode(mode: usize) -> Result { + Ok(FileMode::new(mode as u32 & 0o777)) +} diff --git a/lib/memfs/src/tar.rs b/lib/memfs/src/tar.rs index 6b7ae64a..3408f2e5 100644 --- a/lib/memfs/src/tar.rs +++ b/lib/memfs/src/tar.rs @@ -1,5 +1,4 @@ -use vfs::VnodeKind; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{error::Error, io::FileType}; #[repr(C)] pub(crate) struct OctalField { @@ -126,10 +125,10 @@ impl TarEntry { self.name.data[0] == 0 } - pub fn node_kind(&self) -> VnodeKind { + pub fn node_kind(&self) -> FileType { match self.type_ { - 0 | b'0' => VnodeKind::Regular, - b'5' => VnodeKind::Directory, + 0 | b'0' => FileType::File, + b'5' => FileType::Directory, _ => todo!(), } } diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs index 218a24f3..23f946d6 100644 --- a/lib/vfs/src/device.rs +++ b/lib/vfs/src/device.rs @@ -1,47 +1,74 @@ -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::node::{CommonImpl, NodeRef}; +/// Block device interface +#[allow(unused)] pub trait BlockDevice { - fn read(&self, pos: u64, buf: &mut [u8]) -> Result { + /// Reads data frmo the given offset of the device + fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { Err(Error::NotImplemented) } - fn write(&self, pos: u64, buf: &[u8]) -> Result { + /// Writes the data to the given offset of the device + fn write(&'static self, pos: u64, buf: &[u8]) -> Result { Err(Error::NotImplemented) } + /// Returns the size of the block device in bytes + fn size(&self) -> Result { + Err(Error::NotImplemented) + } + + /// Returns `true` if the device can be read from fn is_readable(&self) -> bool { true } + /// Returns `true` if the device can be written to fn is_writable(&self) -> bool { true } - fn size(&self) -> Result; + + /// Performs a device-specific function + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + Err(Error::NotImplemented) + } } +/// Character device interface +#[allow(unused)] pub trait CharDevice { - fn read(&self, buf: &mut [u8]) -> Result { + /// Reads data from the device + fn read(&'static self, buf: &mut [u8]) -> Result { Err(Error::NotImplemented) } - fn write(&self, buf: &[u8]) -> Result { + /// Writes the data to the device + fn write(&'static self, buf: &[u8]) -> Result { Err(Error::NotImplemented) } + /// Returns `true` if the device can be read from fn is_readable(&self) -> bool { true } + /// Returns `true` if the device can be written to fn is_writable(&self) -> bool { true } + /// Returns `true` if the given device is a terminal fn is_terminal(&self) -> bool { false } + + /// Performs a device-specific function + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + Err(Error::NotImplemented) + } } #[derive(Clone)] -pub struct BlockDeviceWrapper(pub(crate) &'static dyn BlockDevice); +pub(crate) struct BlockDeviceWrapper(pub(crate) &'static dyn BlockDevice); #[derive(Clone)] -pub struct CharDeviceWrapper(pub(crate) &'static dyn CharDevice); +pub(crate) struct CharDeviceWrapper(pub(crate) &'static dyn CharDevice); impl BlockDeviceWrapper { pub fn is_readable(&self) -> bool { diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 36d9282d..33f7acd9 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -22,14 +22,20 @@ mod device; mod directory; mod regular; +/// Describes the starting position of the directory pub enum DirectoryOpenPosition { + /// Contents should be fetched from the directory impl with given offset FromPhysical(u64), + /// Contents should be fetched from the tree cache FromCache, } +/// Wrapper type for a [File] shared reference pub type FileRef = Arc; // TODO some kind of a mutex instead? +/// Describes an open file +#[allow(missing_docs)] pub enum File { Directory(DirectoryFile), Regular(RegularFile), @@ -38,12 +44,12 @@ pub enum File { } impl File { - pub fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = Cell::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) } - pub fn regular( + pub(crate) fn regular( node: NodeRef, position: u64, instance_data: Option>, @@ -61,7 +67,7 @@ impl File { })) } - pub fn block( + pub(crate) fn block( device: BlockDeviceWrapper, node: NodeRef, opts: OpenOptions, @@ -85,7 +91,7 @@ impl File { }))) } - pub fn char( + pub(crate) fn char( device: CharDeviceWrapper, node: NodeRef, opts: OpenOptions, @@ -108,6 +114,7 @@ impl File { }))) } + /// Reads entries from the directory pub fn read_dir(&self, entries: &mut [MaybeUninit]) -> Result { match self { Self::Directory(dir) => dir.read_entries(entries), @@ -115,6 +122,7 @@ impl File { } } + /// Returns the underlying [Node] the file contains pub fn node(&self) -> Option<&NodeRef> { match self { Self::Directory(file) => Some(&file.node), @@ -207,7 +215,7 @@ mod tests { use crate::{ device::{BlockDevice, CharDevice}, file::DirectoryOpenPosition, - node::{CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl}, + node::{AccessToken, CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl}, traits::{Read, Seek, Write}, }; @@ -264,7 +272,7 @@ mod tests { NodeFlags::empty(), ); - let f = d.open_directory().unwrap(); + let f = d.open_directory(AccessToken::test_authorized()).unwrap(); let mut entries = [MaybeUninit::uninit(); 16]; let count = f.read_dir(&mut entries).unwrap(); @@ -310,7 +318,7 @@ mod tests { d.add_child("child1", child).unwrap(); - let f = d.open_directory().unwrap(); + let f = d.open_directory(AccessToken::test_authorized()).unwrap(); let mut entries = [MaybeUninit::uninit(); 16]; let count = f.read_dir(&mut entries).unwrap(); @@ -393,7 +401,12 @@ mod tests { let data = Arc::new(RefCell::new(vec![])); let node = Node::regular(F { data: data.clone() }, NodeFlags::empty()); - let file = node.open(OpenOptions::READ | OpenOptions::WRITE).unwrap(); + let file = node + .open( + OpenOptions::READ | OpenOptions::WRITE, + AccessToken::test_authorized(), + ) + .unwrap(); let mut buf = [0; 512]; assert_eq!(&*data.borrow(), &[]); @@ -460,7 +473,12 @@ mod tests { let node = Node::block(dev, NodeFlags::empty()); - let file = node.open(OpenOptions::READ | OpenOptions::WRITE).unwrap(); + let file = node + .open( + OpenOptions::READ | OpenOptions::WRITE, + AccessToken::test_authorized(), + ) + .unwrap(); assert_eq!(file.seek(SeekFrom::End(0)).unwrap(), 1024); assert_eq!(file.write(b"12345").unwrap(), 0); @@ -495,10 +513,14 @@ mod tests { let node = Node::char(&DEV, NodeFlags::empty()); let mut buf = [0; 512]; - let err = node.open(OpenOptions::WRITE).unwrap_err(); + let err = node + .open(OpenOptions::WRITE, AccessToken::test_authorized()) + .unwrap_err(); assert_eq!(err, Error::ReadOnly); - let file = node.open(OpenOptions::READ).unwrap(); + let file = node + .open(OpenOptions::READ, AccessToken::test_authorized()) + .unwrap(); assert_eq!(file.tell().unwrap_err(), Error::InvalidOperation); assert_eq!( file.seek(SeekFrom::Start(10)).unwrap_err(), diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index dec75333..f40858d2 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -6,10 +6,22 @@ use yggdrasil_abi::{ }; use crate::{ - node::{CreateInfo, SymlinkData}, + node::{AccessToken, CreateInfo, SymlinkData}, FileRef, NodeRef, }; +/// Describes a general filesystem access +pub enum Action { + /// Access involves reading data without modification + Read, + /// Access involves writing file data or modifying directory content + Write, + /// Access involves traversing a path or opening a file for execution + Execute, +} + +/// Contains the state of program's I/O context: current working directory, UID, GID, umask, etc. +#[derive(Clone)] pub struct IoContext { uid: UserId, gid: GroupId, @@ -20,6 +32,8 @@ pub struct IoContext { } impl IoContext { + /// Constructs a new [IoContext] with given root node (which also becomes the cwd). By default, + /// the root user/group is used. Default umask is 0o022. pub fn new(root: NodeRef) -> Self { Self { uid: UserId::root(), @@ -31,20 +45,91 @@ impl IoContext { } } + /// "Clones" an I/O context the way it is inherited by a newly spawned child process + pub fn inherit(other: &IoContext) -> Self { + other.clone() + } + + /// Returns the root node of the [IoContext] pub fn root(&self) -> &NodeRef { &self.root } + /// Returns the current working directory node of the [IoContext] pub fn cwd(&self) -> &NodeRef { &self.cwd_node } + /// Sets the current user ID of the context. Returns [Error::PermissionDenied] if current user + /// is not root. + pub fn set_uid(&mut self, uid: UserId) -> Result<(), Error> { + if uid != self.uid && !self.uid.is_root() { + Err(Error::PermissionDenied) + } else { + self.uid = uid; + Ok(()) + } + } + + #[cfg(test)] + fn set_uid_unchecked(&mut self, uid: UserId) { + self.uid = uid; + } + + /// Sets the current group ID of the context. Returns [Error::PermissionDenied] if current user + /// is not root. + pub fn set_gid(&mut self, gid: GroupId) -> Result<(), Error> { + if gid != self.gid && !self.uid.is_root() { + Err(Error::PermissionDenied) + } else { + self.gid = gid; + Ok(()) + } + } + + #[cfg(test)] + fn set_gid_unchecked(&mut self, gid: GroupId) { + self.gid = gid; + } + + /// Checks if the current user can access given [crate::Node] and, if so, returns an + /// [AccessToken]. + pub fn check_access(&self, action: Action, node: &NodeRef) -> Result { + let metadata = node.metadata()?; + + let allow = match action { + Action::Read => { + self.uid.is_root() + | metadata.user_read(self.uid) + | metadata.group_read(self.gid) + | metadata.other_read() + } + Action::Write => { + self.uid.is_root() + | metadata.user_write(self.uid) + | metadata.group_write(self.gid) + | metadata.other_write() + } + Action::Execute => { + metadata.user_exec(self.uid) | metadata.group_exec(self.gid) | metadata.other_exec() + } + }; + + if allow { + Ok(unsafe { AccessToken::authorized() }) + } else { + Err(Error::PermissionDenied) + } + } + + /// Changes current working directory to `path`. Will fail if access is denied or the path does + /// not point to a directory. pub fn set_cwd>(&mut self, path: P) -> Result<(), Error> { let path = path.as_ref(); if !path.is_absolute() { todo!(); } - let node = Self::_find(self.root.clone(), path.trim_start_separators(), true, true)?; + let node = self._find(self.root.clone(), path.trim_start_separators(), true, true)?; if !node.is_directory() { return Err(Error::NotADirectory); } @@ -53,16 +138,24 @@ impl IoContext { Ok(()) } + /// Returns the current working directory path pub fn cwd_path(&self) -> &Path { self.cwd_path.as_ref() } + /// Makes a directory at given path become a "mountpoint" for the given filesystem root. + /// When accessed, the target directory will return contents of the filesystem root instead of + /// its own. Both the `target` path and `fs_root` Node must be directories. pub fn mount>(&mut self, target: P, fs_root: NodeRef) -> Result<(), Error> { + if !self.uid.is_root() { + return Err(Error::PermissionDenied); + } + let target = target.as_ref(); if !target.is_absolute() { todo!(); } - let target_node = Self::_find( + let target_node = self._find( self.root.clone(), target.trim_start_separators(), true, @@ -72,6 +165,9 @@ impl IoContext { target_node.set_mountpoint_target(fs_root) } + /// Locates a [crate::Node] at given path and opens it with requested access options. If no + /// such node exists and `OpenOptions::CREATE` is specified, will attempt to create a regular + /// file node at given path and then open it. pub fn open>( &mut self, at: Option, @@ -93,14 +189,93 @@ impl IoContext { gid: self.gid, ty: FileType::File, }; - parent.create(&create_info)? + let access = self.check_access(Action::Write, &parent)?; + parent.create(&create_info, access)? } Err(err) => return Err(err), }; - node.open(opts) + // If not read/write is requested, access is granted + let mut access = unsafe { AccessToken::authorized() }; + + if opts.contains(OpenOptions::READ) { + access += self.check_access(Action::Read, &node)?; + } + if opts.contains(OpenOptions::WRITE) { + access += self.check_access(Action::Write, &node)?; + } + + node.open(opts, access) } + /// Opens a file at given path for execution + pub fn open_exec>( + &mut self, + at: Option, + path: P, + ) -> Result { + let node = self.find(at, path, true, true)?; + let access = self.check_access(Action::Execute, &node)?; + node.open(OpenOptions::READ, access) + } + + /// Creates a directory a given path + pub fn create_directory>( + &mut self, + at: Option, + path: P, + mode: FileMode, + ) -> Result { + let path = path.as_ref(); + let (parent, name) = path.split_right(); + let parent = self.find(at, parent, true, true)?; + let access = self.check_access(Action::Write, &parent)?; + let create_info = CreateInfo { + name: name.into(), + ty: FileType::Directory, + uid: self.uid, + gid: self.gid, + mode: mode & !self.umask, + }; + + parent.create(&create_info, access) + } + + fn remove_entry>( + &mut self, + at: Option, + path: P, + directory: bool, + ) -> Result<(), Error> { + let path = path.as_ref(); + let (parent, name) = path.trim_end_separators().split_right(); + + if name.is_empty() { + log::warn!("Tried to remove weird path: {:?}", path); + return Err(Error::DoesNotExist); + } + + let parent = self.find(at, parent, false, false)?; + let access = self.check_access(Action::Write, &parent)?; + + if directory { + // parent.remove_directory(name, access) + todo!() + } else { + parent.remove_file(name, access) + } + } + + /// Removes a device or regular file node at given path + pub fn remove_file>( + &mut self, + at: Option, + path: P, + ) -> Result<(), Error> { + self.remove_entry(at, path, false) + } + + /// Locates a [crate::Node] pointed to by given [Path] pub fn find>( &mut self, at: Option, @@ -118,10 +293,12 @@ impl IoContext { self.cwd_node.clone() }; - Self::_find(at, path, follow_links, follow_mount) + self._find(at, path, follow_links, follow_mount) } - fn _resolve_link(at: &NodeRef, link: &SymlinkData) -> Result { + fn _resolve_link(&self, at: &NodeRef, link: &SymlinkData) -> Result { + self.check_access(Action::Read, at)?; + // If the filesystem itself implements direct target resolution, use that match link.imp.target(at) { Err(Error::NotImplemented) => {} @@ -138,7 +315,12 @@ impl IoContext { todo!() } - fn _resolve(mut at: NodeRef, follow_links: bool, follow_mount: bool) -> Result { + fn _resolve( + &self, + mut at: NodeRef, + follow_links: bool, + follow_mount: bool, + ) -> Result { loop { if follow_mount && let Some(target) = at.mountpoint_target() { at = target.clone(); @@ -147,7 +329,7 @@ impl IoContext { if follow_links && let Ok(link) = at.as_symlink() { // Resolve the link - at = Self::_resolve_link(&at, link)?; + at = self._resolve_link(&at, link)?; continue; } @@ -156,6 +338,7 @@ impl IoContext { } fn _find( + &self, mut at: NodeRef, path: &Path, follow_links: bool, @@ -180,7 +363,7 @@ impl IoContext { } } - at = Self::_resolve(at, follow_links, follow_mount)?; + at = self._resolve(at, follow_links, follow_mount)?; if !at.is_directory() { return Err(Error::NotADirectory); @@ -190,13 +373,14 @@ impl IoContext { return Ok(at); } - let node = at.lookup_or_load(element)?; - let node = Self::_resolve(node, follow_links, follow_mount)?; + let access = self.check_access(Action::Execute, &at)?; + let node = at.lookup_or_load(element, access)?; + let node = self._resolve(node, follow_links, follow_mount)?; if rest.is_empty() { Ok(node) } else { - Self::_find(node, rest, follow_links, follow_mount) + self._find(node, rest, follow_links, follow_mount) } } } @@ -206,17 +390,81 @@ mod tests { use alloc::sync::Arc; use yggdrasil_abi::{ error::Error, - io::{FileMode, OpenOptions}, + io::{FileMode, GroupId, OpenOptions, UserId}, path::Path, }; use crate::{ - impls::{const_value_node, f_symlink, mdir}, + impls::{const_value_node, f_symlink, mdir, value_node}, + node::AccessToken, Read, }; use super::IoContext; + #[test] + fn access() { + let f1 = const_value_node("file1"); + let f2 = const_value_node("file2"); + let f3 = value_node("file3".to_owned()); + let root = mdir([("f1", f1.clone()), ("f2", f2.clone()), ("f3", f3.clone())]); + + let mut ioctx = IoContext::new(root.clone()); + + let uid = UserId::from(1); + let gid = GroupId::from(1); + + // 1:1 + ioctx.set_uid_unchecked(uid); + ioctx.set_gid_unchecked(gid); + + // 1:0, 0444 + f1.set_access( + Some(uid), + None, + Some(FileMode::new(0o444)), + AccessToken::test_authorized(), + ) + .unwrap(); + + // 0:1, 0644 + f2.set_access(None, Some(gid), None, AccessToken::test_authorized()) + .unwrap(); + + // 1:1, 0644 + f3.set_access(Some(uid), Some(gid), None, AccessToken::test_authorized()) + .unwrap(); + + // f1, read-only + ioctx + .open(None, "/f1", OpenOptions::READ, FileMode::empty()) + .unwrap(); + // f1, write + let err = ioctx + .open(None, "/f1", OpenOptions::WRITE, FileMode::empty()) + .unwrap_err(); + assert_eq!(err, Error::PermissionDenied); + + // f2, read-only + ioctx + .open(None, "/f2", OpenOptions::READ, FileMode::empty()) + .unwrap(); + // f2, write + let err = ioctx + .open(None, "/f2", OpenOptions::WRITE, FileMode::empty()) + .unwrap_err(); + assert_eq!(err, Error::PermissionDenied); + + // f3, read-only + ioctx + .open(None, "/f3", OpenOptions::READ, FileMode::empty()) + .unwrap(); + // f3, write + ioctx + .open(None, "/f3", OpenOptions::WRITE, FileMode::empty()) + .unwrap(); + } + #[test] fn cwd() { let d1_f1 = const_value_node("dir1-file1"); diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index c1c6bb7c..bbc7f178 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,4 +1,7 @@ +//! Virtual filesystem interfaces and driver implementation + #![cfg_attr(not(test), no_std)] +#![deny(missing_docs)] #![feature( trait_upcasting, if_let_guard, @@ -21,5 +24,9 @@ pub(crate) mod traits; pub use device::{BlockDevice, CharDevice}; pub use file::{DirectoryOpenPosition, File, FileRef}; -pub use node::{impls, Node, NodeRef}; +pub use ioctx::{Action, IoContext}; +pub use node::{ + impls, AccessToken, CommonImpl, CreateInfo, DirectoryImpl, Metadata, Node, NodeFlags, NodeRef, + RegularImpl, SymlinkImpl, +}; pub use traits::{Read, Seek, Write}; diff --git a/lib/vfs/src/node/access.rs b/lib/vfs/src/node/access.rs new file mode 100644 index 00000000..dc928029 --- /dev/null +++ b/lib/vfs/src/node/access.rs @@ -0,0 +1,78 @@ +use core::{ + marker::PhantomData, + ops::{Add, AddAssign}, +}; + +use yggdrasil_abi::io::{FileMode, GroupId, UserId}; + +use super::Metadata; + +/// Zero-sized token type used to ensure checked access to node functions +pub struct AccessToken(PhantomData<()>); + +#[allow(missing_docs)] +impl Metadata { + pub fn user_read(&self, uid: UserId) -> bool { + self.uid == uid && self.mode.contains(FileMode::USER_READ) + } + + pub fn user_write(&self, uid: UserId) -> bool { + self.uid == uid && self.mode.contains(FileMode::USER_WRITE) + } + + pub fn user_exec(&self, uid: UserId) -> bool { + self.uid == uid && self.mode.contains(FileMode::USER_EXEC) + } + + pub fn group_read(&self, gid: GroupId) -> bool { + self.gid == gid && self.mode.contains(FileMode::GROUP_READ) + } + + pub fn group_write(&self, gid: GroupId) -> bool { + self.gid == gid && self.mode.contains(FileMode::GROUP_WRITE) + } + + pub fn group_exec(&self, gid: GroupId) -> bool { + self.gid == gid && self.mode.contains(FileMode::GROUP_EXEC) + } + + pub fn other_read(&self) -> bool { + self.mode.contains(FileMode::OTHER_READ) + } + + pub fn other_write(&self) -> bool { + self.mode.contains(FileMode::OTHER_WRITE) + } + + pub fn other_exec(&self) -> bool { + self.mode.contains(FileMode::OTHER_EXEC) + } +} + +impl AccessToken { + /// Creates an "authorized" [AccessToken]. + /// + /// # Safety + /// + /// Unsafe: allows for unchecked authorization of any node actions. + pub const unsafe fn authorized() -> Self { + Self(PhantomData) + } + + #[cfg(test)] + pub const fn test_authorized() -> Self { + unsafe { Self::authorized() } + } +} + +impl Add for AccessToken { + type Output = Self; + + fn add(self, _rhs: Self) -> Self::Output { + self + } +} + +impl AddAssign for AccessToken { + fn add_assign(&mut self, _rhs: Self) {} +} diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index 8fdba6ba..e406d29f 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -1,3 +1,4 @@ +//! Various helper node implementations for convenience use core::{any::Any, cell::RefCell, marker::PhantomData, str::FromStr}; use alloc::{ @@ -11,15 +12,13 @@ use yggdrasil_abi::{error::Error, io::OpenOptions}; use crate::DirectoryOpenPosition; -use super::{ - CommonImpl, CreateInfo, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl, SymlinkImpl, -}; +use super::{CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl, SymlinkImpl}; -pub trait SliceRead { +trait SliceRead { fn read_slice(&self, offset: usize, buf: &mut [u8]) -> usize; } -pub trait SliceWrite { +trait SliceWrite { fn write_slice(&mut self, offset: usize, buf: &[u8]) -> usize; } @@ -27,6 +26,15 @@ trait IntoInstanceData { fn into_instance_data(&self) -> Vec; } +/// Closure interface for reading a single value +pub trait ValueReadFn = Fn() -> Result; +/// Closure interface for writing a single value +pub trait ValueWriteFn = Fn(T) -> Result<(), Error>; +/// Closure interface for reading bytes +pub trait ReadFn = Fn(u64, &mut [u8]) -> Result; +/// Closure interface for writing bytes +pub trait WriteFn = Fn(u64, &[u8]) -> Result; + impl> SliceRead for T { fn read_slice(&self, pos: usize, buf: &mut [u8]) -> usize { let value = self.as_ref(); @@ -55,31 +63,40 @@ impl IntoInstanceData for T { } } -pub trait ReadFn = Fn() -> Result; -pub trait WriteFn = Fn(T) -> Result<(), Error>; - enum FnNodeData { Read(Vec), Write(RefCell>), } -pub struct ReadOnlyFnNode> { +/// Allows read-only access to a value. The value is converted to a string representation when it's +/// read. +pub struct ReadOnlyFnValueNode> { read: R, _pd: PhantomData, } -pub struct FnNode, W: WriteFn> { +/// Allows read-write access to a value (but not both at the same time). The value is converted +/// to/from a string representation when it's read/written. +pub struct FnValueNode, W: ValueWriteFn> { read: R, write: W, _pd: PhantomData, } +/// Allows read-only access to a "functional file" +pub struct ReadOnlyFnNode { + read: R, +} + +/// In-memory directory using tree cache pub struct MemoryDirectory; +/// In-memory symlink pointing to a fixed [Node] pub struct FixedSymlink { target: NodeRef, } -impl + 'static> ReadOnlyFnNode { +impl + 'static> ReadOnlyFnValueNode { + /// Creates a new [ReadOnlyFnValueNode] with given read function pub fn new(read: R) -> NodeRef { Node::regular( Self::new_impl(read), @@ -87,7 +104,7 @@ impl + 'static> ReadOnlyFnNode { ) } - pub const fn new_impl(read: R) -> Self { + const fn new_impl(read: R) -> Self { Self { read, _pd: PhantomData, @@ -95,13 +112,13 @@ impl + 'static> ReadOnlyFnNode { } } -impl> CommonImpl for ReadOnlyFnNode { +impl> CommonImpl for ReadOnlyFnValueNode { fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } -impl> RegularImpl for ReadOnlyFnNode { +impl> RegularImpl for ReadOnlyFnValueNode { fn open( &self, _node: &NodeRef, @@ -162,9 +179,13 @@ impl FnNodeData { } } -impl + 'static, W: WriteFn + 'static> - FnNode +impl FnValueNode +where + T: ToString + FromStr + 'static, + R: ValueReadFn + 'static, + W: ValueWriteFn + 'static, { + /// Creates a new [FnValueNode] with given read and write functions pub fn new(read: R, write: W) -> NodeRef { Node::regular( Self::new_impl(read, write), @@ -172,7 +193,7 @@ impl + 'static, W: WriteFn + 's ) } - pub const fn new_impl(read: R, write: W) -> Self { + const fn new_impl(read: R, write: W) -> Self { Self { read, write, @@ -181,13 +202,17 @@ impl + 'static, W: WriteFn + 's } } -impl, W: WriteFn> CommonImpl for FnNode { +impl, W: ValueWriteFn> CommonImpl + for FnValueNode +{ fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } -impl, W: WriteFn> RegularImpl for FnNode { +impl, W: ValueWriteFn> RegularImpl + for FnValueNode +{ fn open( &self, _node: &NodeRef, @@ -247,17 +272,70 @@ impl, W: WriteFn> RegularImpl for FnNode< } } +// Byte read-only node + +impl ReadOnlyFnNode +where + R: ReadFn + 'static, +{ + /// Creates a new [ReadOnlyFnNode] with given read function + pub fn new(read: R) -> NodeRef { + Node::regular(Self { read }, NodeFlags::IN_MEMORY_PROPS) + } +} + +impl CommonImpl for ReadOnlyFnNode +where + R: ReadFn, +{ + fn size(&self, _node: &NodeRef) -> Result { + Ok(0) + } +} + +impl RegularImpl for ReadOnlyFnNode +where + R: ReadFn, +{ + fn open( + &self, + _node: &NodeRef, + opts: OpenOptions, + ) -> Result<(u64, Option>), Error> { + if opts.contains(OpenOptions::WRITE) { + return Err(Error::ReadOnly); + } + Ok((0, None)) + } + + fn read( + &self, + _node: &NodeRef, + _instance: Option<&Box>, + pos: u64, + buf: &mut [u8], + ) -> Result { + (self.read)(pos, buf) + } +} + // In-memory directory +impl MemoryDirectory { + /// Creates a [MemoryDirectory] with no children + pub fn empty() -> NodeRef { + Node::directory( + MemoryDirectory, + NodeFlags::IN_MEMORY_PROPS | NodeFlags::IN_MEMORY_SIZE, + ) + } +} + impl CommonImpl for MemoryDirectory {} impl DirectoryImpl for MemoryDirectory { fn open(&self, _node: &NodeRef) -> Result { Ok(DirectoryOpenPosition::FromCache) } - - fn create(&self, _parent: &NodeRef, _info: &CreateInfo) -> Result { - Err(Error::ReadOnly) - } } // In-memory fixed symlink @@ -269,15 +347,17 @@ impl SymlinkImpl for FixedSymlink { } } +/// Creates a read-only value node with given `value` pub fn const_value_node(value: T) -> NodeRef { - ReadOnlyFnNode::new(move || Ok(value.clone())) + ReadOnlyFnValueNode::new(move || Ok(value.clone())) } +/// Creates a read-write value node with given `value` pub fn value_node(value: T) -> NodeRef { let rd_state = Arc::new(RefCell::new(value)); let wr_state = rd_state.clone(); - FnNode::new( + FnValueNode::new( move || Ok(rd_state.borrow().clone()), move |t| { *wr_state.borrow_mut() = t; @@ -286,6 +366,12 @@ pub fn value_node(value: T) -> NodeRef ) } +/// Creates a read-only node with given read function +pub fn read_fn_node(read: R) -> NodeRef { + ReadOnlyFnNode::new(read) +} + +/// Creates an in-memory directory from the iterator pub fn mdir, I: IntoIterator>(it: I) -> NodeRef { let dir = Node::directory( MemoryDirectory, @@ -297,8 +383,9 @@ pub fn mdir, I: IntoIterator>(it: I) -> Nod dir } +/// Creates a static symlink pointing to given node pub fn f_symlink(target: NodeRef) -> NodeRef { - Node::symlink(FixedSymlink { target }, NodeFlags::empty()) + Node::symlink(FixedSymlink { target }, NodeFlags::IN_MEMORY_PROPS) } #[cfg(test)] @@ -306,14 +393,19 @@ mod tests { use yggdrasil_abi::io::OpenOptions; use crate::{ - node::impls::{const_value_node, value_node}, + node::{ + impls::{const_value_node, value_node}, + AccessToken, + }, traits::{Read, Seek, Write}, }; #[test] fn read_only_fn_node() { let node = const_value_node("abcdef"); - let file = node.open(OpenOptions::READ).unwrap(); + let file = node + .open(OpenOptions::READ, AccessToken::test_authorized()) + .unwrap(); let mut buf = [0; 512]; assert_eq!(file.tell().unwrap(), 0); @@ -331,19 +423,25 @@ mod tests { let mut buf = [0; 512]; // Try read the value - let file = node.open(OpenOptions::READ).unwrap(); + let file = node + .open(OpenOptions::READ, AccessToken::test_authorized()) + .unwrap(); assert_eq!(file.tell().unwrap(), 0); assert_eq!(file.read(&mut buf).unwrap(), 4); assert_eq!(&buf[..4], b"1234"); // Try write the value - let file = node.open(OpenOptions::WRITE).unwrap(); + let file = node + .open(OpenOptions::WRITE, AccessToken::test_authorized()) + .unwrap(); assert_eq!(file.tell().unwrap(), 0); assert_eq!(file.write(b"654321").unwrap(), 6); drop(file); // Try read the value again - let file = node.open(OpenOptions::READ).unwrap(); + let file = node + .open(OpenOptions::READ, AccessToken::test_authorized()) + .unwrap(); assert_eq!(file.tell().unwrap(), 0); assert_eq!(file.read(&mut buf).unwrap(), 6); assert_eq!(&buf[..6], b"654321"); diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index c57c6cb5..c7b55f09 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -1,17 +1,14 @@ -use core::{ - any::Any, - cell::{Cell, RefCell}, - fmt, -}; +use core::{any::Any, fmt}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{ bitflags, error::Error, - io::{FileAttr, FileMode, FileType, GroupId, UserId}, + io::{FileMode, FileType, GroupId, UserId}, }; +mod access; pub mod impls; mod traits; @@ -19,25 +16,36 @@ mod traits; mod ops; mod tree; +pub use access::AccessToken; pub use traits::{CommonImpl, DirectoryImpl, RegularImpl, SymlinkImpl}; use crate::device::{BlockDevice, BlockDeviceWrapper, CharDevice, CharDeviceWrapper}; +/// Wrapper type for a [Node] shared reference pub type NodeRef = Arc; bitflags! { + #[doc = "Describes additional flags for the node"] pub struct NodeFlags: u32 { + #[doc = "Node's metadata only exists within the VFS cache and should not be fetched"] const IN_MEMORY_PROPS: bit 0; + #[doc = "Node's size only exists within the VFS cache"] const IN_MEMORY_SIZE: bit 1; } } +/// Information used when creating an entry within a directory #[derive(Debug, Clone)] pub struct CreateInfo { + /// New entry name pub name: String, + /// User ID of the entry pub uid: UserId, + /// Group ID of the entry pub gid: GroupId, + /// Access mode of the entry pub mode: FileMode, + /// Entry type pub ty: FileType, } @@ -61,19 +69,23 @@ enum NodeImpl { Symlink(SymlinkData), } +/// Metadata of the node #[derive(Clone, Copy)] pub struct Metadata { + /// User ID of the node pub uid: UserId, + /// Group ID of the node pub gid: GroupId, + /// Access mode of the node pub mode: FileMode, } -#[derive(Default)] struct PropertyCache { metadata: Metadata, size: Option, } +/// Decribes a single entry within a filesytem pub struct Node { data: NodeImpl, flags: NodeFlags, @@ -81,47 +93,75 @@ pub struct Node { parent: IrqSafeSpinlock>, } -impl Default for Metadata { - fn default() -> Metadata { +impl Metadata { + pub(crate) const fn default_dir() -> Metadata { Metadata { uid: UserId::root(), gid: GroupId::root(), - mode: FileMode::new(0), + mode: FileMode::new(0o755), + } + } + + pub(crate) const fn default_file() -> Metadata { + Metadata { + uid: UserId::root(), + gid: GroupId::root(), + mode: FileMode::new(0o644), } } } impl Node { - fn new(data: NodeImpl, flags: NodeFlags) -> NodeRef { + fn new(data: NodeImpl, flags: NodeFlags, metadata: Metadata) -> NodeRef { Arc::new(Self { data, flags, - props: IrqSafeSpinlock::new(PropertyCache::default()), + props: IrqSafeSpinlock::new(PropertyCache { + metadata, + size: None, + }), parent: IrqSafeSpinlock::new(None), }) } + /// Creates a new directory node with given [DirectoryImpl] pub fn directory(data: T, flags: NodeFlags) -> NodeRef { let data = NodeImpl::Directory(DirectoryData { imp: Box::new(data), mountpoint: IrqSafeSpinlock::new(None), children: IrqSafeSpinlock::new(Vec::new()), }); - Self::new(data, flags) + Self::new(data, flags, Metadata::default_dir()) } + /// Creates a new file node with given [RegularImpl] pub fn regular(data: T, flags: NodeFlags) -> NodeRef { - Self::new(NodeImpl::Regular(Box::new(data)), flags) + Self::new( + NodeImpl::Regular(Box::new(data)), + flags, + Metadata::default_file(), + ) } + /// Creates a new block device node with given [BlockDevice] pub fn block(device: &'static dyn BlockDevice, flags: NodeFlags) -> NodeRef { - Self::new(NodeImpl::Block(BlockDeviceWrapper(device)), flags) + Self::new( + NodeImpl::Block(BlockDeviceWrapper(device)), + flags, + Metadata::default_file(), + ) } + /// Creates a new character device node with given [CharDevice] pub fn char(device: &'static dyn CharDevice, flags: NodeFlags) -> NodeRef { - Self::new(NodeImpl::Char(CharDeviceWrapper(device)), flags) + Self::new( + NodeImpl::Char(CharDeviceWrapper(device)), + flags, + Metadata::default_file(), + ) } + /// Creates a new symbolic link node with given [SymlinkImpl] pub fn symlink(data: T, flags: NodeFlags) -> NodeRef { Self::new( NodeImpl::Symlink(SymlinkData { @@ -129,9 +169,11 @@ impl Node { imp: Box::new(data), }), flags, + Metadata::default_file(), ) } + /// Returns the impl data of the node as `dyn Any` pub fn data_as_any(&self) -> &dyn Any { match &self.data { NodeImpl::Directory(dir) => dir.imp.as_any(), @@ -142,6 +184,7 @@ impl Node { } } + /// Returns the impl data of the node as `dyn CommonImpl` pub fn data_as_common(&self) -> &dyn CommonImpl { match &self.data { NodeImpl::Directory(dir) => dir.imp.as_ref(), @@ -152,10 +195,12 @@ impl Node { } } + /// Attempts to cast the impl data of the node to `&T` pub fn data_as_ref(&self) -> &T { self.data_as_any().downcast_ref().unwrap() } + /// Returns the type of the node pub fn ty(&self) -> FileType { match &self.data { NodeImpl::Regular(_) => FileType::File, @@ -166,10 +211,13 @@ impl Node { } } + /// Returns `true` if the node represents a directory pub fn is_directory(&self) -> bool { matches!(&self.data, NodeImpl::Directory(_)) } + /// Returns `true` if the node represents a character device and the character device is a + /// terminal pub fn is_terminal(&self) -> bool { if let NodeImpl::Char(dev) = &self.data { dev.is_terminal() diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs index 735c31a1..ff7860f3 100644 --- a/lib/vfs/src/node/ops.rs +++ b/lib/vfs/src/node/ops.rs @@ -2,15 +2,30 @@ use core::mem::MaybeUninit; use yggdrasil_abi::{ error::Error, - io::{DirectoryEntry, OpenOptions}, + io::{DeviceRequest, DirectoryEntry, FileMode, GroupId, OpenOptions, UserId}, }; use crate::file::{File, FileRef}; -use super::{CreateInfo, Metadata, Node, NodeFlags, NodeImpl, NodeRef}; +use super::{AccessToken, CreateInfo, Metadata, Node, NodeFlags, NodeImpl, NodeRef}; impl Node { - pub fn open(self: &NodeRef, opts: OpenOptions) -> Result { + // Devices + + /// Performs a device-specific function on a the device node + pub fn device_request(self: &NodeRef, req: &mut DeviceRequest) -> Result<(), Error> { + match &self.data { + NodeImpl::Block(dev) => dev.0.device_request(req), + NodeImpl::Char(dev) => dev.0.device_request(req), + _ => Err(Error::InvalidOperation), + } + } + + // Devices + files + + /// Opens the node with given [OpenOptions]. Only works for regular files and devices. For + /// directories, use [Node::open_directory]. + pub fn open(self: &NodeRef, opts: OpenOptions, _check: AccessToken) -> Result { match &self.data { NodeImpl::Regular(imp) => { let (pos, instance) = imp.open(self, opts)?; @@ -25,12 +40,14 @@ impl Node { // Directory - pub fn open_directory(self: &NodeRef) -> Result { + /// Opens the node as a directory for reading its entries + pub fn open_directory(self: &NodeRef, _check: AccessToken) -> Result { let dir = self.as_directory()?; let pos = dir.imp.open(self)?; Ok(File::directory(self.clone(), pos)) } + /// Reads entries from the directory pub fn read_directory( self: &NodeRef, pos: u64, @@ -39,7 +56,14 @@ impl Node { self.as_directory()?.imp.read_entries(self, pos, entries) } - pub fn lookup_or_load(self: &NodeRef, name: &str) -> Result { + /// Attempts to look up a child node with given name inside the directory node in the tree + /// cache. If no such node is present there, will attempt to fetch it from the underlying + /// filesystem. + pub fn lookup_or_load( + self: &NodeRef, + name: &str, + _check: AccessToken, + ) -> Result { let dir = self.as_directory()?; let children = dir.children.lock(); @@ -52,16 +76,87 @@ impl Node { Err(Error::DoesNotExist) } - pub fn create(self: &NodeRef, info: &CreateInfo) -> Result { + /// Creates an entry within a directory with given [CreateInfo]. + pub fn create(self: &NodeRef, info: &CreateInfo, check: AccessToken) -> Result { let directory = self.as_directory()?; - let node = directory.imp.create(self, info)?; + let node = directory.imp.create_node(self, info.ty)?; + + // Fill out the node info + node.set_access(Some(info.uid), Some(info.gid), Some(info.mode), check)?; + + match directory.imp.attach_node(self, &node, &info.name) { + Ok(_) | Err(Error::NotImplemented) => (), + Err(err) => return Err(err), + } + // Attach the created node to the directory in memory cache self.add_child(&info.name, node.clone())?; Ok(node) } + /// Removes a regular file, device or symlink from the directory + pub fn remove_file(self: &NodeRef, name: &str, check: AccessToken) -> Result<(), Error> { + let directory = self.as_directory()?; + let child = self.lookup_or_load(name, check)?; + + if child.is_directory() { + return Err(Error::IsADirectory); + } + + // Detach the node in the real filesystem + match directory.imp.unlink_node(self, name) { + Ok(_) | Err(Error::NotImplemented) => (), + Err(err) => return Err(err), + } + + // Detach the node in the tree cache + { + let mut children = directory.children.lock(); + children.retain(|(name_, _)| name != name_); + } + + // TODO child.destroy() or something? + Ok(()) + } + // Common + /// Changes user/group ID or access mode of the node + pub fn set_access( + self: &NodeRef, + uid: Option, + gid: Option, + mode: Option, + _check: AccessToken, + ) -> Result<(), Error> { + if uid.is_none() && gid.is_none() && mode.is_none() { + return Err(Error::InvalidOperation); + } + + let mut metadata = self.metadata()?; + + if let Some(uid) = uid { + metadata.uid = uid; + } + if let Some(gid) = gid { + metadata.gid = gid; + } + if let Some(mode) = mode { + metadata.mode = mode; + } + + // Update cached props + self.props.lock().metadata = metadata; + + if !self.flags.contains(NodeFlags::IN_MEMORY_PROPS) { + // Update permissions in the real node + todo!(); + } + + Ok(()) + } + + /// Returns the "metadata" of the file: uid, gid, access mode pub fn metadata(self: &NodeRef) -> Result { if self.flags.contains(NodeFlags::IN_MEMORY_PROPS) { let props = self.props.lock(); @@ -71,6 +166,8 @@ impl Node { self.data_as_common().metadata(self) } + // TODO clarify directory size + /// Returns the size in bytes of the node pub fn size(self: &NodeRef) -> Result { // Try to fetch the size from the cache let mut props = self.props.lock(); diff --git a/lib/vfs/src/node/traits.rs b/lib/vfs/src/node/traits.rs index 21a94f13..499559dc 100644 --- a/lib/vfs/src/node/traits.rs +++ b/lib/vfs/src/node/traits.rs @@ -3,30 +3,36 @@ use core::{any::Any, mem::MaybeUninit}; use yggdrasil_abi::{ error::Error, - io::{DirectoryEntry, OpenOptions}, + io::{DirectoryEntry, FileType, OpenOptions}, }; use crate::file::DirectoryOpenPosition; -use super::{CreateInfo, Metadata, NodeRef}; +use super::{Metadata, NodeRef}; +/// Common interface shared by all filesystem nodes #[allow(unused)] pub trait CommonImpl { + /// Returns `&self` as a reference to `dyn Any` fn as_any(&self) -> &dyn Any { unimplemented!(); } + /// Fetches the metadata of the file from underlying storage fn metadata(&self, node: &NodeRef) -> Result { Err(Error::NotImplemented) } + /// Fetches the size of the file from underlying storage fn size(&self, node: &NodeRef) -> Result { Err(Error::NotImplemented) } } +/// Regular file interface #[allow(unused)] pub trait RegularImpl: CommonImpl { + /// Opens the file for reading/writing and returns a `(start position, instance data)` tuple fn open( &self, node: &NodeRef, @@ -35,10 +41,12 @@ pub trait RegularImpl: CommonImpl { Err(Error::NotImplemented) } + /// Closes a file fn close(&self, node: &NodeRef, instance: Option<&Box>) -> Result<(), Error> { Ok(()) } + /// Reads data from the file into given buffer fn read( &self, node: &NodeRef, @@ -48,6 +56,7 @@ pub trait RegularImpl: CommonImpl { ) -> Result { Err(Error::NotImplemented) } + /// Writes data to the file from given buffer fn write( &self, node: &NodeRef, @@ -59,12 +68,16 @@ pub trait RegularImpl: CommonImpl { } } +/// Directory implementation #[allow(unused)] pub trait DirectoryImpl: CommonImpl { + /// Opens a directory for reading its entries. Returns [DirectoryOpenPosition] to specify the + /// starting position. fn open(&self, node: &NodeRef) -> Result { Err(Error::NotImplemented) } + /// Fetches entries from a directory into given buffer fn read_entries( &self, node: &NodeRef, @@ -74,25 +87,41 @@ pub trait DirectoryImpl: CommonImpl { Err(Error::NotImplemented) } - fn create(&self, parent: &NodeRef, info: &CreateInfo) -> Result { + /// Creates a child node, but does not associate it with the directory yet + fn create_node(&self, parent: &NodeRef, ty: FileType) -> Result { + Err(Error::ReadOnly) + } + + /// Associates the given node with the directory, creating an entry for it inside + fn attach_node(&self, parent: &NodeRef, child: &NodeRef, name: &str) -> Result<(), Error> { Err(Error::NotImplemented) } + /// Removes an entry of the directory with given name + fn unlink_node(&self, parent: &NodeRef, name: &str) -> Result<(), Error> { + Err(Error::NotImplemented) + } + + /// Fetches the child of the directory with given name fn lookup(&self, node: &NodeRef, name: &str) -> Result { Err(Error::NotImplemented) } + /// Returns the "length" of the directory in entries fn len(&self, node: &NodeRef) -> Result { Err(Error::NotImplemented) } } +/// Symbolic link interface #[allow(unused)] pub trait SymlinkImpl: CommonImpl { + /// Returns the target node (if such is available directly) of the link fn target(&self, node: &NodeRef) -> Result { Err(Error::NotImplemented) } + /// Fetches the contents of the symlink into a [String] fn read_to_string(&self) -> Result { let mut data = Box::new([0; 512]); let len = self.read_link(data.as_mut())?; @@ -103,6 +132,7 @@ pub trait SymlinkImpl: CommonImpl { Ok(String::from(str)) } + /// Fetches the contents of the symlink into a buffer fn read_link(&self, buf: &mut [u8]) -> Result { Err(Error::NotImplemented) } diff --git a/lib/vfs/src/node/tree.rs b/lib/vfs/src/node/tree.rs index 04e2f541..5e701fa1 100644 --- a/lib/vfs/src/node/tree.rs +++ b/lib/vfs/src/node/tree.rs @@ -5,15 +5,23 @@ use alloc::string::String; use super::{Node, NodeRef}; impl Node { + /// Returns the parent node of this node, or itself if no parent is present pub fn parent(self: &NodeRef) -> NodeRef { self.parent.lock().as_ref().unwrap_or(self).clone() } + /// Returns `true` if this node is a root + pub fn is_root(self: &NodeRef) -> bool { + self.parent.lock().is_none() + } + + /// Returns the count of entries in the directory pub fn children_len(&self) -> Result { let directory = self.as_directory()?; Ok(directory.children.lock().len()) } + /// Adds an entry to the directory tree cache pub fn add_child>( self: &NodeRef, name: S, diff --git a/lib/vfs/src/traits.rs b/lib/vfs/src/traits.rs index f5a7525a..96f39940 100644 --- a/lib/vfs/src/traits.rs +++ b/lib/vfs/src/traits.rs @@ -1,16 +1,36 @@ use yggdrasil_abi::{error::Error, io::SeekFrom}; +/// Immutable read interface for VFS objects pub trait Read { + /// Reads bytes into the given buffer fn read(&self, buf: &mut [u8]) -> Result; + + /// Reads exactly `buf.len()` bytes into the given buffer, fails if such amount is not + /// available + fn read_exact(&self, buf: &mut [u8]) -> Result<(), Error> { + match self.read(buf) { + Ok(count) if count == buf.len() => Ok(()), + Ok(_) => Err(Error::InvalidFile), + Err(err) => Err(err), + } + } } +/// Immutable write interface for VFS objects pub trait Write { + /// Writes bytes from the given buffer fn write(&self, buf: &[u8]) -> Result; } +/// Immutable file positioning interface for VFS objects pub trait Seek { + /// Changes position inside the file to a requested one. Fails if the file does not support + /// seeking. fn seek(&self, from: SeekFrom) -> Result; + /// Returns the position within the file, determined as an offset in bytes from the beginning + /// of the file. Fails if the file does not support seeking or the "offset" is not defined for + /// such type of nodes. fn tell(&self) -> Result { self.seek(SeekFrom::Current(0)) } diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index ad97337e..f89317c5 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -12,6 +12,7 @@ use super::{ }; use crate::{ arch::{aarch64::mem::table::L3, Architecture}, + fs::devfs, kernel_main, kernel_secondary_main, mem::{address::IntoRaw, phys, table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET}, task::runtime, @@ -99,11 +100,7 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { exception::init_exceptions(); - // // Setup initrd - // super::setup_initrd(); - - // XXX - // devfs::init(); + devfs::init(); runtime::init_task_queue(); diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 90490168..1163da49 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -27,6 +27,7 @@ use crate::{ devtree::{self, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}, power::arm_psci::Psci, }, + fs::{Initrd, INITRD_DATA}, mem::{ address::{FromRaw, IntoRaw}, device::RawDeviceMemoryMapping, @@ -314,12 +315,11 @@ impl AArch64 { let data = unsafe { PhysicalRef::map_slice(initrd_start, len) }; self.initrd.init(data); - // XXX - // INITRD_DATA.init(Initrd { - // phys_page_start: aligned_start, - // phys_page_len: aligned_end - aligned_start, - // data: self.initrd.get().as_ref(), - // }); + INITRD_DATA.init(Initrd { + phys_page_start: aligned_start, + phys_page_len: aligned_end - aligned_start, + data: self.initrd.get().as_ref(), + }); } Ok(()) diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 9380f8f7..3112c622 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -15,7 +15,7 @@ use device_api::{ interrupt::{InterruptHandler, IpiDeliveryTarget}, Device, }; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use yggdrasil_abi::error::Error; use crate::{ @@ -27,7 +27,6 @@ use crate::{ address::FromRaw, heap::GLOBAL_HEAP, pointer::PhysicalRef, read_memory, write_memory, PhysicalAddress, }, - sync::IrqSafeSpinlock, }; use super::intrinsics; diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 93d8f3d2..9e5c39af 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -8,6 +8,7 @@ use device_api::{ }, Device, }; +use kernel_util::sync::IrqSafeSpinlock; use tock_registers::{ interfaces::{Readable, Writeable}, register_structs, @@ -17,7 +18,6 @@ use tock_registers::{ use crate::{ arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}, mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, - sync::IrqSafeSpinlock, }; use super::{APIC_EXTERNAL_OFFSET, MAX_EXTERNAL_VECTORS}; diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index b0f548e7..740f5583 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -2,7 +2,7 @@ use core::{ptr::null_mut, sync::atomic::Ordering}; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::interfaces::Writeable; use crate::{ @@ -10,7 +10,6 @@ use crate::{ x86_64::{cpuid, gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, CpuMessage, }, - sync::IrqSafeSpinlock, task::sched::CpuQueue, }; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 37272ea3..2f1e55f0 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -9,7 +9,7 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, Device, }; use git_version::git_version; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::SpinFence, util::OneTimeInit}; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; mod acpi; @@ -55,7 +55,6 @@ use crate::{ table::EntryLevel, PhysicalAddress, }, - sync::SpinFence, }; use self::{ diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index ab0bb491..3eab2a46 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -2,6 +2,7 @@ use core::time::Duration; use abi::error::Error; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; +use kernel_util::sync::IrqSafeSpinlock; use crate::{ arch::{ @@ -11,7 +12,6 @@ use crate::{ }, Architecture, ARCHITECTURE, }, - sync::IrqSafeSpinlock, task::runtime, }; diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index 8e50b1a5..dccaa5f3 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -5,16 +5,14 @@ use device_api::{ interrupt::InterruptHandler, Device, }; +use kernel_util::sync::IrqSafeSpinlock; -use crate::{ - arch::{ - x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - IrqNumber, - }, - Architecture, ARCHITECTURE, +use crate::arch::{ + x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + IrqNumber, }, - sync::IrqSafeSpinlock, + Architecture, ARCHITECTURE, }; use self::codeset::{CODE_SET_1_00, CODE_SET_1_00_SHIFT}; diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index f8b87635..ae263164 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -1,6 +1,7 @@ //! Driver for x86 COM ports use abi::error::Error; use device_api::{serial::SerialDevice, Device}; +use kernel_util::sync::IrqSafeSpinlock; use crate::{ arch::x86_64::{ @@ -8,7 +9,6 @@ use crate::{ IrqNumber, }, debug::DebugSink, - sync::IrqSafeSpinlock, }; // Single port diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index c5ee8ad5..a2fbd400 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -5,6 +5,7 @@ use acpi_lib::mcfg::McfgEntry; use alloc::{rc::Rc, vec::Vec}; use bitflags::bitflags; use device_api::Device; +use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::error::Error; mod space; @@ -13,10 +14,7 @@ pub use space::{ ecam::PciEcam, PciConfigSpace, PciConfigurationSpace, PciLegacyConfigurationSpace, }; -use crate::{ - mem::{address::FromRaw, PhysicalAddress}, - sync::IrqSafeSpinlock, -}; +use crate::mem::{address::FromRaw, PhysicalAddress}; bitflags! { /// Command register of the PCI configuration space diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 53fc5225..58910108 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -8,6 +8,7 @@ use tock_registers::{ register_bitfields, register_structs, registers::{ReadOnly, ReadWrite, WriteOnly}, }; +use vfs::CharDevice; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, @@ -18,6 +19,7 @@ use crate::{ tty::{TtyContext, TtyDevice}, }, device_tree_driver, + fs::devfs::{self, CharDeviceType}, mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, task::process::ProcessId, }; @@ -122,22 +124,38 @@ impl TtyDevice for Pl011 { // } // } // -// fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { -// match req { -// &mut DeviceRequest::SetTerminalGroup(id) => { -// self.set_signal_group(ProcessId::from(id)); -// Ok(()) -// } -// DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), -// DeviceRequest::GetTerminalOptions(config) => { -// config.write(self.context.config()); -// Ok(()) -// } -// _ => Err(Error::InvalidArgument), -// } -// } // } +impl CharDevice for Pl011 { + fn write(&self, data: &[u8]) -> Result { + self.line_write(data) + } + + fn read(&self, data: &mut [u8]) -> Result { + match block! { + self.line_read(data).await + } { + Ok(res) => res, + Err(err) => Err(err), + } + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(ProcessId::from(id)); + Ok(()) + } + DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), + DeviceRequest::GetTerminalOptions(config) => { + config.write(self.context.config()); + Ok(()) + } + _ => Err(Error::InvalidArgument), + } + } +} + impl SerialDevice for Pl011 { fn send(&self, byte: u8) -> Result<(), Error> { self.inner.get().lock().send_byte(byte) @@ -190,7 +208,7 @@ impl Device for Pl011 { self.inner.init(IrqSafeSpinlock::new(inner)); debug::add_sink(self, LogLevel::Debug); - // devfs::add_char_device(self, CharDeviceType::TtySerial)?; + devfs::add_char_device(self, CharDeviceType::TtySerial)?; Ok(()) } diff --git a/src/device/tty.rs b/src/device/tty.rs index dfc80398..7cb9bfcc 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -21,6 +21,7 @@ pub mod combined { io::{DeviceRequest, TerminalSize}, }; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; + use vfs::CharDevice; // use vfs::CharDevice; use crate::device::{ @@ -73,43 +74,40 @@ pub mod combined { } } - // impl CharDevice for CombinedTerminal { - // fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { - // assert!(blocking); - // match block! { - // self.line_read(data).await - // } { - // Ok(res) => res, - // Err(err) => Err(err), - // } - // // self.line_read(data) - // } + impl CharDevice for CombinedTerminal { + fn read(&'static self, data: &mut [u8]) -> Result { + match block! { + self.line_read(data).await + } { + Ok(res) => res, + Err(err) => Err(err), + } + } - // fn write(&self, blocking: bool, data: &[u8]) -> Result { - // assert!(blocking); - // self.line_write(data) - // } + fn write(&self, data: &[u8]) -> Result { + self.line_write(data) + } - // fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - // match req { - // &mut DeviceRequest::SetTerminalGroup(id) => { - // self.set_signal_group(ProcessId::from(id)); - // Ok(()) - // } - // DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), - // DeviceRequest::GetTerminalOptions(config) => { - // config.write(self.context.config()); - // Ok(()) - // } - // DeviceRequest::GetTerminalSize(out) => { - // let (rows, columns) = self.output.text_dimensions(); - // out.write(TerminalSize { rows, columns }); - // Ok(()) - // } - // _ => Err(Error::InvalidArgument), - // } - // } - // } + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + &mut DeviceRequest::SetTerminalGroup(id) => { + self.set_signal_group(ProcessId::from(id)); + Ok(()) + } + DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), + DeviceRequest::GetTerminalOptions(config) => { + config.write(self.context.config()); + Ok(()) + } + DeviceRequest::GetTerminalSize(out) => { + let (rows, columns) = self.output.text_dimensions(); + out.write(TerminalSize { rows, columns }); + Ok(()) + } + _ => Err(Error::InvalidArgument), + } + } + } } #[cfg(feature = "fb_console")] diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index 279a82de..1532da11 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -1,18 +1,15 @@ //! Device virtual file system -use core::{ - any::Any, - sync::atomic::{AtomicUsize, Ordering}, +use core::sync::atomic::{AtomicU32, AtomicU8, AtomicUsize, Ordering}; + +use abi::error::Error; +use alloc::{format, string::String, sync::Arc}; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use vfs::{ + impls::{mdir, read_fn_node, MemoryDirectory}, + CharDevice, Node, NodeFlags, NodeRef, }; -use abi::{ - error::Error, - io::{FileAttr, FileMode, FileType, OpenOptions}, -}; -use alloc::{boxed::Box, format, string::String}; -use kernel_util::util::OneTimeInit; -use vfs::{ - CharDevice, CharDeviceWrapper, Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE, -}; +use crate::arch::{Architecture, ARCHITECTURE}; /// Describes the kind of a character device #[derive(Debug)] @@ -23,34 +20,63 @@ pub enum CharDeviceType { TtySerial, } -struct DevfsDirectory; +static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); -impl VnodeImpl for DevfsDirectory { - fn open( - &self, - _node: &VnodeRef, - _opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - Ok((DIR_POSITION_FROM_CACHE, None)) +struct RandomState { + data: [u8; 1024], + pos: usize, + last_state: u32, +} + +impl RandomState { + fn new(seed: u32) -> Self { + Self { + data: [0; 1024], + pos: 1024, + last_state: seed, + } } - fn metadata(&self, _node: &VnodeRef) -> Result { - Ok(FileAttr { - size: 0, - // TODO mutable directory mode - mode: FileMode::from(0o755), - ty: FileType::Directory, - }) + fn next(&mut self) -> u32 { + let mut x = self.last_state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + self.last_state = x; + x + } + + fn fill_buf(&mut self) { + self.pos = 0; + for i in (0..self.data.len()).step_by(4) { + let v = self.next(); + self.data[i..i + 4].copy_from_slice(&v.to_ne_bytes()); + } + } + + fn read_buf(&mut self, buf: &mut [u8]) { + let mut rem = buf.len(); + let mut pos = 0; + + while rem != 0 { + if self.pos == 1024 { + self.fill_buf(); + } + let count = core::cmp::min(rem, 1024 - self.pos); + + buf[pos..pos + count].copy_from_slice(&self.data[self.pos..self.pos + count]); + + self.pos += count; + rem -= count; + pos += count; + } } } -static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); - /// Sets up the device filesystem pub fn init() { - let node = Vnode::new("", VnodeKind::Directory); - node.set_data(Box::new(DevfsDirectory)); - DEVFS_ROOT.init(node); + let root = MemoryDirectory::empty(); + DEVFS_ROOT.init(root); } /// Returns the root of the devfs. @@ -58,19 +84,16 @@ pub fn init() { /// # Panics /// /// Will panic if the devfs hasn't yet been initialized. -pub fn root() -> &'static VnodeRef { +pub fn root() -> &'static NodeRef { DEVFS_ROOT.get() } fn _add_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Error> { infoln!("Add char device: {}", name); - let node = Vnode::new(name, VnodeKind::Char); - node.set_data(Box::new(CharDeviceWrapper::new(dev))); + let node = Node::char(dev, NodeFlags::IN_MEMORY_PROPS); - DEVFS_ROOT.get().add_child(node); - - Ok(()) + DEVFS_ROOT.get().add_child(name, node) } /// Adds a character device to the devfs @@ -88,3 +111,24 @@ pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Re _add_char_device(dev, name) } + +pub fn add_pseudo_devices() -> Result<(), Error> { + let now = ARCHITECTURE + .monotonic_timer() + .monotonic_timestamp() + .unwrap(); + let random_seed = now.subsec_millis(); + let mut random_state = RandomState::new(random_seed); + random_state.fill_buf(); + let random_state = IrqSafeSpinlock::new(random_state); + + let random = read_fn_node(move |_, buf| { + random_state.lock().read_buf(buf); + Ok(buf.len()) + }); + + let root = root(); + root.add_child("random", random)?; + + Ok(()) +} diff --git a/src/fs/mod.rs b/src/fs/mod.rs index f0cd4033..3bd2d748 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -4,7 +4,7 @@ use core::ptr::NonNull; use kernel_util::util::OneTimeInit; use memfs::block::{self, BlockAllocator}; -use vfs::VnodeRef; +use vfs::NodeRef; use yggdrasil_abi::{error::Error, io::MountOptions}; use crate::mem::{phys, PhysicalAddress}; @@ -44,7 +44,7 @@ unsafe impl BlockAllocator for FileBlockAllocator { } /// Constructs an instance of a filesystem for given set of [MountOptions] -pub fn create_filesystem(options: &MountOptions) -> Result { +pub fn create_filesystem(options: &MountOptions) -> Result { let fs_name = options.filesystem.unwrap(); match fs_name { diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 994574fe..88aa1649 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -10,161 +10,16 @@ use alloc::{ }; use git_version::git_version; use kernel_util::util::OneTimeInit; -use vfs::{Vnode, VnodeImpl, VnodeKind, VnodeRef, DIR_POSITION_FROM_CACHE}; +use vfs::{ + impls::{const_value_node, mdir, read_fn_node}, + NodeRef, +}; use crate::{debug, util}; -trait GetterFn = Fn() -> Result; -trait ReaderFn = Fn(u64, &mut [u8]) -> Result; +static ROOT: OneTimeInit = OneTimeInit::new(); -struct SysfsDirectory; - -struct SysfsGetterNode> { - getter: R, - _pd: PhantomData, -} - -struct SysfsReaderNode { - reader: R, -} - -impl> SysfsGetterNode { - pub const fn new(getter: R) -> Self { - Self { - getter, - _pd: PhantomData, - } - } -} - -impl SysfsReaderNode { - pub const fn new(reader: R) -> Self { - Self { reader } - } -} - -impl VnodeImpl for SysfsDirectory { - fn open( - &self, - _node: &VnodeRef, - _opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - Ok((DIR_POSITION_FROM_CACHE, None)) - } - - fn metadata(&self, _node: &VnodeRef) -> Result { - Ok(FileAttr { - size: 0, - // TODO mutable directory mode - mode: FileMode::from(0o755), - ty: FileType::Directory, - }) - } -} - -impl> VnodeImpl for SysfsGetterNode { - fn open( - &self, - _node: &VnodeRef, - _opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - let value = (self.getter)()?; - let inner = Box::new(value.to_string()); - Ok((0, Some(inner))) - } - - fn read( - &self, - node: &VnodeRef, - pos: u64, - inner: Option<&mut Box>, - data: &mut [u8], - ) -> Result { - let string = inner.unwrap().downcast_ref::().unwrap(); - let pos = pos as usize; - let bytes = string.as_bytes(); - - if pos >= bytes.len() { - return Ok(0); - } - - let len = core::cmp::min(bytes.len() - pos, data.len()); - - data[..len].copy_from_slice(&bytes[pos..pos + len]); - - Ok(len) - } - - fn size(&self, node: &VnodeRef) -> Result { - Ok(0) - } - - fn metadata(&self, node: &VnodeRef) -> Result { - Ok(FileAttr { - size: 0, - mode: FileMode::from(0o444), - ty: FileType::File, - }) - } -} - -impl VnodeImpl for SysfsReaderNode { - fn open( - &self, - node: &VnodeRef, - opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { - Ok((0, None)) - } - - fn read( - &self, - node: &VnodeRef, - pos: u64, - _inner: Option<&mut Box>, - data: &mut [u8], - ) -> Result { - (self.reader)(pos, data) - } - - fn size(&self, node: &VnodeRef) -> Result { - todo!() - } - - fn metadata(&self, node: &VnodeRef) -> Result { - todo!() - } -} - -static ROOT: OneTimeInit = OneTimeInit::new(); - -fn getter, T: ToString + 'static, R: GetterFn + 'static>( - name: S, - get: R, -) -> VnodeRef { - let data = Box::new(SysfsGetterNode::new(get)); - let node = Vnode::new(name, VnodeKind::Regular); - node.set_data(data); - node -} - -fn reader, R: ReaderFn + 'static>(name: S, read: R) -> VnodeRef { - let data = Box::new(SysfsReaderNode::new(read)); - let node = Vnode::new(name, VnodeKind::Regular); - node.set_data(data); - node -} - -fn dir, I: IntoIterator>(name: S, items: I) -> VnodeRef { - let node = Vnode::new(name, VnodeKind::Directory); - node.set_data(Box::new(SysfsDirectory)); - for item in items { - node.add_child(item); - } - node -} - -pub fn root() -> &'static VnodeRef { +pub fn root() -> &'static NodeRef { ROOT.get() } @@ -173,15 +28,15 @@ fn read_kernel_log(pos: u64, buffer: &mut [u8]) -> Result { } pub fn init() { - let kernel = dir( - "kernel", - [ - getter("version", || Ok(env!("CARGO_PKG_VERSION"))), - getter("rev", || Ok(git_version!())), - reader("log", read_kernel_log), - ], - ); - let root = dir("", [kernel, getter("arch", || Ok(util::arch_str()))]); + let d_kernel = mdir([ + ("version", const_value_node(env!("CARGO_PKG_VERSION"))), + ("rev", const_value_node(git_version!())), + ("log", read_fn_node(read_kernel_log)), + ]); + let root = mdir([ + ("kernel", d_kernel), + ("arch", const_value_node(util::arch_str())), + ]); ROOT.init(root); } diff --git a/src/init.rs b/src/init.rs index efa38b4e..26e4d011 100644 --- a/src/init.rs +++ b/src/init.rs @@ -3,8 +3,19 @@ use abi::{ error::Error, io::{OpenOptions, RawFd}, }; +use memfs::MemoryFilesystem; +use vfs::{Action, IoContext, NodeRef}; -use crate::proc; +use crate::{ + fs::{devfs, FileBlockAllocator, INITRD_DATA}, + proc, +}; + +fn setup_root() -> Result { + let initrd_data = INITRD_DATA.get(); + let fs = MemoryFilesystem::::from_slice(initrd_data.data).unwrap(); + fs.root() +} /// Kernel's "main" process function. /// @@ -14,47 +25,50 @@ use crate::proc; /// initialization has finished. pub fn kinit() -> Result<(), Error> { infoln!("In main"); - loop {} - // #[cfg(feature = "fb_console")] - // { - // use crate::{device::display::console::update_consoles_task, task::runtime}; + #[cfg(feature = "fb_console")] + { + use crate::{device::display::console::update_consoles_task, task::runtime}; - // runtime::spawn(async move { - // update_consoles_task().await; - // })?; - // } + runtime::spawn(async move { + update_consoles_task().await; + })?; + } - // let root = setup_root()?; + let root = setup_root()?; - // let ioctx = IoContext::new(root); - // let node = ioctx.find(None, "/init", true, true)?; - // let file = node.open(OpenOptions::READ)?; + let mut ioctx = IoContext::new(root); + let file = ioctx.open_exec(None, "/init")?; - // let devfs = devfs::root(); - // #[cfg(target_arch = "x86_64")] - // let console = ioctx.find(Some(devfs.clone()), "tty0", true, true)?; - // #[cfg(target_arch = "aarch64")] - // let console = ioctx.find(Some(devfs.clone()), "ttyS0", true, true)?; - // let stdin = console.open(OpenOptions::READ)?; - // let stdout = console.open(OpenOptions::WRITE)?; - // let stderr = stdout.clone(); + let devfs = devfs::root(); - // { - // // XXX - // let (user_init, user_init_main) = - // proc::exec::load_elf("init", file, &["/init", "xxx"], &[])?; - // let mut io = user_init.io.lock(); - // io.set_ioctx(ioctx); - // io.set_file(RawFd::STDIN, stdin)?; - // io.set_file(RawFd::STDOUT, stdout)?; - // io.set_file(RawFd::STDERR, stderr)?; - // drop(io); + #[cfg(target_arch = "x86_64")] + let console = ioctx.find(Some(devfs.clone()), "tty0", true, true)?; + #[cfg(target_arch = "aarch64")] + let console = ioctx.find(Some(devfs.clone()), "ttyS0", true, true)?; - // user_init.set_session_terminal(console); + let access = ioctx.check_access(Action::Read, &console)?; + let stdin = console.open(OpenOptions::READ, access)?; + let access = ioctx.check_access(Action::Write, &console)?; + let stdout = console.open(OpenOptions::WRITE, access)?; + let stderr = stdout.clone(); - // user_init_main.enqueue_somewhere(); - // } + { + // XXX + let (user_init, user_init_main) = + proc::exec::load_elf("init", file, &["/init", "xxx"], &[])?; - // Ok(()) + let mut io = user_init.io.lock(); + io.set_ioctx(ioctx); + io.set_file(RawFd::STDIN, stdin)?; + io.set_file(RawFd::STDOUT, stdout)?; + io.set_file(RawFd::STDERR, stderr)?; + drop(io); + + user_init.set_session_terminal(console); + + user_init_main.enqueue_somewhere(); + } + + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 34e110b2..d82b454e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,7 @@ use kernel_util::sync::SpinFence; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, + fs::{devfs, sysfs}, mem::heap, task::{spawn_kernel_closure, Cpu}, }; @@ -50,7 +51,7 @@ pub mod debug; pub mod arch; pub mod device; -// pub mod fs; +pub mod fs; pub mod init; pub mod mem; pub mod panic; @@ -87,7 +88,8 @@ pub fn kernel_main() -> ! { debugln!("Heap: {:#x?}", heap::heap_range()); // Setup the sysfs - // sysfs::init(); + sysfs::init(); + devfs::add_pseudo_devices().unwrap(); unsafe { ARCHITECTURE.start_application_processors(); diff --git a/src/proc/elf.rs b/src/proc/elf.rs index a3c6820b..7584972f 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -27,15 +27,11 @@ struct FileReader<'a> { impl elf::io_traits::InputStream for FileReader<'_> { fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), elf::io_traits::StreamError> { - self.file - .borrow_mut() - .read_exact(buf) - .map_err(conv_stream_error) + self.file.read_exact(buf).map_err(conv_stream_error) } fn seek(&mut self, pos: elf::io_traits::SeekFrom) -> Result { self.file - .borrow_mut() .seek(conv_seek_from(pos)) .map_err(conv_stream_error) } @@ -179,9 +175,9 @@ fn load_segment( space, phdr.p_vaddr as usize, |off, mut dst| { - let mut source = file.file.borrow_mut(); - source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; - source.read_exact(dst.deref_mut()) + file.file + .seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + file.file.read_exact(dst.deref_mut()) }, phdr.p_filesz as usize, )?; @@ -247,9 +243,9 @@ fn tls_segment( space, base_address + data_offset, |off, mut dst| { - let mut source = file.file.borrow_mut(); - source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?; - source.read_exact(dst.deref_mut()) + file.file + .seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + file.file.read_exact(dst.deref_mut()) }, data_size, )?; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index d42b0821..f1566591 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -7,6 +7,7 @@ use abi::{ process::ProgramArgumentInner, }; use alloc::{string::String, sync::Arc}; +use vfs::FileRef; use crate::{ mem::{ @@ -145,8 +146,7 @@ fn setup_binary>( } } - // XXX - let tls_address = 0; // proc::elf::clone_tls(&space, &image)?; + let tls_address = proc::elf::clone_tls(&space, &image)?; let context = TaskContext::user( image.entry, @@ -165,15 +165,15 @@ fn setup_binary>( // Ok(Process::new_with_context(name, Some(space), context)) } -// /// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory -// pub fn load_elf>( -// name: S, -// file: FileRef, -// args: &[&str], -// envs: &[&str], -// ) -> Result<(Arc, Arc), Error> { -// let space = ProcessAddressSpace::new()?; -// let image = proc::elf::load_elf_from_file(&space, file)?; -// -// setup_binary(name, space, image, args, envs) -// } +/// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory +pub fn load_elf>( + name: S, + file: FileRef, + args: &[&str], + envs: &[&str], +) -> Result<(Arc, Arc), Error> { + let space = ProcessAddressSpace::new()?; + let image = proc::elf::load_elf_from_file(&space, file)?; + + setup_binary(name, space, image, args, envs) +} diff --git a/src/proc/io.rs b/src/proc/io.rs index 9c42f970..78fc6dea 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -1,12 +1,67 @@ //! Process I/O management +use abi::{error::Error, io::RawFd}; +use alloc::collections::{btree_map::Entry, BTreeMap}; +use kernel_util::sync::IrqSafeSpinlock; +use vfs::{FileRef, IoContext}; + /// I/O context of a process, contains information like root, current directory and file /// descriptor table -pub struct ProcessIo {} +pub struct ProcessIo { + ioctx: Option, + files: BTreeMap, +} impl ProcessIo { /// Constructs an uninitialized I/O context pub fn new() -> Self { - todo!() + Self { + ioctx: None, + files: BTreeMap::new(), + } + } + + pub fn file(&self, fd: RawFd) -> Result<&FileRef, Error> { + self.files.get(&fd).ok_or(Error::InvalidFile) + } + + pub fn ioctx(&mut self) -> &mut IoContext { + self.ioctx.as_mut().unwrap() + } + + pub fn set_ioctx(&mut self, ioctx: IoContext) { + self.ioctx = Some(ioctx); + } + + pub fn set_file(&mut self, fd: RawFd, file: FileRef) -> Result<(), Error> { + if self.files.contains_key(&fd) { + return Err(Error::AlreadyExists); + } + self.files.insert(fd, file); + Ok(()) + } + + pub fn place_file(&mut self, file: FileRef) -> Result { + for idx in 0..64 { + let fd = RawFd::from(idx); + + if let Entry::Vacant(e) = self.files.entry(fd) { + e.insert(file); + return Ok(fd); + } + } + + // TODO OutOfFiles + Err(Error::OutOfMemory) + } + + pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { + // Do nothing, file will be dropped and closed + self.files.remove(&fd).ok_or(Error::InvalidFile)?; + Ok(()) + } + + pub fn retain bool>(&mut self, predicate: F) { + self.files.retain(predicate); } } diff --git a/src/proc/mod.rs b/src/proc/mod.rs index c460b3ac..aa280e3b 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -1,5 +1,5 @@ //! Internal management for processes -// pub mod elf; +pub mod elf; pub mod exec; pub mod io; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index bf401418..96ed3eed 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,11 +4,13 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, + path::Path, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; -use alloc::rc::Rc; +use alloc::{rc::Rc, sync::Arc}; use kernel_util::sync::IrqSafeSpinlockGuard; +use vfs::{CreateInfo, IoContext, NodeRef, Read, Seek, Write}; // use vfs::{IoContext, Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; use yggdrasil_abi::{ error::SyscallResult, @@ -18,6 +20,7 @@ use yggdrasil_abi::{ use crate::{ block, debug::LogLevel, + fs, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo}, task::{ @@ -30,26 +33,29 @@ use crate::{ mod arg; use arg::*; -// fn run_with_io) -> T>(proc: &Process, f: F) -> T { -// let io = proc.io.lock(); -// f(io) -// } -// -// fn run_with_io_at< -// T, -// F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, -// >( -// proc: &Process, -// at: Option, -// f: F, -// ) -> Result { -// let io = proc.io.lock(); -// let at = at -// .map(|fd| io.file(fd).and_then(|f| f.borrow().node())) -// .transpose()?; -// -// f(at, io) -// } +fn run_with_io) -> T>(proc: &Process, f: F) -> T { + let io = proc.io.lock(); + f(io) +} + +fn run_with_io_at< + T, + F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, +>( + proc: &Process, + at: Option, + f: F, +) -> Result { + let io = proc.io.lock(); + let at = at + .map(|fd| { + io.file(fd) + .and_then(|f| f.node().ok_or(Error::InvalidFile).cloned()) + }) + .transpose()?; + + f(at, io) +} fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { let thread = Thread::current(); @@ -118,7 +124,206 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } // I/O + SyscallFunction::Open => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let opts = OpenOptions::from(args[3] as u32); + let mode = FileMode::from(args[4] as u32); + + run_with_io_at(&process, at, |at, mut io| { + let file = io.ioctx().open(at, path, opts, mode)?; + + // TODO NO_CTTY? + if process.session_terminal().is_none() && + let Some(node) = file.node() && node.is_terminal() { + debugln!("Session terminal set for #{}: {}", process.id(), path); + process.set_session_terminal(node.clone()); + } + + let fd = io.place_file(file)?; + Ok(fd.0 as usize) + }) + } + SyscallFunction::OpenDirectory => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + + run_with_io_at(&process, at, |at, mut io| { + let node = io.ioctx().find(at, path, true, true)?; + let access = io.ioctx().check_access(vfs::Action::Read, &node)?; + let file = node.open_directory(access)?; + let fd = io.place_file(file)?; + + Ok(fd.0 as usize) + }) + } + SyscallFunction::Read => { + let fd = RawFd(args[0] as u32); + let data = arg_buffer_mut(args[1] as _, args[2] as _)?; + + run_with_io(&process, |io| io.file(fd)?.read(data)) + } + SyscallFunction::Write => { + let fd = RawFd(args[0] as u32); + let data = arg_buffer_ref(args[1] as _, args[2] as _)?; + + run_with_io(&process, |io| io.file(fd)?.write(data)) + } + SyscallFunction::Seek => { + let fd = RawFd(args[0] as u32); + let pos = SeekFrom::from(args[1]); + + run_with_io(&process, |io| io.file(fd)?.seek(pos).map(|v| v as usize)) + } + SyscallFunction::ReadDirectory => { + let fd = RawFd(args[0] as u32); + let buffer = arg_user_slice_mut::>( + args[1] as usize, + args[2] as usize, + )?; + + run_with_io(&process, |io| io.file(fd)?.read_dir(buffer)) + } + SyscallFunction::Close => { + let fd = RawFd(args[0] as u32); + + run_with_io(&process, |mut io| { + io.close_file(fd)?; + Ok(0) + }) + } + SyscallFunction::Mount => { + let options = arg_user_ref::(args[0] as usize)?; + + run_with_io(&process, |mut io| { + let fs_root = fs::create_filesystem(options)?; + io.ioctx().mount(options.target, fs_root)?; + Ok(0) + }) + } + SyscallFunction::Unmount => { + todo!(); + } + SyscallFunction::DeviceRequest => { + let fd = RawFd(args[0] as u32); + let req = arg_user_mut::(args[1] as usize)?; + + run_with_io(&process, |io| { + let file = io.file(fd)?; + let node = file.node().ok_or(Error::InvalidFile)?; + node.device_request(req)?; + Ok(0) + }) + } + SyscallFunction::GetMetadata => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let buffer = arg_user_mut::>(args[3] as usize)?; + let follow = args[4] != 0; + + run_with_io_at(&process, at, |at, mut io| { + let node = if path.is_empty() { + at.ok_or(Error::InvalidArgument)? + } else { + io.ioctx().find(None, path, follow, true)? + }; + + let metadata = node.metadata()?; + let size = node.size()?; + + buffer.write(FileAttr { + size, + ty: node.ty(), + mode: metadata.mode, + }); + + Ok(0) + }) + } + SyscallFunction::CreateDirectory => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let mode = FileMode::from(args[3] as u32); + + run_with_io_at(&process, at, |at, mut io| { + io.ioctx().create_directory(at, path, mode)?; + Ok(0) + }) + } + SyscallFunction::Remove => { + let at = arg_option_fd(args[0] as u32); + let path = arg_user_str(args[1] as usize, args[2] as usize)?; + let recurse = args[3] != 0; + + run_with_io_at(&process, at, |at, mut io| { + io.ioctx().remove_file(at, path)?; + Ok(0) + }) + } + SyscallFunction::RemoveDirectory => { + todo!() + } // Process management + SyscallFunction::SpawnProcess => { + let options = arg_user_ref::(args[0] as usize)?; + + run_with_io(&process, |mut io| { + // let node = io.ioctx().find(None, options.program, true, true)?; + + // Setup a new process from the file + let file = io.ioctx().open_exec(None, options.program)?; + // let file = node.open(OpenOptions::READ)?; + let (child_process, child_main) = proc::exec::load_elf( + options.program, + file, + options.arguments, + options.environment, + )?; + let pid: u32 = child_process.id().into(); + + // Inherit group and session from the creator + child_process.inherit(&process)?; + + // Inherit root from the creator + // let child_ioctx = IoContext::new(io.ioctx().root().clone()); + let child_ioctx = IoContext::inherit(io.ioctx()); + let mut child_io = child_process.io.lock(); + child_io.set_ioctx(child_ioctx); + + for opt in options.optional { + match opt { + &SpawnOption::InheritFile { source, child } => { + let src_file = io.file(source)?; + child_io.set_file(child, src_file.clone())?; + } + &SpawnOption::SetProcessGroup(pgroup) => { + child_process.set_group_id(pgroup.into()); + } + _ => (), + } + } + + if let Some(fd) = options.optional.iter().find_map(|item| { + if let &SpawnOption::GainTerminal(fd) = item { + Some(fd) + } else { + None + } + }) { + debugln!("{} requested terminal {:?}", pid, fd); + let file = child_io.file(fd)?; + let node = file.node().ok_or(Error::InvalidFile)?; + let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); + + node.device_request(&mut req)?; + } + + drop(child_io); + child_main.enqueue_somewhere(); + + Ok(pid as _) + }) + } SyscallFunction::SpawnThread => { let options = arg_user_ref::(args[0] as usize)?; let id = process.spawn_thread(options)?; @@ -180,6 +385,25 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result process.set_group_id(group_id); Ok(0) } + SyscallFunction::StartSession => { + let session_terminal = process.clear_session_terminal(); + + if let Some(ctty) = session_terminal { + // Drop all FDs referring to the old session terminal + run_with_io(&process, |mut io| { + io.retain(|_, f| { + f.node() + .map(|node| !Arc::ptr_eq(&node, &ctty)) + .unwrap_or(true) + }); + }); + } + + process.set_session_id(process.id()); + process.set_group_id(process.id()); + + Ok(0) + } // Waiting and polling SyscallFunction::WaitProcess => { let pid = ProcessId::from(args[0] as u32); @@ -197,10 +421,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } - - _ => { - todo!("System call: {:?}", func); - } } } diff --git a/src/task/process.rs b/src/task/process.rs index 341649b7..a6863098 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -20,6 +20,7 @@ use alloc::{ }; use futures_util::Future; use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use vfs::NodeRef; use crate::{ mem::{ @@ -129,7 +130,7 @@ struct ProcessInner { session_id: ProcessId, group_id: ProcessId, - // session_terminal: Option, + session_terminal: Option, threads: Vec>, mutexes: BTreeMap>, } @@ -170,7 +171,7 @@ impl Process { state: ProcessState::Running, session_id: id, group_id: id, - // session_terminal: None, + session_terminal: None, threads: Vec::new(), mutexes: BTreeMap::new(), }), @@ -245,28 +246,28 @@ impl Process { } // Resources - // pub fn session_terminal(&self) -> Option { - // self.inner.lock().session_terminal.clone() - // } + pub fn session_terminal(&self) -> Option { + self.inner.lock().session_terminal.clone() + } - // pub fn set_session_terminal(&self, node: VnodeRef) { - // self.inner.lock().session_terminal.replace(node); - // } + pub fn set_session_terminal(&self, node: NodeRef) { + self.inner.lock().session_terminal.replace(node); + } - // pub fn clear_session_terminal(&self) -> Option { - // self.inner.lock().session_terminal.take() - // } + pub fn clear_session_terminal(&self) -> Option { + self.inner.lock().session_terminal.take() + } - // pub fn inherit(&self, parent: &Process) -> Result<(), Error> { - // let mut our_inner = self.inner.lock(); - // let their_inner = parent.inner.lock(); + pub fn inherit(&self, parent: &Process) -> Result<(), Error> { + let mut our_inner = self.inner.lock(); + let their_inner = parent.inner.lock(); - // our_inner.session_id = their_inner.session_id; - // our_inner.group_id = their_inner.group_id; - // our_inner.session_terminal = their_inner.session_terminal.clone(); + our_inner.session_id = their_inner.session_id; + our_inner.group_id = their_inner.group_id; + our_inner.session_terminal = their_inner.session_terminal.clone(); - // Ok(()) - // } + Ok(()) + } // State pub fn get_exit_status(&self) -> Option { From 15d3e38a2dd0963c2bde59f5207d25b36dc50011 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 5 Dec 2023 12:23:44 +0200 Subject: [PATCH 100/211] proc: add SyscallFunction::GetRandom --- src/fs/devfs.rs | 68 ++++-------------------------------------- src/main.rs | 3 ++ src/proc/mod.rs | 1 + src/proc/random.rs | 74 ++++++++++++++++++++++++++++++++++++++++++++++ src/syscall/mod.rs | 7 ++++- 5 files changed, 89 insertions(+), 64 deletions(-) create mode 100644 src/proc/random.rs diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index 1532da11..cbc1170b 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -9,7 +9,10 @@ use vfs::{ CharDevice, Node, NodeFlags, NodeRef, }; -use crate::arch::{Architecture, ARCHITECTURE}; +use crate::{ + arch::{Architecture, ARCHITECTURE}, + proc::random, +}; /// Describes the kind of a character device #[derive(Debug)] @@ -21,58 +24,6 @@ pub enum CharDeviceType { } static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); - -struct RandomState { - data: [u8; 1024], - pos: usize, - last_state: u32, -} - -impl RandomState { - fn new(seed: u32) -> Self { - Self { - data: [0; 1024], - pos: 1024, - last_state: seed, - } - } - - fn next(&mut self) -> u32 { - let mut x = self.last_state; - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - self.last_state = x; - x - } - - fn fill_buf(&mut self) { - self.pos = 0; - for i in (0..self.data.len()).step_by(4) { - let v = self.next(); - self.data[i..i + 4].copy_from_slice(&v.to_ne_bytes()); - } - } - - fn read_buf(&mut self, buf: &mut [u8]) { - let mut rem = buf.len(); - let mut pos = 0; - - while rem != 0 { - if self.pos == 1024 { - self.fill_buf(); - } - let count = core::cmp::min(rem, 1024 - self.pos); - - buf[pos..pos + count].copy_from_slice(&self.data[self.pos..self.pos + count]); - - self.pos += count; - rem -= count; - pos += count; - } - } -} - /// Sets up the device filesystem pub fn init() { let root = MemoryDirectory::empty(); @@ -113,17 +64,8 @@ pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Re } pub fn add_pseudo_devices() -> Result<(), Error> { - let now = ARCHITECTURE - .monotonic_timer() - .monotonic_timestamp() - .unwrap(); - let random_seed = now.subsec_millis(); - let mut random_state = RandomState::new(random_seed); - random_state.fill_buf(); - let random_state = IrqSafeSpinlock::new(random_state); - let random = read_fn_node(move |_, buf| { - random_state.lock().read_buf(buf); + random::read(buf); Ok(buf.len()) }); diff --git a/src/main.rs b/src/main.rs index d82b454e..bc37fb16 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,6 +37,7 @@ use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, fs::{devfs, sysfs}, mem::heap, + proc::random, task::{spawn_kernel_closure, Cpu}, }; @@ -87,6 +88,8 @@ pub fn kernel_secondary_main() -> ! { pub fn kernel_main() -> ! { debugln!("Heap: {:#x?}", heap::heap_range()); + random::init(); + // Setup the sysfs sysfs::init(); devfs::add_pseudo_devices().unwrap(); diff --git a/src/proc/mod.rs b/src/proc/mod.rs index aa280e3b..3e1e2bf1 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -3,3 +3,4 @@ pub mod elf; pub mod exec; pub mod io; +pub mod random; diff --git a/src/proc/random.rs b/src/proc/random.rs new file mode 100644 index 00000000..d1c42212 --- /dev/null +++ b/src/proc/random.rs @@ -0,0 +1,74 @@ +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; + +use crate::arch::{Architecture, ARCHITECTURE}; + +struct RandomState { + data: [u8; 1024], + pos: usize, + last_state: u32, +} + +impl RandomState { + fn new(seed: u32) -> Self { + Self { + data: [0; 1024], + pos: 1024, + last_state: seed, + } + } + + fn next(&mut self) -> u32 { + let mut x = self.last_state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + self.last_state = x; + x + } + + fn fill_buf(&mut self) { + self.pos = 0; + for i in (0..self.data.len()).step_by(4) { + let v = self.next(); + self.data[i..i + 4].copy_from_slice(&v.to_ne_bytes()); + } + } + + fn read_buf(&mut self, buf: &mut [u8]) { + let mut rem = buf.len(); + let mut pos = 0; + + while rem != 0 { + if self.pos == 1024 { + self.fill_buf(); + } + let count = core::cmp::min(rem, 1024 - self.pos); + + buf[pos..pos + count].copy_from_slice(&self.data[self.pos..self.pos + count]); + + self.pos += count; + rem -= count; + pos += count; + } + } +} + +static RANDOM_STATE: OneTimeInit> = OneTimeInit::new(); + +pub fn read(buf: &mut [u8]) { + let state = RANDOM_STATE.get(); + state.lock().read_buf(buf) +} + +pub fn init() { + let now = ARCHITECTURE + .monotonic_timer() + .monotonic_timestamp() + .unwrap(); + let random_seed = now.subsec_millis(); + + let mut state = RandomState::new(random_seed); + state.fill_buf(); + + RANDOM_STATE.init(IrqSafeSpinlock::new(state)); +} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 96ed3eed..1879b955 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -22,7 +22,7 @@ use crate::{ debug::LogLevel, fs, mem::{phys, table::MapAttributes}, - proc::{self, io::ProcessIo}, + proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, runtime, @@ -72,6 +72,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } + SyscallFunction::GetRandom => { + let buf = arg_buffer_mut(args[0] as _, args[1] as _)?; + random::read(buf); + Ok(0) + } SyscallFunction::Nanosleep => { let seconds = args[0]; let nanos = args[1] as u32; From b6946443b87283b3255eeeaf82953b446f9c8a35 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 5 Dec 2023 14:55:12 +0200 Subject: [PATCH 101/211] refactor: fix warnings --- lib/memfs/Cargo.toml | 1 + lib/memfs/src/block.rs | 2 +- lib/memfs/src/file.rs | 27 ++++----- lib/memfs/src/lib.rs | 4 +- lib/vfs/src/device.rs | 4 +- lib/vfs/src/file/device.rs | 24 ++++---- lib/vfs/src/file/directory.rs | 12 ++-- lib/vfs/src/file/mod.rs | 64 +++++++++++++------- lib/vfs/src/file/regular.rs | 27 +++++---- lib/vfs/src/lib.rs | 3 +- lib/vfs/src/node/impls.rs | 105 +++++++++++++++++++------------- lib/vfs/src/node/mod.rs | 11 +++- lib/vfs/src/node/traits.rs | 12 ++-- src/arch/aarch64/exception.rs | 4 +- src/debug.rs | 20 ++----- src/device/tty.rs | 2 +- src/fs/devfs.rs | 13 ++-- src/fs/sysfs.rs | 11 +--- src/panic.rs | 110 +++++++++++++++------------------- src/proc/elf.rs | 8 +-- src/proc/exec.rs | 5 +- src/proc/io.rs | 1 - src/syscall/mod.rs | 44 ++++++-------- src/task/mod.rs | 51 +--------------- src/task/process.rs | 25 ++------ src/task/sched.rs | 40 +------------ src/task/thread.rs | 39 +++++------- src/util/ring.rs | 15 +++-- 28 files changed, 293 insertions(+), 391 deletions(-) diff --git a/lib/memfs/Cargo.toml b/lib/memfs/Cargo.toml index ccfe38f0..fae1b5db 100644 --- a/lib/memfs/Cargo.toml +++ b/lib/memfs/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../kernel-util" } vfs = { path = "../vfs" } static_assertions = "1.1.0" diff --git a/lib/memfs/src/block.rs b/lib/memfs/src/block.rs index 0b732bd1..9510098f 100644 --- a/lib/memfs/src/block.rs +++ b/lib/memfs/src/block.rs @@ -19,7 +19,7 @@ pub const ENTRY_COUNT: usize = SIZE / size_of::(); /// /// This trait is unsafe to implement because it has to provide and accept raw data pointers of /// exactly [SIZE]. -pub unsafe trait BlockAllocator: 'static { +pub unsafe trait BlockAllocator: Send + Sync + 'static { /// Allocates a contiguous block of size [SIZE] fn alloc() -> Result, Error>; diff --git a/lib/memfs/src/file.rs b/lib/memfs/src/file.rs index c1b7bd27..78e95920 100644 --- a/lib/memfs/src/file.rs +++ b/lib/memfs/src/file.rs @@ -1,21 +1,20 @@ -use core::{any::Any, cell::RefCell}; +use core::any::Any; -use alloc::boxed::Box; -use vfs::{CommonImpl, Node, NodeFlags, NodeRef, RegularImpl}; +use kernel_util::sync::IrqSafeSpinlock; +use vfs::{CommonImpl, InstanceData, Node, NodeFlags, NodeRef, RegularImpl}; use yggdrasil_abi::{error::Error, io::OpenOptions}; use crate::{block::BlockAllocator, bvec::BVec}; pub(crate) struct FileNode { - // TODO IrqSafeSpinlock - pub(crate) data: RefCell>, + pub(crate) data: IrqSafeSpinlock>, } impl FileNode { pub fn new() -> NodeRef { Node::regular( Self { - data: RefCell::new(BVec::new()), + data: IrqSafeSpinlock::new(BVec::new()), }, NodeFlags::IN_MEMORY_PROPS, ) @@ -28,7 +27,7 @@ impl CommonImpl for FileNode { } fn size(&self, _node: &NodeRef) -> Result { - Ok(self.data.borrow().size() as u64) + Ok(self.data.lock().size() as u64) } } @@ -37,10 +36,10 @@ impl RegularImpl for FileNode { &self, _node: &NodeRef, opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { + ) -> Result<(u64, Option), Error> { // TODO provide APPEND by vfs driver instead if opts.contains(OpenOptions::APPEND) { - Ok((self.data.borrow().size() as u64, None)) + Ok((self.data.lock().size() as u64, None)) } else { Ok((0, None)) } @@ -49,24 +48,24 @@ impl RegularImpl for FileNode { fn read( &self, _node: &NodeRef, - _instance: Option<&Box>, + _instance: Option<&InstanceData>, pos: u64, buf: &mut [u8], ) -> Result { - self.data.borrow().read(pos, buf) + self.data.lock().read(pos, buf) } fn write( &self, _node: &NodeRef, - _instance: Option<&Box>, + _instance: Option<&InstanceData>, pos: u64, buf: &[u8], ) -> Result { - self.data.borrow_mut().write(pos, buf) + self.data.lock().write(pos, buf) } - fn close(&self, _node: &NodeRef, _instance: Option<&Box>) -> Result<(), Error> { + fn close(&self, _node: &NodeRef, _instance: Option<&InstanceData>) -> Result<(), Error> { Ok(()) } } diff --git a/lib/memfs/src/lib.rs b/lib/memfs/src/lib.rs index c08b0653..cd16449f 100644 --- a/lib/memfs/src/lib.rs +++ b/lib/memfs/src/lib.rs @@ -1,7 +1,7 @@ //! In-memory filesystem driver #![no_std] #![deny(missing_docs)] -#![allow(clippy::new_without_default)] +#![allow(clippy::new_without_default, clippy::new_ret_no_self)] #![feature( const_mut_refs, maybe_uninit_uninit_array, @@ -158,7 +158,7 @@ impl MemoryFilesystem { if hdr.node_kind() == FileType::File { let data = data.unwrap(); let node_data = node.data_as_ref::>(); - let mut bvec = node_data.data.borrow_mut(); + let mut bvec = node_data.data.lock(); bvec.init_with_cow(data)?; assert_eq!(bvec.size(), data.len()); diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs index 23f946d6..b76c6d6c 100644 --- a/lib/vfs/src/device.rs +++ b/lib/vfs/src/device.rs @@ -4,7 +4,7 @@ use crate::node::{CommonImpl, NodeRef}; /// Block device interface #[allow(unused)] -pub trait BlockDevice { +pub trait BlockDevice: Sync { /// Reads data frmo the given offset of the device fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { Err(Error::NotImplemented) @@ -36,7 +36,7 @@ pub trait BlockDevice { /// Character device interface #[allow(unused)] -pub trait CharDevice { +pub trait CharDevice: Sync { /// Reads data from the device fn read(&'static self, buf: &mut [u8]) -> Result { Err(Error::NotImplemented) diff --git a/lib/vfs/src/file/device.rs b/lib/vfs/src/file/device.rs index 58a73826..40c22b55 100644 --- a/lib/vfs/src/file/device.rs +++ b/lib/vfs/src/file/device.rs @@ -1,5 +1,4 @@ -use core::cell::Cell; - +use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ @@ -10,7 +9,7 @@ use crate::{ pub struct BlockFile { pub(super) device: BlockDeviceWrapper, pub(super) node: NodeRef, - pub(super) position: Cell, + pub(super) position: IrqSafeSpinlock, pub(super) read: bool, pub(super) write: bool, } @@ -24,24 +23,25 @@ pub struct CharFile { impl BlockFile { pub fn read(&self, buf: &mut [u8]) -> Result { - let pos = self.position.get(); - let count = self.device.0.read(pos, buf)?; - self.position.set(pos + count as u64); + let mut position = self.position.lock(); + let count = self.device.0.read(*position, buf)?; + *position += count as u64; Ok(count) } pub fn write(&self, buf: &[u8]) -> Result { - let pos = self.position.get(); - let count = self.device.0.write(pos, buf)?; - self.position.set(pos + count as u64); + let mut position = self.position.lock(); + let count = self.device.0.write(*position, buf)?; + *position += count as u64; Ok(count) } pub fn seek(&self, from: SeekFrom) -> Result { - let pos = self.position.get(); + let mut position = self.position.lock(); + let newpos = match from { SeekFrom::Current(off) => { - let newpos = i64::try_from(pos).unwrap() + off; + let newpos = i64::try_from(*position).unwrap() + off; if newpos < 0 { return Err(Error::InvalidArgument); } @@ -60,7 +60,7 @@ impl BlockFile { } }; - self.position.set(newpos); + *position = newpos; Ok(newpos) } } diff --git a/lib/vfs/src/file/directory.rs b/lib/vfs/src/file/directory.rs index 1043a8c5..4dda854c 100644 --- a/lib/vfs/src/file/directory.rs +++ b/lib/vfs/src/file/directory.rs @@ -1,5 +1,6 @@ -use core::{cell::Cell, mem::MaybeUninit, str::FromStr}; +use core::{mem::MaybeUninit, str::FromStr}; +use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::DirectoryEntry, util::FixedString}; use crate::node::NodeRef; @@ -21,7 +22,7 @@ pub(super) enum DirectoryPosition { pub struct DirectoryFile { pub(super) node: NodeRef, - pub(super) position: Cell, + pub(super) position: IrqSafeSpinlock, } impl DirectoryFile { @@ -79,8 +80,9 @@ impl DirectoryFile { &self, entries: &mut [MaybeUninit], ) -> Result { - let pos = self.position.get(); - let (count, pos) = match pos { + let mut position = self.position.lock(); + + let (count, pos) = match *position { DirectoryPosition::Cache(pos) => { let (count, pos) = DirectoryFile::read_cached(&self.node, pos, entries)?; (count, DirectoryPosition::Cache(pos)) @@ -90,7 +92,7 @@ impl DirectoryFile { (count, DirectoryPosition::Physical(pos)) } }; - self.position.set(pos); + *position = pos; Ok(count) } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 33f7acd9..00f0b5c8 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -1,6 +1,7 @@ -use core::{any::Any, cell::Cell, fmt, mem::MaybeUninit}; +use core::{any::Any, fmt, mem::MaybeUninit}; use alloc::{boxed::Box, sync::Arc}; +use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{ error::Error, io::{DirectoryEntry, OpenOptions, SeekFrom}, @@ -22,6 +23,9 @@ mod device; mod directory; mod regular; +/// Per-file optional instance data created when a regular file is opened +pub type InstanceData = Box; + /// Describes the starting position of the directory pub enum DirectoryOpenPosition { /// Contents should be fetched from the directory impl with given offset @@ -45,14 +49,14 @@ pub enum File { impl File { pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { - let position = Cell::new(position.into()); + let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) } pub(crate) fn regular( node: NodeRef, position: u64, - instance_data: Option>, + instance_data: Option, opts: OpenOptions, ) -> Arc { let read = opts.contains(OpenOptions::READ); @@ -63,7 +67,7 @@ impl File { read, write, instance_data, - position: Cell::new(position), + position: IrqSafeSpinlock::new(position), })) } @@ -85,7 +89,7 @@ impl File { Ok(Arc::new(Self::Block(BlockFile { device, node, - position: Cell::new(0), + position: IrqSafeSpinlock::new(0), read, write, }))) @@ -158,8 +162,8 @@ impl Write for File { impl Seek for File { fn tell(&self) -> Result { match self { - Self::Regular(file) => Ok(file.position.get()), - Self::Block(file) => Ok(file.position.get()), + Self::Regular(file) => Ok(*file.position.lock()), + Self::Block(file) => Ok(*file.position.lock()), Self::Char(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } @@ -180,13 +184,13 @@ impl fmt::Debug for File { match self { Self::Regular(file) => f .debug_struct("RegularFile") - .field("position", &file.position.get()) + .field("position", &*file.position.lock()) .field("read", &file.read) .field("write", &file.write) .finish_non_exhaustive(), Self::Block(file) => f .debug_struct("BlockFile") - .field("position", &file.position.get()) + .field("position", &*file.position.lock()) .field("read", &file.read) .field("write", &file.write) .finish_non_exhaustive(), @@ -202,8 +206,8 @@ impl fmt::Debug for File { #[cfg(test)] mod tests { - use core::{any::Any, cell::RefCell, mem::MaybeUninit, str::FromStr}; - use std::sync::Arc; + use core::{mem::MaybeUninit, str::FromStr}; + use std::sync::{Arc, Mutex}; use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{ @@ -215,10 +219,26 @@ mod tests { use crate::{ device::{BlockDevice, CharDevice}, file::DirectoryOpenPosition, + impls::const_value_node, node::{AccessToken, CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl}, traits::{Read, Seek, Write}, + InstanceData, }; + #[test] + fn file_send_sync() { + fn file_send(_f: &T) {} + fn file_sync(_f: &T) {} + + let node = const_value_node("1234"); + let file = node + .open(OpenOptions::READ, AccessToken::test_authorized()) + .unwrap(); + + file_send(&file); + file_sync(&file); + } + #[test] fn physical_dir_read() { struct D { @@ -350,12 +370,12 @@ mod tests { #[test] fn file_read_write() { struct F { - data: Arc>>, + data: Arc>>, } impl CommonImpl for F { fn size(&self, _node: &NodeRef) -> Result { - Ok(self.data.borrow().len() as _) + Ok(self.data.lock().unwrap().len() as _) } } impl RegularImpl for F { @@ -363,19 +383,19 @@ mod tests { &self, _node: &NodeRef, _opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { + ) -> Result<(u64, Option), Error> { Ok((0, None)) } fn read( &self, _node: &NodeRef, - _instance: Option<&Box>, + _instance: Option<&InstanceData>, pos: u64, buf: &mut [u8], ) -> Result { let pos = pos as usize; - let data = self.data.borrow(); + let data = self.data.lock().unwrap(); if pos >= data.len() { return Ok(0); } @@ -387,19 +407,19 @@ mod tests { fn write( &self, _node: &NodeRef, - _instance: Option<&Box>, + _instance: Option<&InstanceData>, pos: u64, buf: &[u8], ) -> Result { let pos = pos as usize; - let mut data = self.data.borrow_mut(); + let mut data = self.data.lock().unwrap(); data.resize(pos + buf.len(), 0); data[pos..pos + buf.len()].copy_from_slice(buf); Ok(buf.len()) } } - let data = Arc::new(RefCell::new(vec![])); + let data = Arc::new(Mutex::new(vec![])); let node = Node::regular(F { data: data.clone() }, NodeFlags::empty()); let file = node .open( @@ -409,19 +429,19 @@ mod tests { .unwrap(); let mut buf = [0; 512]; - assert_eq!(&*data.borrow(), &[]); + assert_eq!(&*data.lock().unwrap(), &[]); assert_eq!(file.tell().unwrap(), 0); assert_eq!(file.write(b"Hello").unwrap(), 5); assert_eq!(file.tell().unwrap(), 5); - assert_eq!(&*data.borrow(), b"Hello"); + assert_eq!(&*data.lock().unwrap(), b"Hello"); assert_eq!(file.seek(SeekFrom::End(-2)).unwrap(), 3); assert_eq!(file.tell().unwrap(), 3); assert_eq!(file.write(b"123456").unwrap(), 6); assert_eq!(file.tell().unwrap(), 9); - assert_eq!(&*data.borrow(), b"Hel123456"); + assert_eq!(&*data.lock().unwrap(), b"Hel123456"); assert_eq!(file.seek(SeekFrom::Start(2)).unwrap(), 2); assert_eq!(file.read(&mut buf).unwrap(), 7); diff --git a/lib/vfs/src/file/regular.rs b/lib/vfs/src/file/regular.rs index b5e64798..8d56df44 100644 --- a/lib/vfs/src/file/regular.rs +++ b/lib/vfs/src/file/regular.rs @@ -1,15 +1,15 @@ -use alloc::boxed::Box; -use core::{any::Any, cell::Cell}; +use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; +use super::InstanceData; use crate::node::NodeRef; pub struct RegularFile { pub(super) node: NodeRef, pub(super) read: bool, pub(super) write: bool, - pub(super) instance_data: Option>, - pub(super) position: Cell, + pub(super) instance_data: Option, + pub(super) position: IrqSafeSpinlock, } impl RegularFile { @@ -17,10 +17,10 @@ impl RegularFile { if !self.read { return Err(Error::InvalidFile); } + let mut position = self.position.lock(); let reg = self.node.as_regular()?; - let pos = self.position.get(); - let count = reg.read(&self.node, self.instance_data.as_ref(), pos, buf)?; - self.position.set(pos + count as u64); + let count = reg.read(&self.node, self.instance_data.as_ref(), *position, buf)?; + *position += count as u64; Ok(count) } @@ -28,19 +28,20 @@ impl RegularFile { if !self.write { return Err(Error::InvalidFile); } + let mut position = self.position.lock(); let reg = self.node.as_regular()?; - let pos = self.position.get(); - let count = reg.write(&self.node, self.instance_data.as_ref(), pos, buf)?; - self.position.set(pos + count as u64); + let count = reg.write(&self.node, self.instance_data.as_ref(), *position, buf)?; + *position += count as u64; Ok(count) } // TODO should seek beyond the end of a read-only file be allowed? pub fn seek(&self, from: SeekFrom) -> Result { - let pos = self.position.get(); + let mut position = self.position.lock(); + let newpos = match from { SeekFrom::Current(off) => { - let newpos = i64::try_from(pos).unwrap() + off; + let newpos = i64::try_from(*position).unwrap() + off; if newpos < 0 { return Err(Error::InvalidArgument); } @@ -60,7 +61,7 @@ impl RegularFile { } }; - self.position.set(newpos); + *position = newpos; Ok(newpos) } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index bbc7f178..33704419 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,6 +1,7 @@ //! Virtual filesystem interfaces and driver implementation #![cfg_attr(not(test), no_std)] +#![allow(clippy::new_ret_no_self)] #![deny(missing_docs)] #![feature( trait_upcasting, @@ -23,7 +24,7 @@ pub(crate) mod path; pub(crate) mod traits; pub use device::{BlockDevice, CharDevice}; -pub use file::{DirectoryOpenPosition, File, FileRef}; +pub use file::{DirectoryOpenPosition, File, FileRef, InstanceData}; pub use ioctx::{Action, IoContext}; pub use node::{ impls, AccessToken, CommonImpl, CreateInfo, DirectoryImpl, Metadata, Node, NodeFlags, NodeRef, diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index e406d29f..08a52f9a 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -1,5 +1,5 @@ //! Various helper node implementations for convenience -use core::{any::Any, cell::RefCell, marker::PhantomData, str::FromStr}; +use core::{marker::PhantomData, str::FromStr}; use alloc::{ boxed::Box, @@ -8,9 +8,10 @@ use alloc::{ vec::Vec, }; +use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::OpenOptions}; -use crate::DirectoryOpenPosition; +use crate::{DirectoryOpenPosition, InstanceData}; use super::{CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl, SymlinkImpl}; @@ -22,18 +23,18 @@ trait SliceWrite { fn write_slice(&mut self, offset: usize, buf: &[u8]) -> usize; } -trait IntoInstanceData { - fn into_instance_data(&self) -> Vec; +trait AsInstanceData { + fn as_instance_data(&self) -> Vec; } /// Closure interface for reading a single value -pub trait ValueReadFn = Fn() -> Result; +pub trait ValueReadFn = Fn() -> Result + Send + Sync; /// Closure interface for writing a single value -pub trait ValueWriteFn = Fn(T) -> Result<(), Error>; +pub trait ValueWriteFn = Fn(T) -> Result<(), Error> + Send + Sync; /// Closure interface for reading bytes -pub trait ReadFn = Fn(u64, &mut [u8]) -> Result; +pub trait ReadFn = Fn(u64, &mut [u8]) -> Result + Send + Sync; /// Closure interface for writing bytes -pub trait WriteFn = Fn(u64, &[u8]) -> Result; +pub trait WriteFn = Fn(u64, &[u8]) -> Result + Send + Sync; impl> SliceRead for T { fn read_slice(&self, pos: usize, buf: &mut [u8]) -> usize { @@ -57,15 +58,15 @@ impl SliceWrite for Vec { } } -impl IntoInstanceData for T { - fn into_instance_data(&self) -> Vec { +impl AsInstanceData for T { + fn as_instance_data(&self) -> Vec { self.to_string().as_bytes().to_vec() } } enum FnNodeData { Read(Vec), - Write(RefCell>), + Write(IrqSafeSpinlock>), } /// Allows read-only access to a value. The value is converted to a string representation when it's @@ -95,7 +96,11 @@ pub struct FixedSymlink { target: NodeRef, } -impl + 'static> ReadOnlyFnValueNode { +impl ReadOnlyFnValueNode +where + T: ToString + Send + Sync + 'static, + R: ValueReadFn + 'static, +{ /// Creates a new [ReadOnlyFnValueNode] with given read function pub fn new(read: R) -> NodeRef { Node::regular( @@ -112,29 +117,37 @@ impl + 'static> ReadOnlyFnValueNode> CommonImpl for ReadOnlyFnValueNode { +impl CommonImpl for ReadOnlyFnValueNode +where + T: ToString + Send + Sync, + R: ValueReadFn, +{ fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } -impl> RegularImpl for ReadOnlyFnValueNode { +impl RegularImpl for ReadOnlyFnValueNode +where + T: ToString + Send + Sync, + R: ValueReadFn, +{ fn open( &self, _node: &NodeRef, opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { + ) -> Result<(u64, Option), Error> { if opts.contains(OpenOptions::WRITE) { return Err(Error::ReadOnly); } let t = (self.read)()?; - Ok((0, Some(Box::new(t.into_instance_data())))) + Ok((0, Some(Box::new(t.as_instance_data())))) } fn read( &self, _node: &NodeRef, - instance: Option<&Box>, + instance: Option<&InstanceData>, pos: u64, buf: &mut [u8], ) -> Result { @@ -145,7 +158,7 @@ impl> RegularImpl for ReadOnlyFnValueNo fn write( &self, _node: &NodeRef, - _instance: Option<&Box>, + _instance: Option<&InstanceData>, _pos: u64, _buf: &[u8], ) -> Result { @@ -157,11 +170,11 @@ impl> RegularImpl for ReadOnlyFnValueNo impl FnNodeData { fn write() -> Self { - Self::Write(RefCell::new(Vec::new())) + Self::Write(IrqSafeSpinlock::new(Vec::new())) } - fn read(value: T) -> Self { - Self::Read(value.into_instance_data()) + fn read(value: T) -> Self { + Self::Read(value.as_instance_data()) } fn as_read(&self) -> Result<&Vec, Error> { @@ -171,7 +184,7 @@ impl FnNodeData { } } - fn as_write(&self) -> Result<&RefCell>, Error> { + fn as_write(&self) -> Result<&IrqSafeSpinlock>, Error> { match self { Self::Write(w) => Ok(w), Self::Read(_) => Err(Error::InvalidOperation), @@ -181,7 +194,7 @@ impl FnNodeData { impl FnValueNode where - T: ToString + FromStr + 'static, + T: ToString + FromStr + Send + Sync + 'static, R: ValueReadFn + 'static, W: ValueWriteFn + 'static, { @@ -202,22 +215,28 @@ where } } -impl, W: ValueWriteFn> CommonImpl - for FnValueNode +impl CommonImpl for FnValueNode +where + T: ToString + FromStr + Send + Sync, + R: ValueReadFn, + W: ValueWriteFn, { fn size(&self, _node: &NodeRef) -> Result { Ok(0) } } -impl, W: ValueWriteFn> RegularImpl - for FnValueNode +impl RegularImpl for FnValueNode +where + T: ToString + FromStr + Send + Sync, + R: ValueReadFn, + W: ValueWriteFn, { fn open( &self, _node: &NodeRef, opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { + ) -> Result<(u64, Option), Error> { if opts.contains(OpenOptions::READ | OpenOptions::WRITE) { Err(Error::InvalidOperation) } else if opts.contains(OpenOptions::WRITE) { @@ -230,14 +249,14 @@ impl, W: ValueWriteFn> RegularImpl } } - fn close(&self, _node: &NodeRef, instance: Option<&Box>) -> Result<(), Error> { + fn close(&self, _node: &NodeRef, instance: Option<&InstanceData>) -> Result<(), Error> { if let Ok(write) = instance .unwrap() .downcast_ref::() .unwrap() .as_write() { - let write = write.borrow(); + let write = write.lock(); // Flush write let str = core::str::from_utf8(write.as_ref()) .map_err(|_| Error::InvalidArgument)? @@ -252,7 +271,7 @@ impl, W: ValueWriteFn> RegularImpl fn read( &self, _node: &NodeRef, - instance: Option<&Box>, + instance: Option<&InstanceData>, pos: u64, buf: &mut [u8], ) -> Result { @@ -263,12 +282,12 @@ impl, W: ValueWriteFn> RegularImpl fn write( &self, _node: &NodeRef, - instance: Option<&Box>, + instance: Option<&InstanceData>, pos: u64, buf: &[u8], ) -> Result { let instance = instance.unwrap().downcast_ref::().unwrap(); - Ok(instance.as_write()?.borrow_mut().write_slice(pos as _, buf)) + Ok(instance.as_write()?.lock().write_slice(pos as _, buf)) } } @@ -301,7 +320,7 @@ where &self, _node: &NodeRef, opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { + ) -> Result<(u64, Option), Error> { if opts.contains(OpenOptions::WRITE) { return Err(Error::ReadOnly); } @@ -311,7 +330,7 @@ where fn read( &self, _node: &NodeRef, - _instance: Option<&Box>, + _instance: Option<&InstanceData>, pos: u64, buf: &mut [u8], ) -> Result { @@ -348,19 +367,25 @@ impl SymlinkImpl for FixedSymlink { } /// Creates a read-only value node with given `value` -pub fn const_value_node(value: T) -> NodeRef { +pub fn const_value_node(value: T) -> NodeRef +where + T: ToString + Clone + Send + Sync + 'static, +{ ReadOnlyFnValueNode::new(move || Ok(value.clone())) } /// Creates a read-write value node with given `value` -pub fn value_node(value: T) -> NodeRef { - let rd_state = Arc::new(RefCell::new(value)); +pub fn value_node(value: T) -> NodeRef +where + T: ToString + FromStr + Clone + Send + Sync + 'static, +{ + let rd_state = Arc::new(IrqSafeSpinlock::new(value)); let wr_state = rd_state.clone(); FnValueNode::new( - move || Ok(rd_state.borrow().clone()), + move || Ok(rd_state.lock().clone()), move |t| { - *wr_state.borrow_mut() = t; + *wr_state.lock() = t; Ok(()) }, ) diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index c7b55f09..9e5a76e4 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -295,7 +295,7 @@ mod tests { use core::any::Any; use std::sync::Arc; - use crate::node::NodeFlags; + use crate::{node::NodeFlags, NodeRef}; use super::{CommonImpl, DirectoryImpl, Node, RegularImpl}; @@ -308,6 +308,15 @@ mod tests { impl CommonImpl for DummyFile {} impl RegularImpl for DummyFile {} + #[test] + fn node_sync_send() { + fn node_send(_n: &T) {} + + let node = Node::regular(DummyFile, NodeFlags::empty()); + + node_send(&node); + } + #[test] fn dir_cache_add() { let d1 = Node::directory(DummyDirectory, NodeFlags::empty()); diff --git a/lib/vfs/src/node/traits.rs b/lib/vfs/src/node/traits.rs index 499559dc..823e3189 100644 --- a/lib/vfs/src/node/traits.rs +++ b/lib/vfs/src/node/traits.rs @@ -6,13 +6,13 @@ use yggdrasil_abi::{ io::{DirectoryEntry, FileType, OpenOptions}, }; -use crate::file::DirectoryOpenPosition; +use crate::file::{DirectoryOpenPosition, InstanceData}; use super::{Metadata, NodeRef}; /// Common interface shared by all filesystem nodes #[allow(unused)] -pub trait CommonImpl { +pub trait CommonImpl: Send + Sync { /// Returns `&self` as a reference to `dyn Any` fn as_any(&self) -> &dyn Any { unimplemented!(); @@ -37,12 +37,12 @@ pub trait RegularImpl: CommonImpl { &self, node: &NodeRef, opts: OpenOptions, - ) -> Result<(u64, Option>), Error> { + ) -> Result<(u64, Option), Error> { Err(Error::NotImplemented) } /// Closes a file - fn close(&self, node: &NodeRef, instance: Option<&Box>) -> Result<(), Error> { + fn close(&self, node: &NodeRef, instance: Option<&InstanceData>) -> Result<(), Error> { Ok(()) } @@ -50,7 +50,7 @@ pub trait RegularImpl: CommonImpl { fn read( &self, node: &NodeRef, - instance: Option<&Box>, + instance: Option<&InstanceData>, pos: u64, buf: &mut [u8], ) -> Result { @@ -60,7 +60,7 @@ pub trait RegularImpl: CommonImpl { fn write( &self, node: &NodeRef, - instance: Option<&Box>, + instance: Option<&InstanceData>, pos: u64, buf: &[u8], ) -> Result { diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 5f0784d9..6984c228 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -16,7 +16,7 @@ use abi::{ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{aarch64::cpu::Cpu, Architecture, ArchitectureImpl}, + arch::{Architecture, ArchitectureImpl}, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, thread::Thread}, @@ -117,7 +117,7 @@ pub fn init_exceptions() { } fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { - let cpu = Cpu::get_local(); + // let cpu = Cpu::get_local(); log_print_raw!(LogLevel::Fatal, "SYNC exception:\n"); log_print_raw!(LogLevel::Fatal, "FAR: {:#x}\n", FAR_EL1.get()); diff --git a/src/debug.rs b/src/debug.rs index e1021547..31f4976d 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,17 +2,9 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use alloc::sync::Arc; -use futures_util::Future; -use kernel_util::{ - sync::IrqSafeSpinlock, - util::{OneTimeInit, StaticVector}, -}; +use kernel_util::{sync::IrqSafeSpinlock, util::StaticVector}; -use crate::{ - task::{process::Process, runtime::QueueWaker}, - util::ring::RingBuffer, -}; +use crate::{task::process::Process, util::ring::RingBuffer}; const MAX_DEBUG_SINKS: usize = 4; const RING_LOGGER_CAPACITY: usize = 65536; @@ -25,7 +17,6 @@ struct RingLoggerInner { pub struct RingLoggerSink { inner: IrqSafeSpinlock, - waker: QueueWaker, } /// Defines the severity of the message @@ -175,12 +166,11 @@ impl RingLoggerSink { inner: IrqSafeSpinlock::new(RingLoggerInner { data: RingBuffer::new(0), }), - waker: QueueWaker::new(), } } pub fn read(&self, pos: usize, buffer: &mut [u8]) -> usize { - unsafe { self.inner.lock().data.read_all_static(pos, buffer) } + self.inner.lock().data.read_all_static(pos, buffer) } fn write_fmt(&self, args: fmt::Arguments<'_>) -> fmt::Result { @@ -192,9 +182,7 @@ impl RingLoggerSink { impl fmt::Write for RingLoggerInner { fn write_str(&mut self, s: &str) -> fmt::Result { for ch in s.bytes() { - unsafe { - self.data.write_unchecked(ch); - } + self.data.write(ch); } Ok(()) } diff --git a/src/device/tty.rs b/src/device/tty.rs index 7cb9bfcc..d4b3fa7f 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -299,7 +299,7 @@ impl TtyContext { } pub fn set_config(&self, config: &TerminalOptions) -> Result<(), Error> { - self.inner.lock().config = config.clone(); + self.inner.lock().config = *config; Ok(()) } diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index cbc1170b..63afcc59 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -1,18 +1,15 @@ //! Device virtual file system -use core::sync::atomic::{AtomicU32, AtomicU8, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicUsize, Ordering}; use abi::error::Error; -use alloc::{format, string::String, sync::Arc}; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use alloc::{format, string::String}; +use kernel_util::util::OneTimeInit; use vfs::{ - impls::{mdir, read_fn_node, MemoryDirectory}, + impls::{read_fn_node, MemoryDirectory}, CharDevice, Node, NodeFlags, NodeRef, }; -use crate::{ - arch::{Architecture, ARCHITECTURE}, - proc::random, -}; +use crate::proc::random; /// Describes the kind of a character device #[derive(Debug)] diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 88aa1649..4f0ff5f5 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -1,13 +1,4 @@ -use core::{any::Any, marker::PhantomData}; - -use abi::{ - error::Error, - io::{FileAttr, FileMode, FileType, OpenOptions}, -}; -use alloc::{ - boxed::Box, - string::{String, ToString}, -}; +use abi::error::Error; use git_version::git_version; use kernel_util::util::OneTimeInit; use vfs::{ diff --git a/src/panic.rs b/src/panic.rs index 14939ecf..2900e0fe 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -8,7 +8,7 @@ use crate::{ arch::{Architecture, ArchitectureImpl, CpuMessage, ARCHITECTURE}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, - task::{sched::CpuQueue, Cpu}, + task::Cpu, }; static PANIC_HANDLED_FENCE: SpinFence = SpinFence::new(); @@ -49,80 +49,66 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { static PANIC_HAPPENED: AtomicBool = AtomicBool::new(false); - infoln!("{:?}", pi); + fatalln!("{:?}", pi); - // if PANIC_HAPPENED - // .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) - // .is_ok() - // { - // let id = Cpu::local_id(); - // // Let other CPUs know we're screwed - // unsafe { - // ARCHITECTURE - // .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) - // .ok(); - // } + if PANIC_HAPPENED + .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) + .is_ok() + { + let id = Cpu::local_id(); + // Let other CPUs know we're screwed + unsafe { + ARCHITECTURE + .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) + .ok(); + } - // let ap_count = ArchitectureImpl::cpu_count() - 1; - // PANIC_HANDLED_FENCE.wait_all(ap_count); + let ap_count = ArchitectureImpl::cpu_count() - 1; + PANIC_HANDLED_FENCE.wait_all(ap_count); - // unsafe { - // hack_locks(); - // } + unsafe { + hack_locks(); + } - // log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); - // log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); - // log_print_raw!(LogLevel::Fatal, "Kernel panic "); + log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); + log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); + log_print_raw!(LogLevel::Fatal, "Kernel panic "); - // if let Some(location) = pi.location() { - // log_print_raw!( - // LogLevel::Fatal, - // "at {}:{}:", - // location.file(), - // location.line() - // ); - // } else { - // log_print_raw!(LogLevel::Fatal, ":"); - // } + if let Some(location) = pi.location() { + log_print_raw!( + LogLevel::Fatal, + "at {}:{}:", + location.file(), + location.line() + ); + } else { + log_print_raw!(LogLevel::Fatal, ":"); + } - // log_print_raw!(LogLevel::Fatal, "\n"); + log_print_raw!(LogLevel::Fatal, "\n"); - // if let Some(msg) = pi.message() { - // debug_internal(*msg, LogLevel::Fatal); - // log_print_raw!(LogLevel::Fatal, "\n"); - // } + if let Some(msg) = pi.message() { + debug_internal(*msg, LogLevel::Fatal); + log_print_raw!(LogLevel::Fatal, "\n"); + } - // for (i, queue) in CpuQueue::all().enumerate() { - // log_print_raw!(LogLevel::Fatal, "queue{}:\n", i); - // let lock = unsafe { queue.grab() }; - // for item in lock.iter() { - // log_print_raw!( - // LogLevel::Fatal, - // "* {} {:?} {:?}\n", - // item.id(), - // item.name(), - // item.state() - // ); - // } - // } + log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); - // log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); + PANIC_FINISHED_FENCE.signal(); + while PANIC_SEQUENCE.load(Ordering::Acquire) != id { + core::hint::spin_loop(); + } - // PANIC_FINISHED_FENCE.signal(); - // while PANIC_SEQUENCE.load(Ordering::Acquire) != id { - // core::hint::spin_loop(); - // } + log_print_raw!(LogLevel::Fatal, "X"); - // log_print_raw!(LogLevel::Fatal, "X"); + flush_consoles(); - // flush_consoles(); + PANIC_SEQUENCE.fetch_add(1, Ordering::Release); - // PANIC_SEQUENCE.fetch_add(1, Ordering::Release); - - // unsafe { - // ARCHITECTURE.reset(); - // } - // } + unsafe { + ARCHITECTURE.reset(); + } + } loop { ArchitectureImpl::wait_for_interrupt(); diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 7584972f..55959dc1 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -255,7 +255,7 @@ fn tls_segment( load_bytes( space, base_address + data_offset + data_size, - |off, mut dst| { + |_, mut dst| { dst.fill(0); Ok(()) }, @@ -274,12 +274,6 @@ pub fn load_elf_from_file( let file = FileReader { file: &file }; let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; - struct TlsInfo { - master_address: usize, - offset: usize, - size: usize, - } - let mut tls = None; for phdr in elf.segments() { diff --git a/src/proc/exec.rs b/src/proc/exec.rs index f1566591..823d1b28 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -11,10 +11,7 @@ use vfs::FileRef; use crate::{ mem::{ - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - process::ProcessAddressSpace, - table::MapAttributes, + phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, table::MapAttributes, ForeignPointer, }, proc, diff --git a/src/proc/io.rs b/src/proc/io.rs index 78fc6dea..51a3d56e 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -2,7 +2,6 @@ use abi::{error::Error, io::RawFd}; use alloc::collections::{btree_map::Entry, BTreeMap}; -use kernel_util::sync::IrqSafeSpinlock; use vfs::{FileRef, IoContext}; /// I/O context of a process, contains information like root, current directory and file diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 1879b955..797aa62e 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,18 +4,13 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, - path::Path, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; -use alloc::{rc::Rc, sync::Arc}; +use alloc::sync::Arc; use kernel_util::sync::IrqSafeSpinlockGuard; -use vfs::{CreateInfo, IoContext, NodeRef, Read, Seek, Write}; -// use vfs::{IoContext, Read, ReadDirectory, Seek, VnodeKind, VnodeRef, Write}; -use yggdrasil_abi::{ - error::SyscallResult, - io::{MountOptions, UnmountOptions}, -}; +use vfs::{IoContext, NodeRef, Read, Seek, Write}; +use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ block, @@ -135,7 +130,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let opts = OpenOptions::from(args[3] as u32); let mode = FileMode::from(args[4] as u32); - run_with_io_at(&process, at, |at, mut io| { + run_with_io_at(process, at, |at, mut io| { let file = io.ioctx().open(at, path, opts, mode)?; // TODO NO_CTTY? @@ -153,7 +148,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - run_with_io_at(&process, at, |at, mut io| { + run_with_io_at(process, at, |at, mut io| { let node = io.ioctx().find(at, path, true, true)?; let access = io.ioctx().check_access(vfs::Action::Read, &node)?; let file = node.open_directory(access)?; @@ -166,19 +161,19 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - run_with_io(&process, |io| io.file(fd)?.read(data)) + run_with_io(process, |io| io.file(fd)?.read(data)) } SyscallFunction::Write => { let fd = RawFd(args[0] as u32); let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - run_with_io(&process, |io| io.file(fd)?.write(data)) + run_with_io(process, |io| io.file(fd)?.write(data)) } SyscallFunction::Seek => { let fd = RawFd(args[0] as u32); let pos = SeekFrom::from(args[1]); - run_with_io(&process, |io| io.file(fd)?.seek(pos).map(|v| v as usize)) + run_with_io(process, |io| io.file(fd)?.seek(pos).map(|v| v as usize)) } SyscallFunction::ReadDirectory => { let fd = RawFd(args[0] as u32); @@ -187,12 +182,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result args[2] as usize, )?; - run_with_io(&process, |io| io.file(fd)?.read_dir(buffer)) + run_with_io(process, |io| io.file(fd)?.read_dir(buffer)) } SyscallFunction::Close => { let fd = RawFd(args[0] as u32); - run_with_io(&process, |mut io| { + run_with_io(process, |mut io| { io.close_file(fd)?; Ok(0) }) @@ -200,7 +195,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::Mount => { let options = arg_user_ref::(args[0] as usize)?; - run_with_io(&process, |mut io| { + run_with_io(process, |mut io| { let fs_root = fs::create_filesystem(options)?; io.ioctx().mount(options.target, fs_root)?; Ok(0) @@ -213,7 +208,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let req = arg_user_mut::(args[1] as usize)?; - run_with_io(&process, |io| { + run_with_io(process, |io| { let file = io.file(fd)?; let node = file.node().ok_or(Error::InvalidFile)?; node.device_request(req)?; @@ -226,7 +221,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let buffer = arg_user_mut::>(args[3] as usize)?; let follow = args[4] != 0; - run_with_io_at(&process, at, |at, mut io| { + run_with_io_at(process, at, |at, mut io| { let node = if path.is_empty() { at.ok_or(Error::InvalidArgument)? } else { @@ -250,7 +245,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; let mode = FileMode::from(args[3] as u32); - run_with_io_at(&process, at, |at, mut io| { + run_with_io_at(process, at, |at, mut io| { io.ioctx().create_directory(at, path, mode)?; Ok(0) }) @@ -258,9 +253,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::Remove => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let recurse = args[3] != 0; - run_with_io_at(&process, at, |at, mut io| { + run_with_io_at(process, at, |at, mut io| { io.ioctx().remove_file(at, path)?; Ok(0) }) @@ -272,7 +266,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; - run_with_io(&process, |mut io| { + run_with_io(process, |mut io| { // let node = io.ioctx().find(None, options.program, true, true)?; // Setup a new process from the file @@ -287,7 +281,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let pid: u32 = child_process.id().into(); // Inherit group and session from the creator - child_process.inherit(&process)?; + child_process.inherit(process)?; // Inherit root from the creator // let child_ioctx = IoContext::new(io.ioctx().root().clone()); @@ -395,10 +389,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result if let Some(ctty) = session_terminal { // Drop all FDs referring to the old session terminal - run_with_io(&process, |mut io| { + run_with_io(process, |mut io| { io.retain(|_, f| { f.node() - .map(|node| !Arc::ptr_eq(&node, &ctty)) + .map(|node| !Arc::ptr_eq(node, &ctty)) .unwrap_or(true) }); }); diff --git a/src/task/mod.rs b/src/task/mod.rs index f3c3c8c5..09a24826 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -3,18 +3,15 @@ #![allow(dead_code)] use abi::error::Error; -use alloc::{string::String, sync::Arc, vec::Vec}; -use kernel_util::sync::{IrqSafeSpinlock, SpinFence}; +use alloc::{string::String, vec::Vec}; +use kernel_util::sync::SpinFence; use crate::{ arch::{Architecture, ArchitectureImpl}, task::{sched::CpuQueue, thread::Thread}, }; -use self::{ - context::{TaskContextImpl, Termination}, - process::Process, -}; +use self::context::{TaskContextImpl, Termination}; pub mod context; pub mod process; @@ -25,48 +22,6 @@ pub mod thread; pub use context::{Cpu, TaskContext}; -// /// Process identifier alias for clarity -// pub type ProcessId = usize; - -// /// Wrapper structure to hold all the system's processes -// pub struct ProcessList { -// data: Vec<(ProcessId, Arc)>, -// last_process_id: ProcessId, -// } -// -// impl ProcessList { -// /// Constructs an empty process list -// pub const fn new() -> Self { -// Self { -// last_process_id: 0, -// data: Vec::new(), -// } -// } -// -// /// Inserts a new process into the list. -// /// -// /// # Safety -// /// -// /// Only meant to be called from inside the Process impl, as this function does not perform any -// /// accounting information updates. -// pub unsafe fn push(&mut self, process: Arc) -> ProcessId { -// self.last_process_id += 1; -// debugln!("Insert process with ID {}", self.last_process_id); -// self.data.push((self.last_process_id, process)); -// self.last_process_id -// } -// -// /// Looks up a process by its ID -// pub fn get(&self, id: ProcessId) -> Option<&Arc> { -// self.data -// .iter() -// .find_map(|(i, p)| if *i == id { Some(p) } else { None }) -// } -// } -// -// /// Global shared process list -// pub static PROCESSES: IrqSafeSpinlock = IrqSafeSpinlock::new(ProcessList::new()); - /// Creates a new kernel-space process to execute a closure and queues it to some CPU pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send + 'static>( name: S, diff --git a/src/task/process.rs b/src/task/process.rs index a6863098..b834cf67 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -4,7 +4,7 @@ use core::{ fmt, mem::size_of, pin::Pin, - sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, + sync::atomic::{AtomicU64, Ordering}, task::{Context, Poll}, }; @@ -12,23 +12,13 @@ use abi::{ error::Error, process::{ExitCode, Signal, ThreadSpawnOptions}, }; -use alloc::{ - collections::{BTreeMap, VecDeque}, - string::String, - sync::Arc, - vec::Vec, -}; +use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; use futures_util::Future; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::sync::IrqSafeSpinlock; use vfs::NodeRef; use crate::{ - mem::{ - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - process::ProcessAddressSpace, - table::MapAttributes, - }, + mem::process::ProcessAddressSpace, proc::{self, io::ProcessIo}, task::context::TaskContextImpl, }; @@ -36,7 +26,7 @@ use crate::{ use super::{ runtime::QueueWaker, sync::UserspaceMutex, - thread::{Thread, ThreadId, ThreadState}, + thread::{Thread, ThreadId}, TaskContext, }; @@ -197,8 +187,7 @@ impl Process { ); let tls_address = if let Some(image) = self.image.as_ref() { - todo!() - // proc::elf::clone_tls(&self.space, image)? + proc::elf::clone_tls(&self.space, image)? } else { 0 }; @@ -378,8 +367,6 @@ impl Process { impl fmt::Display for ProcessId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use fmt::Write; - write!(f, "", self.0) } } diff --git a/src/task/sched.rs b/src/task/sched.rs index 9de56d65..377a7d7f 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -2,20 +2,11 @@ use core::sync::atomic::Ordering; -// use aarch64_cpu::registers::CNTPCT_EL0; -use alloc::{ - collections::{BTreeMap, VecDeque}, - sync::Arc, - vec::Vec, -}; +use alloc::{collections::VecDeque, sync::Arc, vec::Vec}; use cfg_if::cfg_if; -use kernel_util::{ - sync::{IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard}, - util::OneTimeInit, -}; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::{ - // arch::aarch64::{context::TaskContext, cpu::Cpu}, arch::{Architecture, ArchitectureImpl}, task::thread::ThreadState, }; @@ -169,11 +160,7 @@ impl CpuQueue { let mut inner = self.inner.lock(); - // let t = CNTPCT_EL0.get(); - // let delta = t - inner.stats.measure_time; - // inner.stats.measure_time = t; - - let current = inner.current.clone(); + let current = inner.current; let current_t = current.and_then(Thread::get); if let Some(current_t) = current_t.as_ref() { @@ -192,14 +179,10 @@ impl CpuQueue { .is_ok() { inner.queue.push_back(current_t.id()); - // inner.stats.cpu_time += delta; } } - // else - // inner.stats.idle_time += delta; let next_t = inner.next_ready_task(); - // let next_t = next.and_then(Thread::get); inner.current = next_t.as_deref().map(Thread::id); @@ -221,23 +204,6 @@ impl CpuQueue { assert!(ArchitectureImpl::interrupt_mask()); to.switch(from) - - // // log_print_raw!(crate::debug::LogLevel::Info, "{}: ", Cpu::local_id()); - // // if let Some(from) = current.as_ref() { - // // log_print_raw!(crate::debug::LogLevel::Info, "{}", from.id(),); - // // } else { - // // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); - // // } - - // // log_print_raw!(crate::debug::LogLevel::Info, " -> "); - - // // if let Some(to) = next.as_ref() { - // // log_print_raw!(crate::debug::LogLevel::Info, "{}", to.id(),); - // // } else { - // // log_print_raw!(crate::debug::LogLevel::Info, "{{idle}}"); - // // } - - // // log_print_raw!(crate::debug::LogLevel::Info, "\n"); } /// Pushes the process to the back of the execution queue. diff --git a/src/task/thread.rs b/src/task/thread.rs index 84b91cd0..28f3b0aa 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -18,10 +18,7 @@ use alloc::{ }; use atomic_enum::atomic_enum; use futures_util::{task::ArcWake, Future}; -use kernel_util::{ - sync::{IrqGuard, IrqSafeSpinlock}, - util::OneTimeInit, -}; +use kernel_util::sync::{IrqGuard, IrqSafeSpinlock}; use crate::{ block, @@ -179,17 +176,17 @@ impl Thread { .queue() .current_id() .and_then(Self::get) - .map(|t| unsafe { CurrentThread(t, guard) }) + .map(|t| CurrentThread(t, guard)) } - pub fn enqueue_somewhere(&self) { + pub fn enqueue_somewhere(&self) -> usize { // Doesn't have to be precise, so even if something changes, we can still be rebalanced // to another CPU let (index, queue) = CpuQueue::least_loaded().unwrap(); self.enqueue_to(queue); - // index + index } pub fn enqueue_to(&self, queue: &'static CpuQueue) { @@ -264,6 +261,11 @@ impl Thread { } } + /// Marks the process as running and sets its "current" CPU index. + /// + /// # Safety + /// + /// Only meant to be called from within the scheduler. pub unsafe fn set_running(&self, cpu: u32) { self.cpu_id.store(cpu, Ordering::Release); self.state.store(ThreadState::Running, Ordering::Release); @@ -400,9 +402,9 @@ impl CurrentThread { /// This function is only meant to be called right before returning from an userspace /// exception handler. pub unsafe fn handle_pending_signals(&self, frame: &mut F) { - let ThreadId::User(id) = self.id else { + if !self.id.is_user() { return; - }; + } let mut inner = self.inner.lock(); @@ -443,19 +445,6 @@ impl CurrentThread { // Pass the frame pointer as an argument to signal handler entry frame.set_argument(usp as _); } - - // let process = self.process(); - - // if let Some((entry, signal)) = process.pop_signal() { - // debugln!( - // "{} of {}, enter signal handler from pc={:#x}, sp={:#x}", - // self.id, - // process.id(), - // frame.user_ip(), - // frame.user_sp() - // ); - - // } } } @@ -486,12 +475,14 @@ impl ThreadId { &Self::User(id) => id, } } + + pub fn is_user(&self) -> bool { + matches!(self, ThreadId::User(_)) + } } impl fmt::Display for ThreadId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use fmt::Write; - match self { Self::Kernel(id) => write!(f, "#[{id}]"), Self::User(id) => write!(f, "#{id}"), diff --git a/src/util/ring.rs b/src/util/ring.rs index 56e06d2c..3111f514 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -44,13 +44,13 @@ impl RingBuffer { } #[inline] - unsafe fn read_unchecked(&mut self) -> T { + fn read_single(&mut self) -> T { let res = self.data[self.rd]; self.rd = (self.rd + 1) % N; res } - pub unsafe fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize { + pub fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize { let mut pos = (self.rd + pos) % N; let mut off = 0; while off < buffer.len() && self.is_readable_at(pos) { @@ -62,7 +62,7 @@ impl RingBuffer { } #[inline] - pub unsafe fn write_unchecked(&mut self, ch: T) { + pub fn write(&mut self, ch: T) { self.data[self.wr] = ch; self.wr = (self.wr + 1) % N; } @@ -77,11 +77,10 @@ impl AsyncRing { } pub fn try_write(&self, item: T) -> Result<(), Error> { - let mut lock = self.inner.lock(); - unsafe { - lock.write_unchecked(item); + { + let mut lock = self.inner.lock(); + lock.write(item); } - drop(lock); self.read_waker.wake_one(); Ok(()) @@ -102,7 +101,7 @@ impl AsyncRing { let mut inner = self.inner.lock(); if inner.is_readable() { self.read_waker.remove(cx.waker()); - Poll::Ready(unsafe { inner.read_unchecked() }) + Poll::Ready(inner.read_single()) } else { Poll::Pending } From 4168343390f315da26b48b88e87f930c2fe9d308 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 7 Dec 2023 01:04:08 +0200 Subject: [PATCH 102/211] proc: add shebang execution --- src/init.rs | 4 +--- src/main.rs | 4 +++- src/proc/exec.rs | 51 ++++++++++++++++++++++++++++++++++++++-------- src/syscall/mod.rs | 8 ++------ 4 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/init.rs b/src/init.rs index 26e4d011..9579d0ed 100644 --- a/src/init.rs +++ b/src/init.rs @@ -38,7 +38,6 @@ pub fn kinit() -> Result<(), Error> { let root = setup_root()?; let mut ioctx = IoContext::new(root); - let file = ioctx.open_exec(None, "/init")?; let devfs = devfs::root(); @@ -54,9 +53,8 @@ pub fn kinit() -> Result<(), Error> { let stderr = stdout.clone(); { - // XXX let (user_init, user_init_main) = - proc::exec::load_elf("init", file, &["/init", "xxx"], &[])?; + proc::exec::load(&mut ioctx, "/init", &["/init", "xxx"], &[])?; let mut io = user_init.io.lock(); io.set_ioctx(ioctx); diff --git a/src/main.rs b/src/main.rs index bc37fb16..fc52cb9e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,9 @@ allocator_api, trait_alias, strict_provenance, - slice_ptr_get + slice_ptr_get, + slice_split_once, + iter_collect_into )] #![allow( clippy::new_without_default, diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 823d1b28..503cb543 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -3,11 +3,13 @@ use core::{alloc::Layout, ptr::NonNull}; use abi::{ error::Error, + io::SeekFrom, pass::{Place, Placer}, + path::Path, process::ProgramArgumentInner, }; -use alloc::{string::String, sync::Arc}; -use vfs::FileRef; +use alloc::{string::String, sync::Arc, vec::Vec}; +use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ mem::{ @@ -159,18 +161,49 @@ fn setup_binary>( context, Some(image), )) - // Ok(Process::new_with_context(name, Some(space), context)) } -/// Loads an ELF bianary from `file` and sets up all the necessary data/argument memory -pub fn load_elf>( - name: S, +fn load_binary( + head: &[u8], file: FileRef, + space: &ProcessAddressSpace, +) -> Result { + if head.starts_with(b"\x7FELF") { + proc::elf::load_elf_from_file(space, file) + } else { + Err(Error::UnrecognizedExecutable) + } +} + +pub fn load>( + ioctx: &mut IoContext, + path: P, args: &[&str], envs: &[&str], ) -> Result<(Arc, Arc), Error> { - let space = ProcessAddressSpace::new()?; - let image = proc::elf::load_elf_from_file(&space, file)?; + let mut head = [0; 256]; + let path = path.as_ref(); + let file = ioctx.open_exec(None, path)?; - setup_binary(name, space, image, args, envs) + file.seek(SeekFrom::Start(0))?; + let count = file.read(&mut head)?; + let head = &head[..count]; + + if let Some(shebang) = head.strip_prefix(b"#!") && let Some((shebang, _)) = shebang.split_once(|&ch| ch == b'\n') { + let shebang = core::str::from_utf8(shebang).map_err(|_| Error::InvalidFile)?; + let mut shebang_args = shebang.split(' ').collect::>(); + if shebang_args.is_empty() || shebang_args.len() >= 8 { + return Err(Error::UnrecognizedExecutable); + } + shebang_args.extend_from_slice(args); + + return load(ioctx, shebang_args[0], &shebang_args, envs); + } + + file.seek(SeekFrom::Start(0))?; + + let space = ProcessAddressSpace::new()?; + let image = load_binary(head, file, &space)?; + + setup_binary(path.display(), space, image, args, envs) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 797aa62e..7d8b459d 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -267,14 +267,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let options = arg_user_ref::(args[0] as usize)?; run_with_io(process, |mut io| { - // let node = io.ioctx().find(None, options.program, true, true)?; - // Setup a new process from the file - let file = io.ioctx().open_exec(None, options.program)?; - // let file = node.open(OpenOptions::READ)?; - let (child_process, child_main) = proc::exec::load_elf( + let (child_process, child_main) = proc::exec::load( + io.ioctx(), options.program, - file, options.arguments, options.environment, )?; From 57d255d466151669301627b23798b0b8e3372ae1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 7 Dec 2023 10:16:44 +0200 Subject: [PATCH 103/211] refactor: fix x86-64 warnings --- src/arch/x86_64/boot/mod.rs | 2 +- src/arch/x86_64/context.rs | 2 +- src/arch/x86_64/exception.rs | 2 +- src/arch/x86_64/mem/mod.rs | 8 ++++---- src/arch/x86_64/mem/process.rs | 2 +- src/arch/x86_64/mem/table.rs | 14 +++++++++++++- src/arch/x86_64/mod.rs | 2 +- src/arch/x86_64/syscall.rs | 2 +- src/device/bus/pci/mod.rs | 18 +++++++----------- src/device/bus/pci/space/mod.rs | 4 ++-- src/main.rs | 1 + 11 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 679dafdc..c15f29eb 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -63,7 +63,7 @@ static YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 { unsafe fn init_dummy_cpu() { // TODO this is incorrect static UNINIT_CPU_INNER: usize = 0; - static UNINIT_CPU_PTR: &'static usize = &UNINIT_CPU_INNER; + static UNINIT_CPU_PTR: &usize = &UNINIT_CPU_INNER; // Point %gs to a dummy structure so that Cpu::get_local() works properly even before the CPU // data structure is initialized diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 430e9337..f3d79f86 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -7,7 +7,7 @@ use crate::{ arch::x86_64::mem::KERNEL_TABLES, mem::{ address::{AsPhysicalAddress, IntoRaw}, - phys, PhysicalAddress, + phys, }, task::context::TaskContextImpl, }; diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 465de90e..9cb92d8b 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -5,7 +5,7 @@ use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ arch::x86_64::apic, - task::{context::TaskFrame, process::Process, thread::Thread, Cpu}, + task::{context::TaskFrame, thread::Thread, Cpu}, }; use super::ARCHITECTURE; diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index ebdc8f97..d7785ed8 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -98,7 +98,7 @@ unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result= EARLY_MAPPING_OFFSET + L2::SIZE { + if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { panic!("Tried to unmap invalid early mapping: {:#x}", address); } @@ -242,7 +242,7 @@ pub struct EarlyMapping<'a, T: ?Sized> { } impl<'a, T: Sized> EarlyMapping<'a, T> { - pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { + pub(super) unsafe fn map(physical: PhysicalAddress) -> Result, Error> { let layout = Layout::new::(); let aligned = physical.page_align_down::(); let offset = physical.page_offset::(); @@ -254,7 +254,7 @@ impl<'a, T: Sized> EarlyMapping<'a, T> { Ok(EarlyMapping { value, page_count }) } - pub unsafe fn map_slice( + pub(super) unsafe fn map_slice( physical: PhysicalAddress, len: usize, ) -> Result, Error> { @@ -317,7 +317,7 @@ fn clone_kernel_tables(dst: &mut PageTable) { /// * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000 : DEVICE_MAPPING_L2 /// * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000 : DEVICE_MAPPING_L3S /// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ... -pub unsafe fn init_fixed_tables() { +pub(super) unsafe fn init_fixed_tables() { // TODO this could be built in compile-time too? let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index 48963678..bf86dad2 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -37,7 +37,7 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { l0[i] = PageEntry::INVALID; } - clone_kernel_tables(&mut *l0); + clone_kernel_tables(&mut l0); Ok(Self { l0 }) } diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index c4adfa93..2e3ec211 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -113,7 +113,7 @@ impl PageEntry { } impl PageEntry { - pub unsafe fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + pub fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { Self( u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), PhantomData, @@ -156,10 +156,16 @@ impl PageEntry { /// An entry that is not mapped pub const INVALID: Self = Self(0, PhantomData); + /// Reinterprets raw [u64] as a [PageEntry]. + /// + /// # Safety + /// + /// Unsafe: the caller must ensure the value is a valid page translation entry. pub const unsafe fn from_raw(raw: u64) -> Self { Self(raw, PhantomData) } + /// Returns the translation attributes of the entry pub fn attributes(&self) -> PageAttributes { PageAttributes::from_bits_retain(self.0) } @@ -178,6 +184,12 @@ impl PageTable { } } + /// Reinterprets given [PageEntry] slice as a reference to [PageTable]. + /// + /// # Safety + /// + /// Unsafe: the caller must ensure the provided reference is properly aligned and contains sane + /// data. pub unsafe fn from_raw_slice_mut(data: &mut [PageEntry; 512]) -> &mut Self { core::mem::transmute(data) } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 2f1e55f0..5d1ebe90 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -284,7 +284,7 @@ impl Architecture for X86_64 { impl X86_64 { unsafe fn handle_ipi(&self, _msg: CpuMessage) { warnln!("Received an IPI"); - loop {} + todo!(); } fn set_boot_data(&self, data: BootData) { diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 6f46d7ab..fc46a106 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -8,7 +8,7 @@ use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ arch::x86_64::registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, syscall::raw_syscall_handler, - task::{context::TaskFrame, process::Process, thread::Thread}, + task::{context::TaskFrame, thread::Thread}, }; /// Set of registers saved when taking a syscall instruction diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index a2fbd400..a828ff53 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -99,11 +99,7 @@ impl PciBusSegment { } } - fn enumerate_function( - &mut self, - parent: &mut PciBusManager, - address: PciAddress, - ) -> Result<(), Error> { + fn enumerate_function(&mut self, address: PciAddress) -> Result<(), Error> { let Some(config) = self.probe_config_space(address)? else { return Ok(()); }; @@ -113,7 +109,7 @@ impl PciBusSegment { // Enumerate multi-function devices if address.function == 0 && header_type & 0x80 != 0 { for function in 1..8 { - self.enumerate_function(parent, address.with_function(function))?; + self.enumerate_function(address.with_function(function))?; } } @@ -132,22 +128,22 @@ impl PciBusSegment { Ok(()) } - fn enumerate_bus(&mut self, parent: &mut PciBusManager, bus: u8) -> Result<(), Error> { + fn enumerate_bus(&mut self, bus: u8) -> Result<(), Error> { let address = PciAddress::for_bus(self.segment_number, bus); for i in 0..32 { let device_address = address.with_device(i); - self.enumerate_function(parent, device_address)?; + self.enumerate_function(device_address)?; } Ok(()) } /// Enumerates the bus segment, placing found devices into the manager - pub fn enumerate(&mut self, parent: &mut PciBusManager) -> Result<(), Error> { + pub fn enumerate(&mut self) -> Result<(), Error> { for bus in self.bus_number_start..self.bus_number_end { - self.enumerate_bus(parent, bus)?; + self.enumerate_bus(bus)?; } Ok(()) } @@ -199,7 +195,7 @@ impl PciBusManager { }; let mut this = PCI_MANAGER.lock(); - bus_segment.enumerate(&mut *this)?; + bus_segment.enumerate()?; this.segments.push(bus_segment); Ok(()) diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs index e0a6d43d..acf80491 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/src/device/bus/pci/space/mod.rs @@ -160,7 +160,7 @@ pub trait PciConfigurationSpace { if index % 2 == 0 { let w0 = self.read_u32(0x10 + index * 4); - match w0 & 0 { + match w0 & 1 { 0 => match (w0 >> 1) & 3 { 0 => { // 32-bit memory BAR @@ -187,7 +187,7 @@ pub trait PciConfigurationSpace { let w0 = self.read_u32(0x10 + index * 4); - match w0 & 0 { + match w0 & 1 { 0 => match (w0 >> 1) & 3 { 0 => { // 32-bit memory BAR diff --git a/src/main.rs b/src/main.rs index fc52cb9e..bc0bef63 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,7 @@ clippy::new_without_default, clippy::fn_to_numeric_cast, clippy::match_ref_pats, + clippy::match_single_binding, async_fn_in_trait )] // #![warn(missing_docs)] From 2b70a3588209ce4ddc9ce42c5cce77b8ea03ea3b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 7 Dec 2023 12:50:54 +0200 Subject: [PATCH 104/211] refactor: add docs --- src/arch/aarch64/boot/mod.rs | 1 + src/arch/aarch64/mem/mod.rs | 16 +- src/arch/aarch64/mem/process.rs | 2 + src/arch/aarch64/mod.rs | 6 + src/arch/mod.rs | 4 + src/arch/x86_64/mem/mod.rs | 4 +- src/arch/x86_64/mem/process.rs | 1 + src/arch/x86_64/mem/table.rs | 3 + src/arch/x86_64/mod.rs | 7 +- src/debug.rs | 5 +- src/device/devtree.rs | 2 + src/device/display/font.rs | 9 +- src/device/timer.rs | 3 + src/device/tty.rs | 54 +------ src/fs/devfs.rs | 2 + src/fs/sysfs.rs | 4 + src/main.rs | 3 +- src/mem/address.rs | 20 +++ src/mem/device.rs | 14 +- src/mem/mod.rs | 1 + src/mem/phys/mod.rs | 5 + src/mem/pointer.rs | 6 + src/mem/process.rs | 18 +++ src/mem/table.rs | 8 +- src/proc/elf.rs | 1 + src/proc/exec.rs | 3 +- src/proc/io.rs | 7 + src/proc/random.rs | 4 + src/task/context.rs | 2 + src/task/process.rs | 267 ++++++-------------------------- src/task/runtime/executor.rs | 4 + src/task/runtime/macros.rs | 2 + src/task/runtime/mod.rs | 8 +- src/task/runtime/task_queue.rs | 1 + src/task/runtime/waker.rs | 7 + src/task/sched.rs | 49 +----- src/task/sync.rs | 7 + src/task/thread.rs | 66 +++++++- src/util/mod.rs | 12 +- src/util/queue.rs | 6 + src/util/ring.rs | 14 +- 41 files changed, 302 insertions(+), 356 deletions(-) diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index f89317c5..65adfc33 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -1,3 +1,4 @@ +//! AArch64 boot and entry implementation use core::{arch::global_asm, sync::atomic::Ordering}; use aarch64_cpu::{ diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index 38af3515..e607f048 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -1,3 +1,4 @@ +//! AArch64-specific memory management interfaces and functions use core::{ alloc::Layout, ops::{Deref, DerefMut}, @@ -73,27 +74,16 @@ pub(super) const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); #[link_section = ".data.tables"] -pub static mut KERNEL_TABLES: KernelImageObject = +pub(super) static mut KERNEL_TABLES: KernelImageObject = unsafe { KernelImageObject::new(FixedTables::zeroed()) }; +/// Memory mapping which may be used for performing early kernel initialization pub struct EarlyMapping<'a, T: ?Sized> { value: &'a mut T, page_count: usize, } impl<'a, T: Sized> EarlyMapping<'a, T> { - // pub(super) unsafe fn map(physical: PhysicalAddress) -> Result, Error> { - // let layout = Layout::new::(); - // let aligned = physical.page_align_down::(); - // let offset = physical.page_offset::(); - // let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; - - // let virt = map_early_pages(aligned, page_count)?; - // let value = &mut *((virt + offset) as *mut T); - - // Ok(EarlyMapping { value, page_count }) - // } - pub(super) unsafe fn map_slice( physical: PhysicalAddress, len: usize, diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index a5ad6ab0..51f740c6 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -1,3 +1,4 @@ +//! AArch64-specific process address space management use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; @@ -13,6 +14,7 @@ use crate::mem::{ use super::table::{PageEntry, PageTable, L1, L2, L3}; +/// AArch64 implementation of a process address space table #[repr(C)] pub struct ProcessAddressSpaceImpl { l1: PhysicalRefMut<'static, PageTable>, diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 1163da49..5f3b3ddf 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -59,9 +59,11 @@ struct BootStack { data: [u8; BOOT_STACK_SIZE], } +/// AArch64 architecture implementation pub struct AArch64 { dt: OneTimeInit>, + /// Optional instance of PSCI on this platform pub psci: OneTimeInit<&'static Psci>, reset: OneTimeInit<&'static dyn ResetDevice>, @@ -417,6 +419,7 @@ impl AArch64 { } } +/// AArch64 implementation value pub static ARCHITECTURE: AArch64 = AArch64 { dt: OneTimeInit::new(), initrd: OneTimeInit::new(), @@ -430,8 +433,11 @@ pub static ARCHITECTURE: AArch64 = AArch64 { mtimer: OneTimeInit::new(), }; +/// AArch64-specific interrupt number #[derive(Clone, Copy, Debug, PartialEq)] pub enum IrqNumber { + /// CPU-private interrupt Private(u32), + /// Shared interrupt Shared(u32), } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 2584b1a8..fe5241e6 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -107,6 +107,8 @@ pub trait Architecture { /// The caller must ensure the mapping is and will no longer be used. unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping); + /// Maps the physical memory regions into the kernel space so they can later be accessed by the + /// kernel fn map_physical_memory + Clone>( &self, it: I, @@ -114,8 +116,10 @@ pub trait Architecture { memory_end: PhysicalAddress, ) -> Result<(), Error>; + /// Converts a physical address to a virtual one, so it can be accessed by the kernel fn virtualize(address: PhysicalAddress) -> Result; + /// Converts a virtual address created by [Architecture::virtualize] back to its physical form fn physicalize(address: usize) -> Result; // Architecture intrinsics diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index d7785ed8..6e20cc65 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -1,3 +1,4 @@ +//! x86-64-specific memory management functions and interfaces use core::{ alloc::Layout, ops::{Deref, DerefMut}, @@ -46,7 +47,7 @@ const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1; const DEVICE_MAPPING_L3_COUNT: usize = 4; #[link_section = ".data.tables"] -pub static mut KERNEL_TABLES: KernelImageObject = +pub(super) static mut KERNEL_TABLES: KernelImageObject = unsafe { KernelImageObject::new(FixedTables::zeroed()) }; // 2MiB for early mappings @@ -236,6 +237,7 @@ pub(super) unsafe fn map_heap_block(index: usize, page: PhysicalAddress) { HEAP_MAPPING_L2[index] = PageEntry::::block(page, PageAttributes::WRITABLE); } +/// Memory mapping which may be used for performing early kernel initialization pub struct EarlyMapping<'a, T: ?Sized> { value: &'a mut T, page_count: usize, diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index bf86dad2..b3c74439 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,3 +1,4 @@ +//! x86-64-specific process address space management functions use yggdrasil_abi::error::Error; use crate::{ diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index 2e3ec211..3ff305b5 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -1,3 +1,4 @@ +//! x86-64-specific memory translation table management interfaces and functions use core::{ marker::PhantomData, ops::{Index, IndexMut}, @@ -113,6 +114,7 @@ impl PageEntry { } impl PageEntry { + /// Constructs a mapping which points to a 1GiB block pub fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { Self( u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(), @@ -147,6 +149,7 @@ impl PageEntry { } } + /// Returns `true` if the mapping represents a "page"/"block" and not a table pub fn is_block(self) -> bool { self.0 & PageAttributes::BLOCK.bits() != 0 } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5d1ebe90..14b58c2e 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,4 +1,4 @@ -// TODO fix all TODOs +//! x86-64 architecture implementation use core::{mem::size_of, ops::DerefMut, sync::atomic::Ordering}; use abi::error::Error; @@ -74,12 +74,16 @@ use self::{ use super::{Architecture, CpuMessage}; +/// x86-64-specific interrupt number #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum IrqNumber { + /// Legacy (ISA) interrupt Isa(u8), + /// Global System Interrupt Gsi(u8), } +/// x86-64 architecture implementation pub struct X86_64 { boot_data: OneTimeInit, acpi: OneTimeInit>, @@ -95,6 +99,7 @@ pub struct X86_64 { static SHUTDOWN_FENCE: SpinFence = SpinFence::new(); +/// Global x86-64 architecture value pub static ARCHITECTURE: X86_64 = X86_64 { boot_data: OneTimeInit::new(), acpi: OneTimeInit::new(), diff --git a/src/debug.rs b/src/debug.rs index 31f4976d..4b6c7cf7 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -15,6 +15,7 @@ struct RingLoggerInner { data: RingBuffer, } +/// Logger sink which collects output to an internal ring buffer pub struct RingLoggerSink { inner: IrqSafeSpinlock, } @@ -161,7 +162,7 @@ impl fmt::Write for DebugSinkWrapper { } impl RingLoggerSink { - pub const fn new() -> Self { + const fn new() -> Self { Self { inner: IrqSafeSpinlock::new(RingLoggerInner { data: RingBuffer::new(0), @@ -169,6 +170,7 @@ impl RingLoggerSink { } } + /// Reads data from the sink without blocking and waiting for more to arrive pub fn read(&self, pos: usize, buffer: &mut [u8]) -> usize { self.inner.lock().data.read_all_static(pos, buffer) } @@ -193,6 +195,7 @@ static LOGGER: SimpleLogger = SimpleLogger; static DEBUG_SINKS: IrqSafeSpinlock> = IrqSafeSpinlock::new(StaticVector::new()); +/// See [RingLoggerSink] pub static RING_LOGGER_SINK: RingLoggerSink = RingLoggerSink::new(); /// Prints a hex-dump of a slice, appending a virtual address offset to the output diff --git a/src/device/devtree.rs b/src/device/devtree.rs index d85e7d3a..3009ceaa 100644 --- a/src/device/devtree.rs +++ b/src/device/devtree.rs @@ -204,6 +204,8 @@ impl<'a> DeviceTree<'a> { } // Commonly used functions for convenience + + /// Returns the /chosen.stdout-path value pub fn chosen_stdout_path(&self) -> Option<&str> { let chosen = self.node_by_path("/chosen")?; chosen.prop("stdout-path") diff --git a/src/device/display/font.rs b/src/device/display/font.rs index 1ac9412c..76450009 100644 --- a/src/device/display/font.rs +++ b/src/device/display/font.rs @@ -1,10 +1,11 @@ +//! Font management and data structures + use core::mem::size_of; use abi::error::Error; use bytemuck::{Pod, Zeroable}; use kernel_util::AlignedTo; -// static CONSOLE_FONT: &[u8] = include_bytes!("font.psfu"); static CONSOLE_FONT: &AlignedTo = &AlignedTo { align: [], bytes: *include_bytes!("font.psfu"), @@ -23,6 +24,7 @@ struct PsfHeader { width: u32, } +/// Represents a PSF-format font object #[derive(Clone, Copy)] pub struct PcScreenFont<'a> { header: &'a PsfHeader, @@ -36,6 +38,7 @@ impl Default for PcScreenFont<'static> { } impl<'a> PcScreenFont<'a> { + /// Constructs an instance of [PcScreenFont] from its byte representation pub fn from_bytes(bytes: &'a [u8]) -> Result { let header: &PsfHeader = bytemuck::from_bytes(&bytes[..size_of::()]); let glyph_data = &bytes[header.header_size as usize..]; @@ -43,22 +46,26 @@ impl<'a> PcScreenFont<'a> { Ok(Self { header, glyph_data }) } + /// Returns the character width of the font #[inline] pub const fn width(&self) -> u32 { self.header.width } + /// Returns the character height of the font #[inline] pub const fn height(&self) -> u32 { self.header.height } + /// Returns the count of glyphs present in the font #[allow(clippy::len_without_is_empty)] #[inline] pub const fn len(&self) -> u32 { self.header.num_glyph } + /// Returns the data slice of a single glyph within the font #[inline] pub fn raw_glyph_data(&self, index: u32) -> &[u8] { &self.glyph_data[(index * self.header.bytes_per_glyph) as usize..] diff --git a/src/device/timer.rs b/src/device/timer.rs index bbf89fab..a24002bc 100644 --- a/src/device/timer.rs +++ b/src/device/timer.rs @@ -1,7 +1,10 @@ +//! Timer device utilities + use core::time::Duration; use crate::task::runtime; +/// Global system timer tick handler pub fn tick(now: Duration) { runtime::tick(now); } diff --git a/src/device/tty.rs b/src/device/tty.rs index d4b3fa7f..826d942f 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -118,9 +118,9 @@ struct TtyContextInner { process_group: Option, } +/// Represents the context of a terminal device pub struct TtyContext { ring: AsyncRing, - // input_queue: AsyncQueue, inner: IrqSafeSpinlock, } @@ -280,6 +280,7 @@ pub trait TtyDevice: SerialDevice { } impl TtyContext { + /// Constructs a new [TtyContext] pub fn new() -> Self { Self { ring: AsyncRing::new(0), @@ -290,67 +291,24 @@ impl TtyContext { } } + /// Writes a single character to the terminal pub fn putc(&self, ch: u8) { self.ring.try_write(ch).unwrap(); } + /// Performs a blocking read of a single character from the terminal pub async fn getc(&self) -> u8 { self.ring.read().await } + /// Changes the configuration of the terminal pub fn set_config(&self, config: &TerminalOptions) -> Result<(), Error> { self.inner.lock().config = *config; Ok(()) } + /// Returns the configuration of the terminal pub fn config(&self) -> TerminalOptions { self.inner.lock().config } } - -// impl CharRingInner { -// } -// -// impl CharRing { -// /// Constructs an empty ring buffer -// pub const fn new() -> Self { -// Self { -// inner: IrqSafeSpinlock::new(CharRingInner { -// rd: 0, -// wr: 0, -// data: [0; N], -// flags: 0, -// process_group: None, -// }), -// // wait_read: Wait::new("char_ring_read"), -// // wait_write: Wait::new("char_ring_write"), -// config: IrqSafeSpinlock::new(TerminalOptions::const_default()), -// } -// } -// -// /// Reads a single character from the buffer, blocking until available -// pub fn getc(&'static self) -> Result { -// todo!() -// // let mut lock = self.inner.lock(); -// // loop { -// // if !lock.is_readable() && lock.flags == 0 { -// // drop(lock); -// // self.wait_read.wait(None)?; -// // lock = self.inner.lock(); -// // } else { -// // break; -// // } -// // } -// -// // let byte = unsafe { lock.read_unchecked() }; -// // drop(lock); -// // self.wait_write.wakeup_one(); -// // // TODO WAIT_SELECT -// // Ok(byte) -// } -// -// /// Sends a single character to the buffer -// pub fn putc(&self, ch: u8, blocking: bool) -> Result<(), Error> { -// todo!() -// } -// } diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index 63afcc59..c43a7869 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -21,6 +21,7 @@ pub enum CharDeviceType { } static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); + /// Sets up the device filesystem pub fn init() { let root = MemoryDirectory::empty(); @@ -60,6 +61,7 @@ pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Re _add_char_device(dev, name) } +/// Adds "pseudo"-devices to the filesystem (i.e. /dev/random) pub fn add_pseudo_devices() -> Result<(), Error> { let random = read_fn_node(move |_, buf| { random::read(buf); diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 4f0ff5f5..367d73c8 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -1,3 +1,5 @@ +//! "System" filesystem implementation + use abi::error::Error; use git_version::git_version; use kernel_util::util::OneTimeInit; @@ -10,6 +12,7 @@ use crate::{debug, util}; static ROOT: OneTimeInit = OneTimeInit::new(); +/// Returns the root of the filesystem pub fn root() -> &'static NodeRef { ROOT.get() } @@ -18,6 +21,7 @@ fn read_kernel_log(pos: u64, buffer: &mut [u8]) -> Result { Ok(debug::RING_LOGGER_SINK.read(pos as usize, buffer)) } +/// Sets up the entries within the filesystem pub fn init() { let d_kernel = mdir([ ("version", const_value_node(env!("CARGO_PKG_VERSION"))), diff --git a/src/main.rs b/src/main.rs index bc0bef63..4fdc7cb7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,8 +28,7 @@ clippy::match_single_binding, async_fn_in_trait )] -// #![warn(missing_docs)] -#![allow(missing_docs)] +#![deny(missing_docs)] #![no_std] #![no_main] diff --git a/src/mem/address.rs b/src/mem/address.rs index c03b834a..d4d32cda 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -1,3 +1,5 @@ +//! Address manipulation interfaces and utilities + use core::{ fmt, iter::Step, @@ -9,25 +11,32 @@ use crate::arch::{Architecture, ArchitectureImpl}; use super::{table::EntryLevel, KERNEL_VIRT_OFFSET}; +/// Wrapper type to represent an object residing within the kernel #[repr(transparent)] pub struct KernelImageObject { inner: T, } +/// Wrapper type to represent a physical memory address #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] #[repr(transparent)] pub struct PhysicalAddress(u64); +/// Interface for converting addresses from their raw values to more specific types #[const_trait] pub trait FromRaw { + /// Converts a raw value into the address wrapper type fn from_raw(value: T) -> Self; } +/// Interface for converting wrapper types into their raw address representations #[const_trait] pub trait IntoRaw { + /// Converts a wrapper type value into its raw address fn into_raw(self) -> T; } +/// Interface for obtaining physical addresses of values pub trait AsPhysicalAddress { /// Returns the value's physical address. /// @@ -73,36 +82,46 @@ impl DerefMut for KernelImageObject { // impl PhysicalAddress { + /// Physical address of zero pub const ZERO: Self = Self(0); + /// Maximum representable physical address pub const MAX: Self = Self(u64::MAX); + /// Minumum representable physical address pub const MIN: Self = Self(u64::MIN); + /// Applies an offset to the address pub const fn add(self, offset: usize) -> Self { Self(self.0 + offset as u64) } + /// Returns `true` if the address is zero #[inline(always)] pub const fn is_zero(self) -> bool { self.0 == 0 } + /// Returns the offset this address has within a page of level `L` pub const fn page_offset(self) -> usize { L::page_offset(self.0 as usize) } + /// Aligns the address down to a boundary of a page of level `L` pub const fn page_align_down(self) -> Self { Self(self.0 & !(L::SIZE as u64 - 1)) } + /// Aligns the address up to a boundary of a page of level `L` pub const fn page_align_up(self) -> Self { Self((self.0 + L::SIZE as u64 - 1) & !(L::SIZE as u64 - 1)) } + /// Returns the page index this address has at level `L` pub const fn page_index(self) -> usize { L::index(self.0 as usize) } + /// Returns `true` if the address is aligned to a boundary of a page at level `L` #[inline] pub const fn is_aligned_for(self) -> bool { self.0 as usize % align_of::() == 0 @@ -119,6 +138,7 @@ impl PhysicalAddress { ArchitectureImpl::physicalize(address).unwrap() } + /// Converts the physical address to a virtual one pub fn virtualize_raw(self) -> usize { ArchitectureImpl::virtualize(self).unwrap() } diff --git a/src/mem/device.rs b/src/mem/device.rs index a0e2f46b..7a190134 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -1,6 +1,4 @@ //! Facilities for mapping devices to virtual address space -// TODO -#![allow(unused)] use core::{mem::size_of, ops::Deref}; use abi::error::Error; @@ -10,22 +8,31 @@ use crate::arch::{Architecture, ARCHITECTURE}; use super::PhysicalAddress; +/// Describes a single device memory mapping #[derive(Debug)] pub struct RawDeviceMemoryMapping { + /// Virtual address of the mapped object pub address: usize, + /// Base address of the mapping start pub base_address: usize, + /// Page size used for the mapping pub page_size: usize, + /// Number of pages used to map the object pub page_count: usize, } +/// Describes a single untyped device memory mapping #[derive(Clone, Debug)] pub struct DeviceMemoryMapping { + #[allow(unused)] inner: Arc, address: usize, } +/// Describes a single typed device memory mapping #[derive(Clone, Debug)] pub struct DeviceMemoryIo<'a, T: ?Sized> { + #[allow(unused)] inner: Arc, value: &'a T, } @@ -41,6 +48,8 @@ impl RawDeviceMemoryMapping { ARCHITECTURE.map_device_memory(base, size) } + /// Consumes the device mapping, leaking its address without deallocating the translation + /// mapping itself pub fn leak(self) -> usize { let address = self.address; core::mem::forget(self); @@ -73,6 +82,7 @@ impl DeviceMemoryMapping { }) } + /// Returns the address to which the object is mapped pub fn address(&self) -> usize { self.address } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 28950dc2..3ee99e87 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -22,6 +22,7 @@ pub use address::PhysicalAddress; use self::{device::DeviceMemoryMapping, process::ProcessAddressSpace}; +/// Offset applied to the physical kernel image when translating it into the virtual address space pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; /// Reads a value from an arbitrary physical address. diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index b645fe58..3a0db3f8 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,3 +1,5 @@ +//! Physical memory management utilities + use core::ops::Range; use abi::error::Error; @@ -41,6 +43,7 @@ impl PhysicalMemoryRegion { self.base..self.end() } + /// Constrains the [PhysicalMemoryRegion] to global memory limits set in the kernel pub fn clamp(self) -> Option<(PhysicalAddress, PhysicalAddress)> { let start = self.base.min(MEMORY_UPPER_LIMIT); let end = self.end().min(MEMORY_UPPER_LIMIT); @@ -67,6 +70,7 @@ pub fn alloc_pages_contiguous(count: usize) -> Result { PHYSICAL_MEMORY.get().lock().alloc_contiguous_pages(count) } +/// Allocates a single 2MiB page of physical memory from the global manager pub fn alloc_2m_page() -> Result { PHYSICAL_MEMORY.get().lock().alloc_2m_page() } @@ -102,6 +106,7 @@ fn physical_memory_range>( } } +/// Locates a contiguous region of available physical memory within the memory region list pub fn find_contiguous_region>( it: I, count: usize, diff --git a/src/mem/pointer.rs b/src/mem/pointer.rs index c328d2db..4ab32f3a 100644 --- a/src/mem/pointer.rs +++ b/src/mem/pointer.rs @@ -1,3 +1,5 @@ +//! Pointer utilities and interfaces + use core::{ fmt, ops::{Deref, DerefMut}, @@ -5,11 +7,13 @@ use core::{ use super::{address::AsPhysicalAddress, PhysicalAddress}; +/// Wrapper for immutably accessing a value at a physical address #[repr(transparent)] pub struct PhysicalRef<'a, T: ?Sized> { value: &'a T, } +/// Wrapper for mutably accessing a value at a physical address #[repr(transparent)] pub struct PhysicalRefMut<'a, T: ?Sized> { value: &'a mut T, @@ -44,6 +48,7 @@ impl<'a, T: Sized> PhysicalRefMut<'a, T> { } impl PhysicalRefMut<'_, T> { + /// Returns the "address" part of the reference #[inline] pub fn as_address(&self) -> usize { (self.value as *const T).addr() @@ -104,6 +109,7 @@ impl<'a, T: Sized> PhysicalRef<'a, T> { } impl PhysicalRef<'_, T> { + /// Returns the "address" part of the reference #[inline] pub fn as_address(&self) -> usize { (self.value as *const T).addr() diff --git a/src/mem/process.rs b/src/mem/process.rs index ef5e23dd..cdd3d66b 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,3 +1,5 @@ +//! Process address space structures and management functions + use abi::error::Error; use cfg_if::cfg_if; use kernel_util::sync::IrqSafeSpinlock; @@ -17,10 +19,14 @@ cfg_if! { /// Interface for virtual memory address space management pub trait ProcessAddressSpaceManager: Sized { + /// Page size used by this implementation const PAGE_SIZE: usize; + /// PFN of a minimum address allowed for virtual region allocation const LOWER_LIMIT_PFN: usize; + /// PFN of a maximum address allowed for virtual region allocation const UPPER_LIMIT_PFN: usize; + /// Constructs a new implementation-specific per-process address space fn new() -> Result; /// Places a single PAGE_SIZE mapping into the address space. @@ -43,8 +49,11 @@ pub trait ProcessAddressSpaceManager: Sized { /// will not access this page. unsafe fn unmap_page(&mut self, address: usize) -> Result; + /// Returns the [PhysicalAddress] and [MapAttributes] associated with given virtual `address`, + /// if one is mapped fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; + /// Returns the implementation specific physical address of this space, with ASID applied fn as_address_with_asid(&self) -> u64; } @@ -53,6 +62,7 @@ struct Inner { table: ProcessAddressSpaceImpl, } +/// Data structure for managing the address translation and allocation for a single process pub struct ProcessAddressSpace { inner: IrqSafeSpinlock, } @@ -144,6 +154,7 @@ impl Inner { } impl ProcessAddressSpace { + /// Constructs a new [ProcessAddressSpace] pub fn new() -> Result { let table = ProcessAddressSpaceImpl::new()?; let allocator = VirtualMemoryAllocator::new( @@ -155,6 +166,8 @@ impl ProcessAddressSpace { }) } + /// Allocates a region of virtual memory within the address space and maps the pages to the + /// ones returned from `get_page` function pub fn allocate Result>( &self, _hint: Option, @@ -173,6 +186,7 @@ impl ProcessAddressSpace { ) } + /// Maps a region of memory in the address space pub fn map Result>( &self, address: usize, @@ -193,6 +207,7 @@ impl ProcessAddressSpace { ) } + /// Adds a single-page mapping to the address space pub fn map_single( &self, address: usize, @@ -208,6 +223,8 @@ impl ProcessAddressSpace { .map_range(address, 1, |_| Ok(physical), attributes) } + /// Returns the [PhysicalAddress] associated with given virtual `address`, + /// if one is mapped pub fn translate(&self, address: usize) -> Result { // Offset is handled at impl level self.inner.lock().table.translate(address).map(|e| e.0) @@ -234,6 +251,7 @@ impl ProcessAddressSpace { ) } + /// Returns the physical address of this table, with ASID applied pub fn as_address_with_asid(&self) -> u64 { self.inner.lock().table.as_address_with_asid() } diff --git a/src/mem/table.rs b/src/mem/table.rs index 057946ad..20b857b4 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -22,7 +22,9 @@ bitflags! { pub trait NextPageTable { /// Type for the next-level page table type NextLevel; + /// Type for an immutable reference to the next-level page table type TableRef: Deref; + /// Type for a mutable reference to the next-level page table type TableRefMut: DerefMut; /// Tries looking up a next-level table at given index, allocating and mapping one if it is not @@ -30,14 +32,16 @@ pub trait NextPageTable { fn get_mut_or_alloc(&mut self, index: usize) -> Result; /// Returns a mutable reference to a next-level table at `index`, if present fn get_mut(&mut self, index: usize) -> Option; - + /// Returns an immutable reference to a next-level table at `index`, if present fn get(&self, index: usize) -> Option; } /// Interface for a single level of address translation #[const_trait] pub trait EntryLevel: Copy { + /// The right shift needed to obtain an index of an entry at this level from an address const SHIFT: usize; + /// The size of a page at this entry level const SIZE: usize = 1 << Self::SHIFT; /// Returns the index into a page table for a given address @@ -51,11 +55,13 @@ pub trait EntryLevel: Copy { addr & (Self::SIZE - 1) } + /// Aligns the `addr` up to the level page boundary #[inline] fn align_up(addr: usize) -> usize { (addr + Self::SIZE - 1) & !(Self::SIZE - 1) } + /// Returns the page count needed to fully contain a block of size `addr` #[inline] fn page_count(addr: usize) -> usize { (addr + Self::SIZE - 1) / Self::SIZE diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 55959dc1..633b8cef 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -59,6 +59,7 @@ fn from_parse_error(v: ParseError) -> Error { Error::InvalidFile } +/// Creates a new copy of the TLS from given master image pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result { let Some(tls) = image.tls.as_ref() else { // No TLS diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 503cb543..24d88b44 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -25,7 +25,7 @@ use crate::{ }, }; -pub struct BufferPlacer<'a> { +struct BufferPlacer<'a> { buffer: &'a mut [u8], virtual_offset: usize, offset: usize, @@ -175,6 +175,7 @@ fn load_binary( } } +/// Loads a program from given `path` pub fn load>( ioctx: &mut IoContext, path: P, diff --git a/src/proc/io.rs b/src/proc/io.rs index 51a3d56e..160e56a9 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -20,18 +20,22 @@ impl ProcessIo { } } + /// Returns the [FileRef] associated with given `fd` or an error if it does not exist pub fn file(&self, fd: RawFd) -> Result<&FileRef, Error> { self.files.get(&fd).ok_or(Error::InvalidFile) } + /// Returns the associated [IoContext] reference pub fn ioctx(&mut self) -> &mut IoContext { self.ioctx.as_mut().unwrap() } + /// Changes the [IoContext] of this struct pub fn set_ioctx(&mut self, ioctx: IoContext) { self.ioctx = Some(ioctx); } + /// Associates a `file` with `fd`, returning an error if requested `fd` is already taken pub fn set_file(&mut self, fd: RawFd, file: FileRef) -> Result<(), Error> { if self.files.contains_key(&fd) { return Err(Error::AlreadyExists); @@ -40,6 +44,7 @@ impl ProcessIo { Ok(()) } + /// Associates a `file` with any available [RawFd] and returns it pub fn place_file(&mut self, file: FileRef) -> Result { for idx in 0..64 { let fd = RawFd::from(idx); @@ -54,12 +59,14 @@ impl ProcessIo { Err(Error::OutOfMemory) } + /// Removes and closes a [FileRef] from the struct pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { // Do nothing, file will be dropped and closed self.files.remove(&fd).ok_or(Error::InvalidFile)?; Ok(()) } + /// Removes all [FileRef]s from the struct which do not pass the `predicate` check pub fn retain bool>(&mut self, predicate: F) { self.files.retain(predicate); } diff --git a/src/proc/random.rs b/src/proc/random.rs index d1c42212..6bba63de 100644 --- a/src/proc/random.rs +++ b/src/proc/random.rs @@ -1,3 +1,5 @@ +//! Random generation utilities + use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::arch::{Architecture, ARCHITECTURE}; @@ -55,11 +57,13 @@ impl RandomState { static RANDOM_STATE: OneTimeInit> = OneTimeInit::new(); +/// Fills `buf` with random bytes pub fn read(buf: &mut [u8]) { let state = RANDOM_STATE.get(); state.lock().read_buf(buf) } +/// Initializes the random generator state pub fn init() { let now = ARCHITECTURE .monotonic_timer() diff --git a/src/task/context.rs b/src/task/context.rs index 96fff708..ecac1854 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -18,7 +18,9 @@ cfg_if! { } } +/// Conversion trait to allow multiple kernel closure return types pub trait Termination { + /// Converts the closure return type into [ExitCode] fn into_exit_code(self) -> ExitCode; } diff --git a/src/task/process.rs b/src/task/process.rs index b834cf67..55897ace 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -30,12 +30,16 @@ use super::{ TaskContext, }; +/// Represents a process state #[derive(PartialEq)] pub enum ProcessState { + /// Process is running, meaning it still has at least one thread alive Running, + /// Process has finished with some [ExitCode] Terminated(ExitCode), } +/// Unique number assigned to each [Process] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] pub struct ProcessId(u64); @@ -47,26 +51,38 @@ pub struct ProcessId(u64); // TLS layout (aarch64): // | uthread_size (0x10?) | mem_size | // | ??? | Data .....| + +/// Describes Thread-Local Storage of a process #[derive(Debug)] pub struct ProcessTlsInfo { + /// Location of the TLS master copy within the process's memory pub master_copy_base: usize, + /// Layout of the TLS pub layout: ProcessTlsLayout, } +/// Describes TLS layout for a program image #[derive(Debug)] pub struct ProcessTlsLayout { + /// Data offset from the TLS base pub data_offset: usize, + /// struct uthread offset from the TLS base pub uthread_offset: usize, + /// Pointer offset from the TLS base. The pointer is passed to the userspace pub ptr_offset: usize, + /// Data size of the TLS segment pub data_size: usize, + /// Memory size of the TLS segment (mem_size >= data_size) pub mem_size: usize, + /// Overall allocation size of the TLS data pub full_size: usize, } #[cfg(target_arch = "aarch64")] impl ProcessTlsLayout { + /// Constructs a new thread-local storage layout info struct pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { debug_assert!(align.is_power_of_two()); let tls_block0_offset = (size_of::() * 2 + align - 1) & !(align - 1); @@ -87,6 +103,7 @@ impl ProcessTlsLayout { #[cfg(target_arch = "x86_64")] impl ProcessTlsLayout { + /// Constructs a new thread-local storage layout info struct pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { // The static TLS blocks are placed below TP // TP points to the TCB @@ -109,8 +126,11 @@ impl ProcessTlsLayout { } } +/// Describes information about a program's image in memory pub struct ProcessImage { + /// Entry point address pub entry: usize, + /// Thread-local storage information pub tls: Option, } @@ -125,6 +145,7 @@ struct ProcessInner { mutexes: BTreeMap>, } +/// Describes a process within the system pub struct Process { name: String, id: ProcessId, @@ -134,6 +155,7 @@ pub struct Process { image: Option, exit_waker: QueueWaker, + /// Process I/O information pub io: IrqSafeSpinlock, } @@ -141,6 +163,7 @@ static PROCESSES: IrqSafeSpinlock>> = IrqSafeSpinlock::new(BTreeMap::new()); impl Process { + /// Creates a new process with given main thread pub fn new_with_main>( name: S, space: Arc, @@ -179,6 +202,7 @@ impl Process { (process, thread) } + /// Spawns a new child thread within the process pub fn spawn_thread(self: &Arc, options: &ThreadSpawnOptions) -> Result { debugln!( "Spawn thread in {} with options: {:#x?}", @@ -210,43 +234,54 @@ impl Process { Ok(id) } + /// Returns the [ProcessId] of this process pub fn id(&self) -> ProcessId { self.id } + /// Returns the process group ID of the process pub fn group_id(&self) -> ProcessId { self.inner.lock().group_id } + /// Returns the process session ID of the process pub fn session_id(&self) -> ProcessId { self.inner.lock().session_id } + /// Returns the process name pub fn name(&self) -> &str { self.name.as_ref() } + /// Changes the process's group ID pub fn set_group_id(&self, id: ProcessId) { self.inner.lock().group_id = id; } + /// Changes the process's session ID pub fn set_session_id(&self, id: ProcessId) { self.inner.lock().session_id = id; } // Resources + + /// Returns the current session terminal of the process, if set pub fn session_terminal(&self) -> Option { self.inner.lock().session_terminal.clone() } + /// Changes the current session terminal of the process pub fn set_session_terminal(&self, node: NodeRef) { self.inner.lock().session_terminal.replace(node); } + /// Resets the current session terminal of the process pub fn clear_session_terminal(&self) -> Option { self.inner.lock().session_terminal.take() } + /// Inherits the process information from the `parent` pub fn inherit(&self, parent: &Process) -> Result<(), Error> { let mut our_inner = self.inner.lock(); let their_inner = parent.inner.lock(); @@ -259,6 +294,8 @@ impl Process { } // State + + /// Returns the [ExitCode] of the process, if it has exited pub fn get_exit_status(&self) -> Option { match self.inner.lock().state { ProcessState::Running => None, @@ -266,6 +303,7 @@ impl Process { } } + /// Suspends the task until the process exits pub fn wait_for_exit(process: Arc) -> impl Future { struct ProcessExitFuture { process: Arc, @@ -291,6 +329,7 @@ impl Process { ProcessExitFuture { process } } + /// Handles exit of a single child thread pub fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { debugln!("Thread {} of process {}: {:?}", thread, self.id, code); let mut inner = self.inner.lock(); @@ -335,6 +374,8 @@ impl Process { } } + /// Returns a [UserspaceMutex] associated with the `address`. If one does not exist, will + /// create it. pub fn get_or_insert_mutex(&self, address: usize) -> Arc { let mut inner = self.inner.lock(); inner @@ -345,10 +386,13 @@ impl Process { } // Process list + + /// Returns the process with given [ProcessId], if it exists pub fn get(id: ProcessId) -> Option> { PROCESSES.lock().get(&id).cloned() } + /// Terminates all children of the process, `except` one pub async fn terminate_others(&self, except: ThreadId) { let mut inner = self.inner.lock(); @@ -385,231 +429,10 @@ impl From for ProcessId { } impl ProcessId { + /// Generates a new [ProcessId] pub fn next() -> Self { static COUNTER: AtomicU64 = AtomicU64::new(1); let id = COUNTER.fetch_add(1, Ordering::SeqCst); Self(id) } } - -// /// Returns the state of the process. -// /// -// /// # Note -// /// -// /// Maybe I should remove this and make ALL state changes atomic. -// pub fn state(&self) -> ProcessState { -// self.state.load(Ordering::Acquire) -// } -// -// /// Atomically updates the state of the process and returns the previous one. -// pub fn set_state(&self, state: ProcessState) -> ProcessState { -// self.state.swap(state, Ordering::SeqCst) -// } -// -// /// Marks the task as running on the specified CPU. -// /// -// /// # Safety -// /// -// /// Only meant to be called from scheduler routines. -// pub unsafe fn set_running(&self, cpu: u32) { -// self.cpu_id.store(cpu, Ordering::Release); -// self.state.store(ProcessState::Running, Ordering::Release); -// } -// -// /// Returns the address space of the task -// pub fn address_space(&self) -> &ProcessAddressSpace { -// self.space.as_ref().unwrap() -// } -// -// /// Returns the address space of the task, if one is set -// pub fn get_address_space(&self) -> Option<&ProcessAddressSpace> { -// self.space.as_ref() -// } -// -// /// Replaces the task's session terminal device with another one -// pub fn set_session_terminal(&self, terminal: VnodeRef) { -// self.inner.lock().session_terminal.replace(terminal); -// } -// -// /// Removes the task's current terminal -// pub fn clear_session_terminal(&self) -> Option { -// } -// -// /// Returns the current terminal of the task -// pub fn session_terminal(&self) -> Option { -// self.inner.lock().session_terminal.clone() -// } -// -// /// Sets the session ID of the task -// pub fn set_session_id(&self, sid: ProcessId) { -// self.inner.lock().session_id = sid; -// } -// -// /// Sets the process group ID of the task -// pub fn set_group_id(&self, mut gid: ProcessId) { -// if gid == 0 { -// gid = self.id(); -// } -// self.inner.lock().group_id = gid; -// } -// -// /// Returns the process group ID of the task -// pub fn group_id(&self) -> ProcessId { -// self.inner.lock().group_id -// } -// -// /// Returns the CPU number this task in running on (or the last one) -// pub fn cpu_id(&self) -> u32 { -// self.cpu_id.load(Ordering::Acquire) -// } -// -// /// Selects a suitable CPU queue and submits the process for execution. -// /// -// /// # Panics -// /// -// /// Currently, the code will panic if the process is queued/executing on any queue. -// pub fn enqueue_somewhere(self: Arc) -> usize { -// // Doesn't have to be precise, so even if something changes, we can still be rebalanced -// // to another CPU -// let (index, queue) = CpuQueue::least_loaded().unwrap(); -// -// self.enqueue_to(queue); -// -// index -// } -// -// /// Submits the process to a specific queue. -// /// -// /// # Panics -// /// -// /// Currently, the code will panic if the process is queued/executing on any queue. -// pub fn enqueue_to(self: Arc, queue: &'static CpuQueue) { -// let _irq = IrqGuard::acquire(); -// -// { -// let mut inner = self.inner.lock(); -// let old_queue = inner.queue.replace(queue); -// if old_queue.is_some() { -// // Already in some queue -// return; -// } -// } -// match self.state.compare_exchange( -// ProcessState::Suspended, -// ProcessState::Ready, -// Ordering::SeqCst, -// Ordering::Relaxed, -// ) { -// Err(ProcessState::Terminated) => { -// // Process might've been killed while `await`ing in a `block!` -// debugln!( -// "Process {} {:?} already terminated, dropping", -// self.id(), -// self.name() -// ); -// } -// Err(state) => { -// todo!("Unexpected process state when enqueueing: {:?}", state) -// } -// Ok(_) => unsafe { -// queue.enqueue(self); -// }, -// } -// } -// -// /// Returns an exit code if the process exited, [None] if it didn't -// pub fn get_exit_status(&self) -> Option { -// if self.state() == ProcessState::Terminated { -// Some(ExitCode::from(self.inner.lock().exit_status)) -// } else { -// None -// } -// } -// -// /// Returns the [Process] currently executing on local CPU, None if idling. -// pub fn get_current() -> Option { -// let queue = Cpu::local().queue(); -// queue.current_process() -// } -// -// /// Returns a process by its ID -// pub fn get(pid: ProcessId) -> Option> { -// PROCESSES.lock().get(pid).cloned() -// } -// -// /// Wraps [Process::get_current()] for cases when the caller is absolutely sure there is a -// /// running process (e.g. the call itself comes from a process). -// pub fn current() -> CurrentProcess { -// Self::get_current().unwrap() -// } -// -// /// Handles the cleanup of an exited process -// pub fn handle_exit(&self) { -// } -// -// /// Inherits the data from a parent process. Meant to be called from SpawnProcess handler. -// pub fn inherit(&self, parent: &Arc) -> Result<(), Error> { -// } -// -// -// pub fn wait_for_exit(process: Arc) -> impl Future { -// } -// } -// -// impl ArcWake for Process { -// fn wake_by_ref(arc_self: &Arc) { -// arc_self.clone().enqueue_somewhere(); -// } -// } -// -// impl Drop for Process { -// fn drop(&mut self) { -// infoln!("Drop process!"); -// } -// } -// -// impl CurrentProcess { -// /// Wraps a process in this structure. -// /// -// /// # Safety -// /// -// /// Only meant to be called from [Process::current] or [CpuQueue::current_process]. -// pub unsafe fn new(inner: Arc, guard: IrqGuard) -> Self { -// Self(inner, guard) -// } -// -// /// Configures signal entry information for the process -// pub fn set_signal_entry(&self, entry: usize, stack: usize) { -// let mut inner = self.inner.lock(); -// inner.signal_entry.replace(SignalEntry { entry, stack }); -// } -// -// pub fn suspend(&self) -> Result<(), Error> { -// self.dequeue(ProcessState::Suspended); -// -// let inner = self.inner.lock(); -// if !inner.signal_stack.is_empty() { -// return Err(Error::Interrupted); -// } -// -// Ok(()) -// } -// -// /// Terminate the current process -// pub fn exit(&self, status: ExitCode) { -// self.inner.lock().exit_status = status.into(); -// debugln!("Process {} exited with code {:?}", self.id(), status); -// -// self.handle_exit(); -// self.dequeue(ProcessState::Terminated); -// } -// -// } -// -// impl Deref for CurrentProcess { -// type Target = Arc; -// -// fn deref(&self) -> &Self::Target { -// &self.0 -// } -// } diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs index 7509bc7f..53f79473 100644 --- a/src/task/runtime/executor.rs +++ b/src/task/runtime/executor.rs @@ -11,10 +11,12 @@ use super::{ task_queue::{self}, }; +/// Pushes a task into the executor's queue pub fn enqueue(task: Arc) -> Result<(), Error> { task_queue::push_task(task) } +/// Spawns a background worker to execute the tasks from the global queue pub fn spawn_async_worker(index: usize) -> Result<(), Error> { let name = format!("[async-worker-{}]", index); @@ -35,10 +37,12 @@ pub fn spawn_async_worker(index: usize) -> Result<(), Error> { }) } +/// Creates a new task for the [Future] and queues it for execution in background pub fn spawn + Send + 'static>(future: F) -> Result<(), Error> { enqueue(Task::new(future)) } +/// Runs a [Future] to its completion on the current thread pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { let thread = Thread::current(); let mut future = Box::pin(future); diff --git a/src/task/runtime/macros.rs b/src/task/runtime/macros.rs index 18aa331e..cdb0db3b 100644 --- a/src/task/runtime/macros.rs +++ b/src/task/runtime/macros.rs @@ -1,3 +1,4 @@ +/// Runs a [futures_util::Future] to its completion #[macro_export] macro_rules! block { ($($stmt:tt)*) => { @@ -7,6 +8,7 @@ macro_rules! block { }; } +/// Runs two Futures, returning the result of whichever finishes first #[macro_export] macro_rules! any { ($fut0:ident = $pat0:pat => $body0:expr, $fut1:ident = $pat1:pat => $body1:expr) => { diff --git a/src/task/runtime/mod.rs b/src/task/runtime/mod.rs index 4b6e9533..b20bfa02 100644 --- a/src/task/runtime/mod.rs +++ b/src/task/runtime/mod.rs @@ -1,3 +1,5 @@ +//! Async/await runtime implementation + use core::{ pin::Pin, task::{Context, Poll}, @@ -18,6 +20,7 @@ pub use executor::{run_to_completion, spawn, spawn_async_worker}; pub use task_queue::init_task_queue; pub use waker::QueueWaker; +/// [Future] implementation that wraps a poll-function pub struct PollFn { f: F, } @@ -34,6 +37,7 @@ where } } +/// Constructs a [PollFn] from given poll-function pub fn poll_fn(f: F) -> PollFn where F: FnMut(&mut Context) -> Poll, @@ -41,8 +45,9 @@ where PollFn { f } } -pub static SLEEP_WAKER: QueueWaker = QueueWaker::new(); +static SLEEP_WAKER: QueueWaker = QueueWaker::new(); +/// Suspends the task until given duration passes pub fn sleep(duration: Duration) -> impl Future { struct SleepFuture { deadline: Duration, @@ -75,6 +80,7 @@ pub fn sleep(duration: Duration) -> impl Future { SleepFuture { deadline } } +/// Updates the runtime's time pub fn tick(_now: Duration) { SLEEP_WAKER.wake_all(); } diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs index 43970baf..306f61cd 100644 --- a/src/task/runtime/task_queue.rs +++ b/src/task/runtime/task_queue.rs @@ -60,6 +60,7 @@ impl TaskQueue { } } +/// Initializes the global async/await task queue pub fn init_task_queue() { TASK_QUEUE.init(TaskQueue::new(128)); } diff --git a/src/task/runtime/waker.rs b/src/task/runtime/waker.rs index 2f20f992..76aceda1 100644 --- a/src/task/runtime/waker.rs +++ b/src/task/runtime/waker.rs @@ -3,17 +3,20 @@ use core::task::Waker; use alloc::collections::VecDeque; use kernel_util::sync::IrqSafeSpinlock; +/// Async/await primitive to suspend and wake up tasks waiting on some shared resource pub struct QueueWaker { queue: IrqSafeSpinlock>, } impl QueueWaker { + /// Constructs an empty [QueueWaker] pub const fn new() -> Self { Self { queue: IrqSafeSpinlock::new(VecDeque::new()), } } + /// Registers a [Waker] reference to be waken up by this [QueueWaker] pub fn register(&self, waker: &Waker) { let mut queue = self.queue.lock(); @@ -24,6 +27,7 @@ impl QueueWaker { queue.push_back(waker.clone()); } + /// Removes a [Waker] reference from this [QueueWaker] pub fn remove(&self, waker: &Waker) -> bool { let mut queue = self.queue.lock(); let mut index = 0; @@ -40,6 +44,7 @@ impl QueueWaker { removed } + /// Wakes up up to `limit` tasks waiting on this queue pub fn wake_some(&self, limit: usize) -> usize { let mut queue = self.queue.lock(); let mut count = 0; @@ -52,10 +57,12 @@ impl QueueWaker { count } + /// Wakes up a single task waiting on this queue pub fn wake_one(&self) -> bool { self.wake_some(1) != 0 } + /// Wakes up all tasks waiting on this queue pub fn wake_all(&self) -> usize { self.wake_some(usize::MAX) } diff --git a/src/task/sched.rs b/src/task/sched.rs index 377a7d7f..995082d2 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -29,17 +29,6 @@ pub struct CpuQueueStats { measure_time: u64, } -// /// Per-CPU queue's inner data, normally resides under a lock -// pub struct CpuQueueInner { -// /// Current process, None if idling -// pub current: Option>, -// /// LIFO queue for processes waiting for execution -// pub queue: VecDeque>, -// -// /// CPU time usage statistics -// pub stats: CpuQueueStats, -// } - struct CpuQueueInner { current: Option, queue: VecDeque, @@ -117,7 +106,7 @@ impl CpuQueueInner { } impl CpuQueue { - // /// Constructs an empty queue with its own idle task + /// Constructs an empty queue with its own idle task pub fn new(index: usize) -> Self { let idle = TaskContext::kernel(__idle, Cpu::local_id() as usize) .expect("Could not construct an idle task"); @@ -246,40 +235,7 @@ impl CpuQueue { self.inner.lock().queue.is_empty() } - // /// Returns a safe reference to the inner data structure. - // pub fn lock(&self) -> IrqSafeSpinlockGuard { - // self.inner.lock() - // } - - // /// Returns an unsafe reference to the queue. - // /// - // /// # Safety - // /// - // /// Only meant to be called to dump the queue contents when panicking. - // #[allow(clippy::mut_from_ref)] - // pub unsafe fn grab(&self) -> &mut CpuQueueInner { - // self.inner.grab() - // } - // - // /// Returns the process currently being executed. - // /// - // /// # Note - // /// - // /// This function should be safe in all kernel thread/interrupt contexts: - // /// - // /// * (in kthread) the code calling this will still remain on the same thread. - // /// * (in irq) the code cannot be interrupted and other CPUs shouldn't change this queue, so it - // /// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu] - // /// is called. - // pub fn current_process(&self) -> Option { - // let guard = IrqGuard::acquire(); - // self.inner - // .lock() - // .current - // .clone() - // .map(|p| unsafe { CurrentProcess::new(p, guard) }) - // } - + /// Returns the current [ThreadId], or [None] if idle pub fn current_id(&self) -> Option { self.inner.lock().current } @@ -302,6 +258,7 @@ impl CpuQueue { queues.iter().enumerate().min_by_key(|(_, q)| q.len()) } + /// Returns the CPU index of the queue pub fn index(&self) -> usize { self.index } diff --git a/src/task/sync.rs b/src/task/sync.rs index a0baa1a9..fcc0fb5a 100644 --- a/src/task/sync.rs +++ b/src/task/sync.rs @@ -1,3 +1,5 @@ +//! Higher-order task synchronization primitives + use core::{ pin::Pin, sync::atomic::{AtomicU32, Ordering}, @@ -9,12 +11,14 @@ use futures_util::Future; use super::runtime::QueueWaker; +/// User-space mutex (like BSD/Linux's futex) data structure pub struct UserspaceMutex { queue: QueueWaker, address: usize, } impl UserspaceMutex { + /// Creates a new [UserspaceMutex] associated with given `address` pub fn new(address: usize) -> Self { Self { queue: QueueWaker::new(), @@ -22,6 +26,7 @@ impl UserspaceMutex { } } + /// Blocks until the value at the mutex's address becomes different from `compare_value` pub fn wait(self: Arc, compare_value: u32) -> impl Future { struct WaitFuture { mutex: Arc, @@ -57,10 +62,12 @@ impl UserspaceMutex { } } + /// Wakes up a single task waiting on the mutex pub fn wake(&self) { self.queue.wake_one(); } + /// Wakes up all tasks waiting on the mutex pub fn wake_all(&self) { self.queue.wake_all(); } diff --git a/src/task/thread.rs b/src/task/thread.rs index 28f3b0aa..9aa9d51b 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -1,3 +1,5 @@ +//! Thread data structures and management + use core::{ fmt, mem::size_of, @@ -44,12 +46,16 @@ pub enum ThreadState { Terminated, } +/// Unique number describing a single kernel or userspace thread #[derive(Debug, PartialEq, Eq, Clone, Copy, Ord, PartialOrd)] pub enum ThreadId { + /// Describes a kernel-space thread Kernel(u64), + /// Describes an user-space thread User(u64), } +/// Wrapper which guarantees the thread referred to is the current one on the current CPU pub struct CurrentThread(Arc, IrqGuard); struct SignalEntry { @@ -64,6 +70,7 @@ struct ThreadInner { signal_stack: VecDeque, } +/// Describes a single thread within the system pub struct Thread { context: TaskContext, name: Option, @@ -83,10 +90,6 @@ static THREADS: IrqSafeSpinlock>> = IrqSafeSpinlock::new(BTreeMap::new()); impl Thread { - // pub fn new(index: usize, parent: &Process, context: TaskContext) -> Self { - // todo!() - // } - fn new( id: ThreadId, name: Option, @@ -120,6 +123,7 @@ impl Thread { thread } + /// Constructs a new user-space thread pub fn new_uthread( parent: Arc, space: Arc, @@ -134,6 +138,7 @@ impl Thread { ) } + /// Constructs a new kernel-space thread pub fn new_kthread>(name: S, context: TaskContext) -> Arc { Self::new( ThreadId::next_kernel(), @@ -145,31 +150,53 @@ impl Thread { } // Info + + /// Returns the thread's ID pub fn id(&self) -> ThreadId { self.id } + /// Returns the thread's name, if set pub fn name(&self) -> Option<&String> { self.name.as_ref() } + /// Returns the thread's [TaskContext] pub fn context(&self) -> &TaskContext { &self.context } + /// Returns the thread's [ProcessAddressSpace] reference. + /// + /// # Panics + /// + /// Will panic if the thread has no associated address space. pub fn address_space(&self) -> &Arc { self.space.as_ref().unwrap() } + /// Returns the thread's parent [Process] reference. + /// + /// # Panics + /// + /// Will panic if the thread has no process associated (i.e. it's a kernel thread). pub fn process(&self) -> &Arc { self.process.as_ref().unwrap() } // Queue operation + + /// Returns the current thread on the CPU. + /// + /// # Panics + /// + /// Will panic if no current thread is present. For try-style getter, see + /// [Thread::get_current]. pub fn current() -> CurrentThread { Self::get_current().unwrap() } + /// Returns the current thread on the CPU, if any is present pub fn get_current() -> Option { let guard = IrqGuard::acquire(); Cpu::local() @@ -179,6 +206,8 @@ impl Thread { .map(|t| CurrentThread(t, guard)) } + /// Enqueues the thread onto any (usually the least loaded) CPU queue and returns its index. + /// See [Thread::enqueue_to]. pub fn enqueue_somewhere(&self) -> usize { // Doesn't have to be precise, so even if something changes, we can still be rebalanced // to another CPU @@ -189,6 +218,11 @@ impl Thread { index } + /// Enqueues the thread onto the specific CPU's queue. + /// + /// # Panics + /// + /// Will panic if the process is in some weird state while being queued. pub fn enqueue_to(&self, queue: &'static CpuQueue) { let _irq = IrqGuard::acquire(); @@ -261,7 +295,7 @@ impl Thread { } } - /// Marks the process as running and sets its "current" CPU index. + /// Marks the thread as running and sets its "current" CPU index. /// /// # Safety /// @@ -271,21 +305,31 @@ impl Thread { self.state.store(ThreadState::Running, Ordering::Release); } + /// Suspends the thread and removes it from the queue until it is "waken up" again pub fn suspend(&self) { self.dequeue(ThreadState::Suspended); } // Accounting + + /// Returns the thread with given [ThreadId], if it exists pub fn get(id: ThreadId) -> Option> { THREADS.lock().get(&id).cloned() } // Thread inner + + /// Changes the thread's signal entry point information pub fn set_signal_entry(&self, entry: usize, stack: usize) { let mut inner = self.inner.lock(); inner.signal_entry.replace(SignalEntry { entry, stack }); } + /// Pushes a [Signal] onto the thread's signal stack. + /// + /// When executed on a current thread, the signal is guaranteed to be handled exactly before + /// returning to user context (i.e. from syscall). Otherwise, the signal handling order and + /// whether it will be delivered at all is not guaranteed. pub fn raise_signal(self: &Arc, signal: Signal) { self.inner.lock().signal_stack.push_back(signal); @@ -294,6 +338,7 @@ impl Thread { } } + /// Requests thread termination and blocks until said thread finishes fully: pub fn terminate(self: &Arc) -> impl Future { struct F(Arc); @@ -365,6 +410,8 @@ impl CurrentThread { self.dequeue_terminate(code) } + /// Terminate the parent process of the thread, including all other threads and the current + /// thread itself pub fn exit_process(&self, code: ExitCode) { let _guard = IrqGuard::acquire(); @@ -384,6 +431,7 @@ impl CurrentThread { unreachable!(); } + /// Suspends the current thread until it is waken up again. Guaranteed to happen immediately. pub fn suspend(&self) -> Result<(), Error> { self.dequeue(ThreadState::Suspended); @@ -457,18 +505,25 @@ impl Deref for CurrentThread { } impl ThreadId { + /// Generates a new kernel-space thread ID pub fn next_kernel() -> Self { static COUNT: AtomicU64 = AtomicU64::new(1); let id = COUNT.fetch_add(1, Ordering::SeqCst); Self::Kernel(id) } + /// Generates a new user-space thread ID pub fn next_user() -> Self { static COUNT: AtomicU64 = AtomicU64::new(1); let id = COUNT.fetch_add(1, Ordering::SeqCst); Self::User(id) } + /// Returns the number of the userspace thread represented by this ID. + /// + /// # Panics + /// + /// Will panic if this [ThreadId] does not represent a user-space thread. pub fn as_user(&self) -> u64 { match self { Self::Kernel(_) => panic!(), @@ -476,6 +531,7 @@ impl ThreadId { } } + /// Returns `true` if the [ThreadId] represents a user-space thread pub fn is_user(&self) -> bool { matches!(self, ThreadId::User(_)) } diff --git a/src/util/mod.rs b/src/util/mod.rs index 1ee02742..44184486 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -3,7 +3,9 @@ pub mod queue; pub mod ring; +/// Extension trait for [Iterator]s of [Result]s pub trait ResultIterator { + /// Drops entries from the iterator until the first error fn collect_error(self) -> Option; } @@ -18,6 +20,7 @@ impl>> ResultIterator for I { } } +/// Returns the architecture name string pub const fn arch_str() -> &'static str { #[cfg(target_arch = "aarch64")] { @@ -28,12 +31,3 @@ pub const fn arch_str() -> &'static str { "x86_64" } } - -// /// Performs a busy-loop sleep until the specified duration has passed -// pub fn polling_sleep(duration: Duration) -> Result<(), Error> { -// // TODO no non-IRQ mode timestamp provider -// for i in 0..1000000 { -// core::hint::spin_loop(); -// } -// Ok(()) -// } diff --git a/src/util/queue.rs b/src/util/queue.rs index 09b0d55d..fe888221 100644 --- a/src/util/queue.rs +++ b/src/util/queue.rs @@ -1,3 +1,5 @@ +//! Asynchronous array queue implementation + use core::{ pin::Pin, task::{Context, Poll}, @@ -9,12 +11,14 @@ use futures_util::Future; use crate::task::runtime::QueueWaker; +/// Asynchronous queue pub struct AsyncQueue { waker: Arc, queue: Arc>, } impl AsyncQueue { + /// Constructs a new [AsyncQueue] of requested capacity pub fn new(capacity: usize) -> Self { Self { waker: Arc::new(QueueWaker::new()), @@ -22,6 +26,7 @@ impl AsyncQueue { } } + /// Pushes an entry to the queue and signals a single task about the available value pub fn send(&self, value: T) -> Result<(), T> { let result = self.queue.push(value); if result.is_ok() { @@ -30,6 +35,7 @@ impl AsyncQueue { result } + /// Asynchronously receives an entry from the queue pub fn recv(&self) -> impl Future { struct AsyncQueueRecvFuture { waker: Arc, diff --git a/src/util/ring.rs b/src/util/ring.rs index 3111f514..15ad752a 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -1,3 +1,5 @@ +//! Ring buffer implementation + use core::{ pin::Pin, task::{Context, Poll}, @@ -10,18 +12,21 @@ use kernel_util::sync::IrqSafeSpinlock; use crate::task::runtime::QueueWaker; +/// Ring buffer base pub struct RingBuffer { rd: usize, wr: usize, data: [T; N], } +/// Ring buffer with async read support pub struct AsyncRing { inner: Arc>>, read_waker: Arc, } impl RingBuffer { + /// Constructs a new [RingBuffer] and fills it with `value` pub const fn new(value: T) -> Self { Self { rd: 0, @@ -44,12 +49,13 @@ impl RingBuffer { } #[inline] - fn read_single(&mut self) -> T { + fn read_single_unchecked(&mut self) -> T { let res = self.data[self.rd]; self.rd = (self.rd + 1) % N; res } + /// Reads all entries available from `pos` to the write head pub fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize { let mut pos = (self.rd + pos) % N; let mut off = 0; @@ -61,6 +67,7 @@ impl RingBuffer { off } + /// Writes a single entry to the buffer #[inline] pub fn write(&mut self, ch: T) { self.data[self.wr] = ch; @@ -69,6 +76,7 @@ impl RingBuffer { } impl AsyncRing { + /// Constructs a new [AsyncRing] and fills it with `value` pub fn new(value: T) -> Self { Self { inner: Arc::new(IrqSafeSpinlock::new(RingBuffer::new(value))), @@ -76,6 +84,7 @@ impl AsyncRing { } } + /// Writes a single entry to the buffer and signals readers pub fn try_write(&self, item: T) -> Result<(), Error> { { let mut lock = self.inner.lock(); @@ -86,6 +95,7 @@ impl AsyncRing { Ok(()) } + /// Asynchronously reads an entry from the buffer pub fn read(&self) -> impl Future { struct ReadFuture { inner: Arc>>, @@ -101,7 +111,7 @@ impl AsyncRing { let mut inner = self.inner.lock(); if inner.is_readable() { self.read_waker.remove(cx.waker()); - Poll::Ready(inner.read_single()) + Poll::Ready(inner.read_single_unchecked()) } else { Poll::Pending } From d1fe89c134c7b1adbc56b040b77dca1d69f034d1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 8 Dec 2023 14:30:49 +0200 Subject: [PATCH 105/211] dev/block: nvme hello world --- src/device/bus/pci/mod.rs | 21 +- src/device/bus/pci/space/ecam.rs | 5 + src/device/bus/pci/space/mod.rs | 46 ++++- src/device/mod.rs | 2 + src/device/nvme/command.rs | 36 ++++ src/device/nvme/mod.rs | 261 ++++++++++++++++++++++++ src/device/nvme/queue.rs | 334 +++++++++++++++++++++++++++++++ src/init.rs | 1 + 8 files changed, 701 insertions(+), 5 deletions(-) create mode 100644 src/device/nvme/command.rs create mode 100644 src/device/nvme/mod.rs create mode 100644 src/device/nvme/queue.rs diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index a828ff53..abb3f8c1 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -2,7 +2,7 @@ use core::fmt; use acpi_lib::mcfg::McfgEntry; -use alloc::{rc::Rc, vec::Vec}; +use alloc::{boxed::Box, rc::Rc, vec::Vec}; use bitflags::bitflags; use device_api::Device; use kernel_util::sync::IrqSafeSpinlock; @@ -14,7 +14,10 @@ pub use space::{ ecam::PciEcam, PciConfigSpace, PciConfigurationSpace, PciLegacyConfigurationSpace, }; -use crate::mem::{address::FromRaw, PhysicalAddress}; +use crate::{ + device::nvme, + mem::{address::FromRaw, PhysicalAddress}, +}; bitflags! { /// Command register of the PCI configuration space @@ -241,6 +244,13 @@ impl PciConfigurationSpace for PciConfigSpace { _ => todo!(), } } + + fn write_u32(&self, offset: usize, value: u32) { + match self { + Self::Ecam(ecam) => ecam.write_u32(offset, value), + _ => todo!(), + } + } } fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> { @@ -259,6 +269,13 @@ fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> { // Match by class match (config.class_code(), config.subclass(), config.prog_if()) { + (0x01, 0x08, 0x02) => { + let dev = Box::leak(Box::new(nvme::NvmeController::from_pci_bus(&device.info)?)); + + unsafe { + dev.init()?; + } + } _ => { debugln!(" -> No driver"); } diff --git a/src/device/bus/pci/space/ecam.rs b/src/device/bus/pci/space/ecam.rs index 35781baf..7fac9d45 100644 --- a/src/device/bus/pci/space/ecam.rs +++ b/src/device/bus/pci/space/ecam.rs @@ -17,6 +17,11 @@ impl PciConfigurationSpace for PciEcam { assert_eq!(offset & 3, 0); unsafe { ((self.mapping.address() + offset) as *const u32).read_volatile() } } + + fn write_u32(&self, offset: usize, value: u32) { + assert_eq!(offset & 3, 0); + unsafe { ((self.mapping.address() + offset) as *mut u32).write_volatile(value) } + } } impl PciEcam { diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs index acf80491..72a5018d 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/src/device/bus/pci/space/mod.rs @@ -16,6 +16,20 @@ macro_rules! pci_config_field_getter { }; } +macro_rules! pci_config_field_setter { + ($self:ident, u32, $offset:expr, $value:expr) => { + $self.write_u32($offset, $value) + }; + + ($self:ident, u16, $offset:expr, $value:expr) => {{ + $self.write_u16($offset, $value) + }}; + + ($self:ident, u8, $offset:expr, $value:expr) => { + $self.write_u8($offset, $value) + }; +} + macro_rules! pci_config_field { ( $offset:expr => $ty:ident, @@ -29,8 +43,8 @@ macro_rules! pci_config_field { $( $(#[$setter_meta])* - fn $setter(&self, _value: $ty) { - todo!() + fn $setter(&self, value: $ty) { + pci_config_field_setter!(self, $ty, $offset, value) } )? }; @@ -67,6 +81,13 @@ pub trait PciConfigurationSpace { /// The `offset` must be u32-aligned. fn read_u32(&self, offset: usize) -> u32; + /// Writes a 32-bit value to the device configuration space. + /// + /// # Note + /// + /// The `offset` must be u32-aligned. + fn write_u32(&self, offset: usize, value: u32); + /// Reads a 16-bit value from the device configuration space. /// /// # Note @@ -78,12 +99,31 @@ pub trait PciConfigurationSpace { (value >> ((offset & 3) * 8)) as u16 } - /// Reads a byte from the device configuration space. + /// Reads a byte from the device configuration space fn read_u8(&self, offset: usize) -> u8 { let value = self.read_u32(offset & !3); (value >> ((offset & 3) * 8)) as u8 } + /// Writes a 16-bit value to the device configuration space. + /// + /// # Note + /// + /// The `offset` must be u16-aligned. + fn write_u16(&self, offset: usize, value: u16) { + let shift = ((offset >> 1) & 1) << 4; + assert_eq!(offset & 1, 0); + let mut tmp = self.read_u32(offset & !3); + tmp &= !(0xFFFF << shift); + tmp |= (value as u32) << shift; + self.write_u32(offset & !3, tmp); + } + + /// Writes a byte to the device configuration space + fn write_u8(&self, offset: usize, value: u16) { + todo!() + } + /// Returns `true` if the device is present on the bus (i.e. configuration space is not filled /// with only 1's) fn is_valid(&self) -> bool { diff --git a/src/device/mod.rs b/src/device/mod.rs index 7fa4d9e5..e5c73988 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -10,6 +10,8 @@ pub mod devtree; #[cfg(not(target_arch = "aarch64"))] pub mod bus; +pub mod nvme; + pub mod display; pub mod power; pub mod serial; diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs new file mode 100644 index 00000000..d963691e --- /dev/null +++ b/src/device/nvme/command.rs @@ -0,0 +1,36 @@ +use super::queue::SubmissionQueueEntry; + +pub trait Command { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry); +} + +#[derive(Clone, Copy, Debug)] +#[repr(u32)] +pub enum IdentifyCommand { + Controller = 0x01, +} + +#[repr(C)] +pub struct IdentifyControllerResponse { + pub pci_vid: u16, + pub pci_ssvid: u16, + pub serial_number: [u8; 20], + pub model_number: [u8; 40], + pub firmware_rev: u64, +} + +pub enum AdminCommand { + Identify { nsid: u32, cns: IdentifyCommand }, +} + +impl Command for AdminCommand { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + match self { + AdminCommand::Identify { nsid, cns } => { + sqe.command.set_opcode(0x06); + sqe.nsid = *nsid; + sqe.command_specific[0] = *cns as u32; + } + } + } +} diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs new file mode 100644 index 00000000..3adcd165 --- /dev/null +++ b/src/device/nvme/mod.rs @@ -0,0 +1,261 @@ +#![allow(missing_docs)] +use core::{mem::size_of, ptr::null_mut, time::Duration}; + +use abi::error::Error; +use bytemuck::{Pod, Zeroable}; +use device_api::Device; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use static_assertions::{const_assert, const_assert_eq}; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite, WriteOnly}, +}; + +use crate::{ + arch::{x86_64::mem::table::L3, Architecture, ARCHITECTURE}, + debug, + device::{ + bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace}, + nvme::{ + command::{AdminCommand, IdentifyCommand}, + queue::{CompletionQueueEntry, SubmissionQueueEntry}, + }, + }, + mem::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw}, + device::DeviceMemoryIo, + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + table::EntryLevel, + PhysicalAddress, + }, + task::runtime, +}; + +use self::queue::QueuePair; + +use super::bus::pci::{FromPciBus, PciDeviceInfo}; + +mod command; +mod queue; + +register_bitfields! { + u32, + CC [ + IOCQES OFFSET(20) NUMBITS(4) [], + IOSQES OFFSET(16) NUMBITS(4) [], + AMS OFFSET(11) NUMBITS(3) [], + MPS OFFSET(7) NUMBITS(4) [], + CSS OFFSET(4) NUMBITS(3) [ + NvmCommandSet = 0 + ], + ENABLE OFFSET(0) NUMBITS(1) [], + ], + CSTS [ + CFS OFFSET(1) NUMBITS(1) [], + RDY OFFSET(0) NUMBITS(1) [], + ], + AQA [ + /// Admin Completion Queue Size in entries - 1 + ACQS OFFSET(16) NUMBITS(12) [], + /// Admin Submission Queue Size in entries - 1 + ASQS OFFSET(0) NUMBITS(12) [], + ] +} + +register_bitfields! { + u64, + CAP [ + /// Maximum Queue Entries Supported - 1. i.e., 0 means maximum queue len of 1, 1 = 2 etc. + MQES OFFSET(0) NUMBITS(16) [], + /// Timeout. Represents the worst-case time the host software should wait for CSTS.RDY to + /// change its state. + TO OFFSET(24) NUMBITS(8) [], + /// Doorbell stride. Stride in bytes = pow(2, 2 + DSTRD). + DSTRD OFFSET(32) NUMBITS(4) [], + /// NVM Subsystem Reset Supported (see NVMe BS Section 3.7.1) + NSSRS OFFSET(36) NUMBITS(1) [], + /// Controller supports one or more I/O command sets + CSS_IO_COMMANDS OFFSET(43) NUMBITS(1) [], + /// Controller only supports admin commands and no I/O commands + CSS_ADMIN_ONLY OFFSET(44) NUMBITS(1) [], + /// Memory page size minimum (bytes = pow(2, 12 + MPSMIN)) + MPSMIN OFFSET(48) NUMBITS(4) [], + /// Memory page size maximum -|- + MPSMAX OFFSET(52) NUMBITS(4) [], + ] +} + +register_structs! { + #[allow(non_snake_case)] + Regs { + (0x00 => CAP: ReadOnly), + (0x08 => VS: ReadOnly), + (0x0C => INTMS: WriteOnly), + (0x10 => INTMC: WriteOnly), + (0x14 => CC: ReadWrite), + (0x18 => _0), + (0x1C => CSTS: ReadOnly), + (0x20 => _1), + (0x24 => AQA: ReadWrite), + (0x28 => ASQ: ReadWrite), + (0x30 => ACQ: ReadWrite), + (0x38 => _2), + (0x2000 => @END), + } +} + +pub struct NvmeController { + regs: IrqSafeSpinlock>, + admin_q: OneTimeInit>>, +} + +impl Regs { + unsafe fn doorbell_ptr(&self, shift: u64, completion: bool, queue_index: usize) -> *mut u32 { + let doorbell_base = (self as *const Regs as *mut Regs).addr() + 0x1000; + let offset = (queue_index << shift) + completion as usize * 4; + (doorbell_base + offset) as *mut u32 + } +} + +impl NvmeController { + async fn late_init(&'static self) { + let mut admin_q = self.admin_q.get().lock(); + + // Identify + let page = phys::alloc_page().unwrap(); + let mut ranges = [page]; + + debugln!("output range = {:#x}", ranges[0]); + admin_q + .perform_request( + AdminCommand::Identify { + nsid: 0, + cns: IdentifyCommand::Controller, + }, + &ranges, + ) + .await; + + let pref = unsafe { PhysicalRef::::map(page) }; + + fn cstr(s: &[u8]) -> &str { + let i = s + .iter() + .position(|x| *x == 0 || *x == b' ') + .unwrap_or_else(|| s.len()); + core::str::from_utf8(&s[..i]).unwrap() + } + + let sn = cstr(&pref.serial_number); + let mn = cstr(&pref.model_number); + + debugln!("Serial Number: {:?}", sn); + debugln!("Model Number: {:?}", mn); + + loop {} + } +} + +impl Device for NvmeController { + unsafe fn init(&'static self) -> Result<(), Error> { + let regs = self.regs.lock(); + + let min_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMIN)); + let max_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMAX)); + + if min_page_size > 4096 { + panic!(); + } + + if regs.CAP.read(CAP::CSS_IO_COMMANDS) != 1 { + panic!(); + } + + let timeout = Duration::from_millis(regs.CAP.read(CAP::TO) * 500); + debugln!("Worst-case timeout: {:?}", timeout); + + while regs.CSTS.matches_any(CSTS::RDY::SET) { + core::hint::spin_loop(); + } + + let queue_slots = 32; + if queue_slots > regs.CAP.read(CAP::MQES) + 1 { + todo!( + "queue_slots too big, max = {}", + regs.CAP.read(CAP::MQES) + 1 + ); + } + + // Setup the admin queue (index 0) + let doorbell_shift = regs.CAP.read(CAP::DSTRD) + 2; + let admin_sq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, false, 0) }; + let admin_cq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, true, 0) }; + let mut admin_q = + QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap(); + + regs.AQA + .modify(AQA::ASQS.val(queue_slots as u32 - 1) + AQA::ACQS.val(queue_slots as u32 - 1)); + regs.ASQ.set(admin_q.sq_physical_pointer().into_raw()); + regs.ACQ.set(admin_q.cq_physical_pointer().into_raw()); + + // Configure the controller + const IOSQES: u32 = size_of::().ilog2(); + const IOCQES: u32 = size_of::().ilog2(); + + regs.CC.modify( + CC::IOCQES.val(IOCQES) + + CC::IOSQES.val(IOSQES) + + CC::MPS.val(0) + + CC::CSS::NvmCommandSet, + ); + + // Enable the controller + regs.CC.modify(CC::ENABLE::SET); + + debugln!("Reset the controller"); + + while !regs.CSTS.matches_any(CSTS::RDY::SET + CSTS::CFS::SET) { + core::hint::spin_loop(); + } + + if regs.CSTS.matches_any(CSTS::CFS::SET) { + todo!("CFS set after reset!"); + } + + self.admin_q.init(IrqSafeSpinlock::new(admin_q)); + + // Schedule late_init task + runtime::spawn(self.late_init())?; + + Ok(()) + } + + fn display_name(&self) -> &'static str { + "NVM Express Controller" + } +} + +impl FromPciBus for NvmeController { + fn from_pci_bus(info: &PciDeviceInfo) -> Result { + let PciBaseAddress::Memory(bar0) = info.config_space.bar(0).unwrap() else { + panic!(); + }; + + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); + cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); + cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; + info.config_space.set_command(cmd.bits()); + + let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar0)) }?; + + // Disable the controller + regs.CC.modify(CC::ENABLE::CLEAR); + + Ok(Self { + regs: IrqSafeSpinlock::new(regs), + admin_q: OneTimeInit::new(), + }) + } +} diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs new file mode 100644 index 00000000..72b8c379 --- /dev/null +++ b/src/device/nvme/queue.rs @@ -0,0 +1,334 @@ +use core::{ + mem::size_of, + pin::Pin, + ptr::null_mut, + task::{Context, Poll}, +}; + +use abi::error::Error; +use bytemuck::{Pod, Zeroable}; +use futures_util::Future; +use static_assertions::const_assert; + +use crate::{ + arch::x86_64::mem::table::L3, + mem::{ + address::{AsPhysicalAddress, IntoRaw}, + phys, + pointer::PhysicalRefMut, + table::EntryLevel, + PhysicalAddress, + }, + proc, +}; + +use super::command::Command; + +#[derive(Zeroable, Pod, Clone, Copy, Debug)] +#[repr(C)] +pub struct PhysicalRegionPage(u64); + +// Bits: +// +// 16..32 - CID. Command identifier +// 14..16 - PSDT. PRP or SGL for data transfer. +// 0b00 - PRP used +// 0b01 - SGL used. Not implemented +// 0b10 - SGL used. Not implemented +// 0b11 - Reserved +// 10..14 - Reserved +// 08..10 - FUSE. Fused Operation +// 00..08 - OPC. Opcode +#[derive(Zeroable, Pod, Clone, Copy, Debug)] +#[repr(C)] +pub struct CommandDword0(u32); + +#[derive(Zeroable, Pod, Clone, Copy, Debug)] +#[repr(C)] +pub struct SubmissionQueueEntry { + pub command: CommandDword0, // 0 + pub nsid: u32, // 1 + _0: [u32; 2], // 2, 3 + pub metadata_pointer: u64, // 4, 5 + pub data_pointer: [PhysicalRegionPage; 2], // 6, 7, 8, 9 + pub command_specific: [u32; 6], // 10, 11, 12, 13, 14, 15 +} + +#[repr(C)] +pub struct CompletionQueueEntry { + dw: [u32; 4], +} + +pub struct Queue<'a, T> { + data: PhysicalRefMut<'a, [T]>, + + mask: usize, + head: usize, + tail: usize, + phase: bool, + + head_doorbell: *mut u32, + tail_doorbell: *mut u32, +} + +unsafe impl<'a, T> Sync for Queue<'a, T> {} +unsafe impl<'a, T> Send for Queue<'a, T> {} + +// TODO PageBox? +pub struct QueuePair<'a> { + base: PhysicalAddress, + page_count: usize, + + sq: Queue<'a, SubmissionQueueEntry>, + cq: Queue<'a, CompletionQueueEntry>, +} + +const_assert!(size_of::().is_power_of_two()); + +impl PhysicalRegionPage { + pub const fn null() -> Self { + Self(0) + } + + pub const fn with_addr(addr: u64) -> Self { + Self(addr) + } +} + +impl CommandDword0 { + pub fn set_command_id(&mut self, id: u16) { + self.0 &= !(0xFFFF << 16); + self.0 |= (id as u32) << 16; + } + + pub fn set_opcode(&mut self, opcode: u8) { + self.0 &= !0xFF; + self.0 |= opcode as u32; + } +} + +impl CompletionQueueEntry { + pub fn phase(&self) -> bool { + self.dw[3] & (1 << 16) != 0 + } + + pub fn subqueue_id(&self) -> u32 { + self.dw[2] >> 16 + } + + pub fn subqueue_head(&self) -> usize { + (self.dw[2] & 0xFFFF) as _ + } +} + +impl<'a, T> Queue<'a, T> { + pub unsafe fn from_raw_parts( + base: PhysicalAddress, + len: usize, + head_doorbell: *mut u32, + tail_doorbell: *mut u32, + phase: bool, + ) -> Self { + // Submission queues have tail doorbells, completion queues have head doorbells + assert!( + (head_doorbell.is_null() && !tail_doorbell.is_null()) + || (!head_doorbell.is_null() && tail_doorbell.is_null()) + ); + + Self { + data: PhysicalRefMut::map_slice(base, len), + mask: len - 1, + head: 0, + tail: 0, + head_doorbell, + tail_doorbell, + phase, + } + } + + pub fn physical_pointer(&self) -> PhysicalAddress { + unsafe { self.data.as_physical_address() } + } + + pub fn enqueue(&mut self, item: T) -> usize { + let index = self.tail; + self.data[self.tail] = item; + self.phase ^= self.set_tail(self.next_index(self.tail)); + index + } + + pub fn at_head(&self, offset: usize) -> (&T, bool) { + let index = (self.head + offset) & self.mask; + let expected_phase = self.phase ^ (index < self.head); + (&self.data[index], expected_phase) + } + + pub fn take(&mut self, count: usize) { + let index = (self.head + count) & self.mask; + self.phase ^= self.set_head(index); + } + + pub fn take_until(&mut self, new_head: usize) { + self.phase ^= self.set_head(new_head); + } + + fn next_index(&self, index: usize) -> usize { + (index + 1) & self.mask + } + + fn set_tail(&mut self, new_tail: usize) -> bool { + let wrapped = new_tail < self.tail; + + self.tail = new_tail; + + if !self.tail_doorbell.is_null() { + unsafe { + self.tail_doorbell + .write_volatile(self.tail.try_into().unwrap()); + } + } + + wrapped + } + + fn set_head(&mut self, new_head: usize) -> bool { + let wrapped = new_head < self.head; + + self.head = new_head; + + if !self.head_doorbell.is_null() { + unsafe { + self.head_doorbell + .write_volatile(self.head.try_into().unwrap()); + } + } + + wrapped + } + + pub fn is_empty(&self) -> bool { + self.head == self.tail + } +} + +impl<'a> QueuePair<'a> { + pub fn new( + capacity: usize, + sq_doorbell: *mut u32, + cq_doorbell: *mut u32, + ) -> Result { + let sq_size = capacity * size_of::(); + let cq_size = capacity * size_of::(); + + let page_count = L3::page_count(sq_size) + L3::page_count(cq_size); + let base = phys::alloc_pages_contiguous(page_count)?; + + let sq_base = base; + let cq_base = base.add(L3::align_up(sq_size)); + + debugln!( + "Allocated queue pair: sq={:x?}, cq={:x?} ({} pages)", + sq_base..sq_base.add(sq_size), + cq_base..cq_base.add(cq_size), + page_count + ); + + let sq = unsafe { Queue::from_raw_parts(sq_base, capacity, null_mut(), sq_doorbell, true) }; + let cq = unsafe { Queue::from_raw_parts(cq_base, capacity, cq_doorbell, null_mut(), true) }; + + Ok(Self { + base, + page_count, + sq, + cq, + }) + } + + #[inline] + pub fn sq_physical_pointer(&self) -> PhysicalAddress { + self.sq.physical_pointer() + } + + #[inline] + pub fn cq_physical_pointer(&self) -> PhysicalAddress { + self.cq.physical_pointer() + } + + pub fn submit( + &mut self, + command: C, + ranges: &[PhysicalAddress], + ) -> Result<(), Error> { + let index = self.sq.tail; + let mut sqe = SubmissionQueueEntry::zeroed(); + + sqe.command.set_command_id(index.try_into().unwrap()); + if ranges.len() != 1 { + todo!(); + } + sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw()); + sqe.data_pointer[1] = PhysicalRegionPage::null(); + + command.fill_sqe(&mut sqe); + + self.sq.enqueue(sqe); + + Ok(()) + } + + pub fn advance_head(&mut self, new_head: usize) { + self.sq.take_until(new_head); + } + + pub fn process_completions(&mut self) -> usize { + let mut i = 0; + loop { + let (packet, expected_phase) = self.cq.at_head(i); + let packet_phase = packet.phase(); + + if packet_phase != expected_phase { + break; + } + + i += 1; + + let sqid = packet.subqueue_id(); + // TODO + assert_eq!(sqid, 0); + + let sqhd = packet.subqueue_head(); + self.advance_head(sqhd); + } + + if i != 0 { + self.cq.take(i); + } + + i + } + + pub async fn perform_request(&mut self, command: C, ranges: &[PhysicalAddress]) { + self.submit(command, ranges).unwrap(); + + while self.process_completions() == 0 { + core::hint::spin_loop(); + } + } + + pub fn wait_all_complete(&'a self, index: usize) -> impl Future + 'a { + struct F<'q> { + queue: &'q QueuePair<'q>, + index: usize, + } + + impl<'q> Future for F<'q> { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + todo!() + } + } + + F { queue: self, index } + } +} diff --git a/src/init.rs b/src/init.rs index 9579d0ed..be2265b3 100644 --- a/src/init.rs +++ b/src/init.rs @@ -25,6 +25,7 @@ fn setup_root() -> Result { /// initialization has finished. pub fn kinit() -> Result<(), Error> { infoln!("In main"); + loop {} #[cfg(feature = "fb_console")] { From 148acca561fdd1ebba47b099601312c3c4af055a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 8 Dec 2023 17:23:03 +0200 Subject: [PATCH 106/211] dev/block: better nvme completion await --- src/device/nvme/command.rs | 74 +++++++++--- src/device/nvme/mod.rs | 59 ++++----- src/device/nvme/queue.rs | 237 ++++++++++++++++++++++++++----------- 3 files changed, 249 insertions(+), 121 deletions(-) diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs index d963691e..893ee927 100644 --- a/src/device/nvme/command.rs +++ b/src/device/nvme/command.rs @@ -1,36 +1,78 @@ +use core::fmt::{self, Write}; + +use bytemuck::{Pod, Zeroable}; + use super::queue::SubmissionQueueEntry; pub trait Command { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry); } -#[derive(Clone, Copy, Debug)] -#[repr(u32)] -pub enum IdentifyCommand { - Controller = 0x01, +pub trait Request: Command { + type Response; } +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct String { + data: [u8; N], +} + +#[derive(Clone, Copy, Debug)] +#[non_exhaustive] +#[repr(u8)] +pub enum ControllerType { + Reserved, + Io, + Discovery, + Administrative, +} + +#[derive(Clone, Copy, Debug)] #[repr(C)] pub struct IdentifyControllerResponse { pub pci_vid: u16, pub pci_ssvid: u16, - pub serial_number: [u8; 20], - pub model_number: [u8; 40], + pub serial_number: String<20>, + pub model_number: String<40>, pub firmware_rev: u64, + _0: [u8; 5], // 72..77 + pub mdts: u8, + pub cntlid: u16, + pub ver: u32, + _1: [u8; 12], // 84..96 + pub ctratt: u32, + _2: [u8; 11], // 100..111 + pub cntrltype: ControllerType, } -pub enum AdminCommand { - Identify { nsid: u32, cns: IdentifyCommand }, +#[derive(Clone, Copy, Debug)] +pub struct IdentifyControllerRequest { + pub nsid: u32, } -impl Command for AdminCommand { +impl Command for IdentifyControllerRequest { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { - match self { - AdminCommand::Identify { nsid, cns } => { - sqe.command.set_opcode(0x06); - sqe.nsid = *nsid; - sqe.command_specific[0] = *cns as u32; - } - } + sqe.command.set_opcode(0x06); + sqe.command_specific[0] = 0x01; + sqe.nsid = self.nsid; + } +} + +impl Request for IdentifyControllerRequest { + type Response = IdentifyControllerResponse; +} + +impl fmt::Debug for String { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + for ch in self.data { + if ch == b' ' || ch == 0 { + break; + } + f.write_char(ch as _)?; + } + f.write_char('"')?; + Ok(()) } } diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index 3adcd165..f76cae88 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -18,7 +18,7 @@ use crate::{ device::{ bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace}, nvme::{ - command::{AdminCommand, IdentifyCommand}, + command::IdentifyControllerRequest, queue::{CompletionQueueEntry, SubmissionQueueEntry}, }, }, @@ -108,7 +108,7 @@ register_structs! { pub struct NvmeController { regs: IrqSafeSpinlock>, - admin_q: OneTimeInit>>, + admin_q: OneTimeInit>, } impl Regs { @@ -121,41 +121,30 @@ impl Regs { impl NvmeController { async fn late_init(&'static self) { - let mut admin_q = self.admin_q.get().lock(); + runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); - // Identify - let page = phys::alloc_page().unwrap(); - let mut ranges = [page]; + let admin_q = self.admin_q.get(); - debugln!("output range = {:#x}", ranges[0]); - admin_q - .perform_request( - AdminCommand::Identify { - nsid: 0, - cns: IdentifyCommand::Controller, - }, - &ranges, - ) - .await; + let response = admin_q + .request(IdentifyControllerRequest { nsid: 0 }) + .unwrap() + .await + .unwrap(); - let pref = unsafe { PhysicalRef::::map(page) }; - - fn cstr(s: &[u8]) -> &str { - let i = s - .iter() - .position(|x| *x == 0 || *x == b' ') - .unwrap_or_else(|| s.len()); - core::str::from_utf8(&s[..i]).unwrap() - } - - let sn = cstr(&pref.serial_number); - let mn = cstr(&pref.model_number); - - debugln!("Serial Number: {:?}", sn); - debugln!("Model Number: {:?}", mn); + infoln!("Model: {:#?}", response.model_number); + infoln!("Serial: {:#?}", response.serial_number); + infoln!("Type: {:?}", response.cntrltype); loop {} } + + // TODO MSI(-X) or IRQ (ACPI currently broken) support for PCIe-based NVMe + async fn poll_task(&'static self) { + loop { + self.admin_q.get().process_completions(); + runtime::sleep(Duration::from_millis(100)).await; + } + } } impl Device for NvmeController { @@ -169,10 +158,6 @@ impl Device for NvmeController { panic!(); } - if regs.CAP.read(CAP::CSS_IO_COMMANDS) != 1 { - panic!(); - } - let timeout = Duration::from_millis(regs.CAP.read(CAP::TO) * 500); debugln!("Worst-case timeout: {:?}", timeout); @@ -192,7 +177,7 @@ impl Device for NvmeController { let doorbell_shift = regs.CAP.read(CAP::DSTRD) + 2; let admin_sq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, false, 0) }; let admin_cq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, true, 0) }; - let mut admin_q = + let admin_q = QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap(); regs.AQA @@ -224,7 +209,7 @@ impl Device for NvmeController { todo!("CFS set after reset!"); } - self.admin_q.init(IrqSafeSpinlock::new(admin_q)); + self.admin_q.init(admin_q); // Schedule late_init task runtime::spawn(self.late_init())?; diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 72b8c379..06c7ec67 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -1,13 +1,19 @@ use core::{ mem::size_of, + ops::DerefMut, pin::Pin, ptr::null_mut, task::{Context, Poll}, }; use abi::error::Error; +use alloc::{ + collections::{BTreeMap, BTreeSet}, + vec::Vec, +}; use bytemuck::{Pod, Zeroable}; use futures_util::Future; +use kernel_util::sync::IrqSafeSpinlock; use static_assertions::const_assert; use crate::{ @@ -20,9 +26,10 @@ use crate::{ PhysicalAddress, }, proc, + task::runtime::QueueWaker, }; -use super::command::Command; +use super::command::{Command, Request}; #[derive(Zeroable, Pod, Clone, Copy, Debug)] #[repr(C)] @@ -54,6 +61,13 @@ pub struct SubmissionQueueEntry { pub command_specific: [u32; 6], // 10, 11, 12, 13, 14, 15 } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct CommandError { + sct: u8, + sc: u8, +} + +#[derive(Zeroable, Pod, Clone, Copy, Debug)] #[repr(C)] pub struct CompletionQueueEntry { dw: [u32; 4], @@ -71,16 +85,25 @@ pub struct Queue<'a, T> { tail_doorbell: *mut u32, } -unsafe impl<'a, T> Sync for Queue<'a, T> {} -unsafe impl<'a, T> Send for Queue<'a, T> {} +struct Inner<'a> { + sq: Queue<'a, SubmissionQueueEntry>, + cq: Queue<'a, CompletionQueueEntry>, + + completed: BTreeMap, + pending: BTreeSet, +} // TODO PageBox? pub struct QueuePair<'a> { base: PhysicalAddress, page_count: usize, - sq: Queue<'a, SubmissionQueueEntry>, - cq: Queue<'a, CompletionQueueEntry>, + sq_base: PhysicalAddress, + cq_base: PhysicalAddress, + + completion_notify: QueueWaker, + + inner: IrqSafeSpinlock>, } const_assert!(size_of::().is_power_of_two()); @@ -96,9 +119,10 @@ impl PhysicalRegionPage { } impl CommandDword0 { - pub fn set_command_id(&mut self, id: u16) { + pub fn set_command_id(&mut self, id: u32) { + debug_assert!(id & 0xFFFF0000 == 0); self.0 &= !(0xFFFF << 16); - self.0 |= (id as u32) << 16; + self.0 |= id << 16; } pub fn set_opcode(&mut self, opcode: u8) { @@ -112,13 +136,30 @@ impl CompletionQueueEntry { self.dw[3] & (1 << 16) != 0 } - pub fn subqueue_id(&self) -> u32 { + pub fn sub_queue_id(&self) -> u32 { self.dw[2] >> 16 } - pub fn subqueue_head(&self) -> usize { + pub fn sub_queue_head(&self) -> usize { (self.dw[2] & 0xFFFF) as _ } + + pub fn command_id(&self) -> u32 { + self.dw[3] & 0xFFFF + } + + pub fn error(&self) -> Option { + let status = (self.dw[3] >> 17) as u16; + + if status != 0 { + Some(CommandError { + sct: ((status >> 8) & 0x7) as u8, + sc: status as u8, + }) + } else { + None + } + } } impl<'a, T> Queue<'a, T> { @@ -146,10 +187,6 @@ impl<'a, T> Queue<'a, T> { } } - pub fn physical_pointer(&self) -> PhysicalAddress { - unsafe { self.data.as_physical_address() } - } - pub fn enqueue(&mut self, item: T) -> usize { let index = self.tail; self.data[self.tail] = item; @@ -236,99 +273,163 @@ impl<'a> QueuePair<'a> { let sq = unsafe { Queue::from_raw_parts(sq_base, capacity, null_mut(), sq_doorbell, true) }; let cq = unsafe { Queue::from_raw_parts(cq_base, capacity, cq_doorbell, null_mut(), true) }; - Ok(Self { - base, - page_count, + let inner = IrqSafeSpinlock::new(Inner { sq, cq, + pending: BTreeSet::new(), + completed: BTreeMap::new(), + }); + + Ok(Self { + completion_notify: QueueWaker::new(), + + base, + page_count, + sq_base, + cq_base, + inner, }) } #[inline] pub fn sq_physical_pointer(&self) -> PhysicalAddress { - self.sq.physical_pointer() + self.sq_base } #[inline] pub fn cq_physical_pointer(&self) -> PhysicalAddress { - self.cq.physical_pointer() + self.cq_base + } + + pub fn wait_for_completion<'r, T: Unpin + 'r>( + &'r self, + command_id: u32, + result: T, + ) -> impl Future> + 'r + where + 'r: 'a, + { + struct Fut<'r, R: Unpin + 'r> { + this: &'r QueuePair<'r>, + response: Option, + command_id: u32, + }; + + impl<'r, R: Unpin + 'r> Future for Fut<'r, R> { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.this.completion_notify.register(cx.waker()); + let mut inner = self.this.inner.lock(); + + if let Some(entry) = inner.completed.remove(&self.command_id) { + self.this.completion_notify.remove(cx.waker()); + + let result = if let Some(error) = entry.error() { + Err(error) + } else { + Ok(self.response.take().unwrap()) + }; + + Poll::Ready(result) + } else { + Poll::Pending + } + } + } + + Fut { + this: self, + response: Some(result), + command_id, + } } pub fn submit( - &mut self, - command: C, + &self, + cmd: C, ranges: &[PhysicalAddress], - ) -> Result<(), Error> { - let index = self.sq.tail; + set_pending: bool, + ) -> Result { + let mut inner = self.inner.lock(); let mut sqe = SubmissionQueueEntry::zeroed(); - sqe.command.set_command_id(index.try_into().unwrap()); - if ranges.len() != 1 { - todo!(); + cmd.fill_sqe(&mut sqe); + + let command_id = inner.sq.tail.try_into().unwrap(); + sqe.command.set_command_id(command_id); + + if set_pending { + inner.pending.insert(command_id); } - sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw()); - sqe.data_pointer[1] = PhysicalRegionPage::null(); - command.fill_sqe(&mut sqe); + inner.sq.enqueue(sqe); - self.sq.enqueue(sqe); - - Ok(()) + Ok(command_id) } - pub fn advance_head(&mut self, new_head: usize) { - self.sq.take_until(new_head); + pub fn request<'r, R: Request>( + &'r self, + req: R, + ) -> Result, CommandError>>, Error> + where + R::Response: 'r, + 'r: 'a, + { + assert!(size_of::() < 0x1000); + + let page = phys::alloc_page()?; + // TODO PageBox + let response = unsafe { PhysicalRefMut::map(page) }; + + let command_id = self.submit(req, &[page], true)?; + Ok(self.wait_for_completion(command_id, response)) } - pub fn process_completions(&mut self) -> usize { - let mut i = 0; + pub fn process_completions(&self) -> usize { + let mut inner = self.inner.lock(); + let mut n = 0; + + let mut completion_list = Vec::new(); + loop { - let (packet, expected_phase) = self.cq.at_head(i); - let packet_phase = packet.phase(); + let (cmp, expected_phase) = inner.cq.at_head(n); + let cmp_phase = cmp.phase(); - if packet_phase != expected_phase { + if cmp_phase != expected_phase { break; } - i += 1; + n += 1; - let sqid = packet.subqueue_id(); - // TODO - assert_eq!(sqid, 0); + let sub_queue_id = cmp.sub_queue_id(); + // TODO support queues other than admin q + assert_eq!(sub_queue_id, 0); - let sqhd = packet.subqueue_head(); - self.advance_head(sqhd); + let sub_queue_head = cmp.sub_queue_head(); + let cmp = *cmp; + inner.sq.take_until(sub_queue_head); + + completion_list.push(cmp); } - if i != 0 { - self.cq.take(i); + if n != 0 { + inner.cq.take(n); } - i - } + for cmp in completion_list { + let command_id = cmp.command_id(); - pub async fn perform_request(&mut self, command: C, ranges: &[PhysicalAddress]) { - self.submit(command, ranges).unwrap(); - - while self.process_completions() == 0 { - core::hint::spin_loop(); - } - } - - pub fn wait_all_complete(&'a self, index: usize) -> impl Future + 'a { - struct F<'q> { - queue: &'q QueuePair<'q>, - index: usize, - } - - impl<'q> Future for F<'q> { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - todo!() + if inner.pending.remove(&command_id) { + debugln!("Insert completion: {}", command_id); + inner.completed.insert(command_id, cmp); } } - F { queue: self, index } + if n != 0 { + self.completion_notify.wake_all(); + } + + n } } From 2770281213a75e8df83f05743dac86b46e391da0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 8 Dec 2023 22:49:33 +0200 Subject: [PATCH 107/211] dev/block: create io cq/sq pair for nvme --- src/device/nvme/command.rs | 75 +++++++++++++++++++++++++++++++++--- src/device/nvme/mod.rs | 78 ++++++++++++++++++++++++++++---------- src/device/nvme/queue.rs | 35 ++++++++++++----- 3 files changed, 154 insertions(+), 34 deletions(-) diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs index 893ee927..9f6231f4 100644 --- a/src/device/nvme/command.rs +++ b/src/device/nvme/command.rs @@ -1,6 +1,9 @@ use core::fmt::{self, Write}; -use bytemuck::{Pod, Zeroable}; +use crate::{ + device::nvme::queue::PhysicalRegionPage, + mem::{address::IntoRaw, PhysicalAddress}, +}; use super::queue::SubmissionQueueEntry; @@ -28,6 +31,35 @@ pub enum ControllerType { Administrative, } +// Requests + +#[derive(Clone, Copy, Debug)] +pub enum SetFeatureRequest { + NumberOfQueues(u32, u32), +} + +#[derive(Clone, Copy, Debug)] +pub struct IdentifyControllerRequest { + pub nsid: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct CreateIoCompletionQueue { + pub id: u32, + pub size: usize, + pub data: PhysicalAddress, +} + +#[derive(Clone, Copy, Debug)] +pub struct CreateIoSubmissionQueue { + pub id: u32, + pub cq_id: u32, + pub size: usize, + pub data: PhysicalAddress, +} + +// Replies + #[derive(Clone, Copy, Debug)] #[repr(C)] pub struct IdentifyControllerResponse { @@ -46,11 +78,6 @@ pub struct IdentifyControllerResponse { pub cntrltype: ControllerType, } -#[derive(Clone, Copy, Debug)] -pub struct IdentifyControllerRequest { - pub nsid: u32, -} - impl Command for IdentifyControllerRequest { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { sqe.command.set_opcode(0x06); @@ -63,6 +90,42 @@ impl Request for IdentifyControllerRequest { type Response = IdentifyControllerResponse; } +impl Command for SetFeatureRequest { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + sqe.command.set_opcode(0x09); + + match self { + Self::NumberOfQueues(cq, sq) => { + let dw11 = (cq << 16) | sq; + + sqe.command_specific[0] = 0x07; + sqe.command_specific[1] = dw11; + } + } + } +} + +impl Command for CreateIoCompletionQueue { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + sqe.command.set_opcode(0x05); + sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw()); + sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id; + sqe.command_specific[1] = 1; + // TODO ENABLE IRQS HERE + sqe.command_specific[2] = 0; + } +} + +impl Command for CreateIoSubmissionQueue { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + sqe.command.set_opcode(0x01); + sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw()); + sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id; + // Medium priority + sqe.command_specific[1] = (self.cq_id << 16) | 1; + } +} + impl fmt::Debug for String { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_char('"')?; diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index f76cae88..67548164 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -1,11 +1,10 @@ #![allow(missing_docs)] -use core::{mem::size_of, ptr::null_mut, time::Duration}; +use core::{mem::size_of, time::Duration}; use abi::error::Error; -use bytemuck::{Pod, Zeroable}; +use alloc::vec::Vec; use device_api::Device; use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; -use static_assertions::{const_assert, const_assert_eq}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -13,8 +12,6 @@ use tock_registers::{ }; use crate::{ - arch::{x86_64::mem::table::L3, Architecture, ARCHITECTURE}, - debug, device::{ bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace}, nvme::{ @@ -23,17 +20,17 @@ use crate::{ }, }, mem::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw}, + address::{FromRaw, IntoRaw}, device::DeviceMemoryIo, - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - table::EntryLevel, PhysicalAddress, }, task::runtime, }; -use self::queue::QueuePair; +use self::{ + command::{CreateIoCompletionQueue, CreateIoSubmissionQueue, SetFeatureRequest}, + queue::QueuePair, +}; use super::bus::pci::{FromPciBus, PciDeviceInfo}; @@ -109,10 +106,13 @@ register_structs! { pub struct NvmeController { regs: IrqSafeSpinlock>, admin_q: OneTimeInit>, + ioqs: OneTimeInit>>, + + doorbell_shift: usize, } impl Regs { - unsafe fn doorbell_ptr(&self, shift: u64, completion: bool, queue_index: usize) -> *mut u32 { + unsafe fn doorbell_ptr(&self, shift: usize, completion: bool, queue_index: usize) -> *mut u32 { let doorbell_base = (self as *const Regs as *mut Regs).addr() + 0x1000; let offset = (queue_index << shift) + completion as usize * 4; (doorbell_base + offset) as *mut u32 @@ -121,19 +121,49 @@ impl Regs { impl NvmeController { async fn late_init(&'static self) { - runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); + // let ioq = QueuePair::new(capacity, sq_doorbell, cq_doorbell).unwrap(); + runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); let admin_q = self.admin_q.get(); - let response = admin_q + // Request a CQ/SQ pair for I/O + admin_q + .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1)) + .unwrap() + .await + .unwrap(); + + // Allocate the queue + let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) }; + let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).unwrap(); + + // Identify the controller + let identify = admin_q .request(IdentifyControllerRequest { nsid: 0 }) .unwrap() .await .unwrap(); - infoln!("Model: {:#?}", response.model_number); - infoln!("Serial: {:#?}", response.serial_number); - infoln!("Type: {:?}", response.cntrltype); + // Create the queue on the device side + admin_q + .request_no_data(CreateIoCompletionQueue { + id: 1, + size: 32, + data: io_q.cq_physical_pointer(), + }) + .unwrap() + .await + .unwrap(); + admin_q + .request_no_data(CreateIoSubmissionQueue { + id: 1, + cq_id: 1, + size: 32, + data: io_q.sq_physical_pointer(), + }) + .unwrap() + .await + .unwrap(); loop {} } @@ -145,6 +175,13 @@ impl NvmeController { runtime::sleep(Duration::from_millis(100)).await; } } + + unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) { + let regs = self.regs.lock(); + let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx); + let cq_ptr = regs.doorbell_ptr(self.doorbell_shift, true, idx); + (sq_ptr, cq_ptr) + } } impl Device for NvmeController { @@ -174,9 +211,8 @@ impl Device for NvmeController { } // Setup the admin queue (index 0) - let doorbell_shift = regs.CAP.read(CAP::DSTRD) + 2; - let admin_sq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, false, 0) }; - let admin_cq_doorbell = unsafe { regs.doorbell_ptr(doorbell_shift, true, 0) }; + let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) }; + let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) }; let admin_q = QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap(); @@ -238,9 +274,13 @@ impl FromPciBus for NvmeController { // Disable the controller regs.CC.modify(CC::ENABLE::CLEAR); + let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 2; + Ok(Self { regs: IrqSafeSpinlock::new(regs), admin_q: OneTimeInit::new(), + ioqs: OneTimeInit::new(), + doorbell_shift, }) } } diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 06c7ec67..7cece0c1 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -18,14 +18,7 @@ use static_assertions::const_assert; use crate::{ arch::x86_64::mem::table::L3, - mem::{ - address::{AsPhysicalAddress, IntoRaw}, - phys, - pointer::PhysicalRefMut, - table::EntryLevel, - PhysicalAddress, - }, - proc, + mem::{address::IntoRaw, phys, pointer::PhysicalRefMut, table::EntryLevel, PhysicalAddress}, task::runtime::QueueWaker, }; @@ -313,7 +306,7 @@ impl<'a> QueuePair<'a> { this: &'r QueuePair<'r>, response: Option, command_id: u32, - }; + } impl<'r, R: Unpin + 'r> Future for Fut<'r, R> { type Output = Result; @@ -354,6 +347,18 @@ impl<'a> QueuePair<'a> { let mut inner = self.inner.lock(); let mut sqe = SubmissionQueueEntry::zeroed(); + match ranges.len() { + 1 => { + sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw()); + sqe.data_pointer[1] = PhysicalRegionPage::null(); + } + 0 => { + sqe.data_pointer[0] = PhysicalRegionPage::null(); + sqe.data_pointer[1] = PhysicalRegionPage::null(); + } + _ => todo!(), + } + cmd.fill_sqe(&mut sqe); let command_id = inner.sq.tail.try_into().unwrap(); @@ -368,6 +373,17 @@ impl<'a> QueuePair<'a> { Ok(command_id) } + pub fn request_no_data<'r, C: Command>( + &'r self, + req: C, + ) -> Result> + 'r, Error> + where + 'r: 'a, + { + let command_id = self.submit(req, &[], true)?; + Ok(self.wait_for_completion(command_id, ())) + } + pub fn request<'r, R: Request>( &'r self, req: R, @@ -376,6 +392,7 @@ impl<'a> QueuePair<'a> { R::Response: 'r, 'r: 'a, { + assert_ne!(size_of::(), 0); assert!(size_of::() < 0x1000); let page = phys::alloc_page()?; From f166968e5755aa8f185ffe1bdbe0c32453349ff0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 8 Dec 2023 23:19:12 +0200 Subject: [PATCH 108/211] task: better async error handling --- src/device/nvme/error.rs | 15 +++++++++++++++ src/device/nvme/mod.rs | 26 +++++++++----------------- src/device/nvme/queue.rs | 29 +++++++++++++++-------------- src/task/runtime/executor.rs | 6 ++++-- src/task/runtime/task.rs | 27 +++++++++++++++++++++++++-- 5 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 src/device/nvme/error.rs diff --git a/src/device/nvme/error.rs b/src/device/nvme/error.rs new file mode 100644 index 00000000..86553300 --- /dev/null +++ b/src/device/nvme/error.rs @@ -0,0 +1,15 @@ +use abi::error::Error; + +use super::queue::CommandError; + +#[derive(Debug)] +pub enum NvmeError { + MemoryError(Error), + CommandError(CommandError), +} + +impl From for NvmeError { + fn from(value: CommandError) -> Self { + Self::CommandError(value) + } +} diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index 67548164..f11928da 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -29,12 +29,14 @@ use crate::{ use self::{ command::{CreateIoCompletionQueue, CreateIoSubmissionQueue, SetFeatureRequest}, + error::NvmeError, queue::QueuePair, }; use super::bus::pci::{FromPciBus, PciDeviceInfo}; mod command; +mod error; mod queue; register_bitfields! { @@ -120,29 +122,23 @@ impl Regs { } impl NvmeController { - async fn late_init(&'static self) { - // let ioq = QueuePair::new(capacity, sq_doorbell, cq_doorbell).unwrap(); - + async fn late_init(&'static self) -> Result<(), NvmeError> { runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); let admin_q = self.admin_q.get(); // Request a CQ/SQ pair for I/O admin_q .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1)) - .unwrap() - .await - .unwrap(); + .await?; // Allocate the queue let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) }; - let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).unwrap(); + let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; // Identify the controller let identify = admin_q - .request(IdentifyControllerRequest { nsid: 0 }) - .unwrap() - .await - .unwrap(); + .request(IdentifyControllerRequest { nsid: 0 })? + .await?; // Create the queue on the device side admin_q @@ -151,9 +147,7 @@ impl NvmeController { size: 32, data: io_q.cq_physical_pointer(), }) - .unwrap() - .await - .unwrap(); + .await?; admin_q .request_no_data(CreateIoSubmissionQueue { id: 1, @@ -161,9 +155,7 @@ impl NvmeController { size: 32, data: io_q.sq_physical_pointer(), }) - .unwrap() - .await - .unwrap(); + .await?; loop {} } diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 7cece0c1..afab4c02 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -22,7 +22,10 @@ use crate::{ task::runtime::QueueWaker, }; -use super::command::{Command, Request}; +use super::{ + command::{Command, Request}, + error::NvmeError, +}; #[derive(Zeroable, Pod, Clone, Copy, Debug)] #[repr(C)] @@ -338,12 +341,7 @@ impl<'a> QueuePair<'a> { } } - pub fn submit( - &self, - cmd: C, - ranges: &[PhysicalAddress], - set_pending: bool, - ) -> Result { + pub fn submit(&self, cmd: C, ranges: &[PhysicalAddress], set_pending: bool) -> u32 { let mut inner = self.inner.lock(); let mut sqe = SubmissionQueueEntry::zeroed(); @@ -370,24 +368,27 @@ impl<'a> QueuePair<'a> { inner.sq.enqueue(sqe); - Ok(command_id) + command_id } pub fn request_no_data<'r, C: Command>( &'r self, req: C, - ) -> Result> + 'r, Error> + ) -> impl Future> + 'r where 'r: 'a, { - let command_id = self.submit(req, &[], true)?; - Ok(self.wait_for_completion(command_id, ())) + let command_id = self.submit(req, &[], true); + self.wait_for_completion(command_id, ()) } pub fn request<'r, R: Request>( &'r self, req: R, - ) -> Result, CommandError>>, Error> + ) -> Result< + impl Future, CommandError>>, + NvmeError, + > where R::Response: 'r, 'r: 'a, @@ -395,11 +396,11 @@ impl<'a> QueuePair<'a> { assert_ne!(size_of::(), 0); assert!(size_of::() < 0x1000); - let page = phys::alloc_page()?; + let page = phys::alloc_page().map_err(NvmeError::MemoryError)?; // TODO PageBox let response = unsafe { PhysicalRefMut::map(page) }; - let command_id = self.submit(req, &[page], true)?; + let command_id = self.submit(req, &[page], true); Ok(self.wait_for_completion(command_id, response)) } diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs index 53f79473..5dab913c 100644 --- a/src/task/runtime/executor.rs +++ b/src/task/runtime/executor.rs @@ -7,7 +7,7 @@ use futures_util::{task::waker_ref, Future}; use crate::task::{spawn_kernel_closure, thread::Thread}; use super::{ - task::Task, + task::{Task, Termination}, task_queue::{self}, }; @@ -38,7 +38,9 @@ pub fn spawn_async_worker(index: usize) -> Result<(), Error> { } /// Creates a new task for the [Future] and queues it for execution in background -pub fn spawn + Send + 'static>(future: F) -> Result<(), Error> { +pub fn spawn + Send + 'static>( + future: F, +) -> Result<(), Error> { enqueue(Task::new(future)) } diff --git a/src/task/runtime/task.rs b/src/task/runtime/task.rs index 7a064c32..f28e034d 100644 --- a/src/task/runtime/task.rs +++ b/src/task/runtime/task.rs @@ -1,9 +1,15 @@ +use core::fmt; + use alloc::sync::Arc; use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt}; use kernel_util::sync::IrqSafeSpinlock; use super::executor; +pub trait Termination { + fn print(&self); +} + pub struct Task { pub(super) future: IrqSafeSpinlock>>, } @@ -15,8 +21,25 @@ impl ArcWake for Task { } impl Task { - pub fn new + Send + 'static>(future: F) -> Arc { - let future = IrqSafeSpinlock::new(Some(future.boxed())); + pub fn new + Send + 'static>(future: F) -> Arc { + let future = IrqSafeSpinlock::new(Some( + async move { + future.await.print(); + } + .boxed(), + )); Arc::new(Self { future }) } } + +impl Termination for () { + fn print(&self) {} +} + +impl Termination for Result { + fn print(&self) { + if let Err(error) = self { + errorln!("A task finished with an error: {:?}", error); + } + } +} From 506476e9c352d21c68f715f3fef6001f6e0ae0b3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 9 Dec 2023 15:27:09 +0200 Subject: [PATCH 109/211] dev/pci: MSI-X capability and APIC support for it --- lib/device-api/src/interrupt.rs | 17 +++++ src/arch/mod.rs | 18 +++++- src/arch/x86_64/apic/ioapic.rs | 4 +- src/arch/x86_64/apic/local.rs | 74 ++++++++++++++++++---- src/arch/x86_64/apic/mod.rs | 40 +++++++++++- src/arch/x86_64/apic/vectors.S | 59 ++++++++++++++++- src/arch/x86_64/mod.rs | 10 ++- src/device/bus/pci/capability.rs | 105 +++++++++++++++++++++++++++++++ src/device/bus/pci/mod.rs | 36 +++++++++++ src/device/bus/pci/space/mod.rs | 72 ++++++++++++++++++++- src/device/nvme/command.rs | 7 ++- src/device/nvme/mod.rs | 65 ++++++++++++++----- src/device/nvme/queue.rs | 5 +- src/mem/device.rs | 58 ++++++++++++++++- 14 files changed, 525 insertions(+), 45 deletions(-) create mode 100644 src/device/bus/pci/capability.rs diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index e8d9cf2a..f64e026e 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -33,10 +33,23 @@ pub struct IrqOptions { pub trigger: IrqTrigger, } +#[derive(Clone, Copy, Debug)] +pub struct MsiInfo { + pub address: usize, + pub value: u32, + pub vector: usize, +} + pub trait InterruptTable { fn handler(&self, index: usize) -> Option<&'static dyn InterruptHandler>; } +pub trait MessageInterruptController { + fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result; + + fn handle_msi(&self, #[allow(unused)] vector: usize) {} +} + pub trait ExternalInterruptController { type IrqNumber; @@ -80,6 +93,10 @@ pub trait InterruptHandler: Device { fn handle_irq(&self) -> bool; } +pub trait MsiHandler: Device { + fn handle_msi(&self, vector: usize) -> bool; +} + pub struct FixedInterruptTable { entries: [Option<&'static dyn InterruptHandler>; SIZE], } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index fe5241e6..c021f3be 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -21,7 +21,10 @@ macro_rules! absolute_address { use cfg_if::cfg_if; use device_api::{ - interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, + interrupt::{ + ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController, + MessageInterruptController, + }, timer::MonotonicTimestampProviderDevice, ResetDevice, }; @@ -157,6 +160,14 @@ pub trait Architecture { Err(Error::NotImplemented) } + /// Adds a message-signalled interrupt (MSI/MSI-X) controller to the system + fn register_message_interrupt_controller( + &self, + intc: &'static dyn MessageInterruptController, + ) -> Result<(), Error> { + Err(Error::NotImplemented) + } + /// Adds a monotonic timer to the system fn register_monotonic_timer( &self, @@ -185,6 +196,11 @@ pub trait Architecture { unimplemented!() } + /// Returns the MSI/MSI-X-capable interrupt controller + fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { + unimplemented!() + } + /// Returns the monotonic timer fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { unimplemented!() diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 9e5c39af..a6be3d91 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -20,7 +20,7 @@ use crate::{ mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, }; -use super::{APIC_EXTERNAL_OFFSET, MAX_EXTERNAL_VECTORS}; +use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS}; // IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET; @@ -64,7 +64,7 @@ pub struct IoApic { inner: IrqSafeSpinlock, isa_redirections: [Option; 16], - table: IrqSafeSpinlock>, + table: IrqSafeSpinlock>, } impl Regs { diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 3cb014e0..6158a4c6 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -1,11 +1,16 @@ //! x86-64 Local APIC driver implementation use core::sync::atomic::Ordering; +use abi::error::Error; +use alloc::vec::Vec; use device_api::{ - interrupt::{IpiDeliveryTarget, LocalInterruptController}, + interrupt::{ + IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, MsiHandler, + MsiInfo, + }, Device, }; -use kernel_util::util::OneTimeInit; +use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -14,10 +19,16 @@ use tock_registers::{ use crate::{ arch::{ - x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT}, + x86_64::{ + apic::APIC_MSI_OFFSET, mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT, + }, CpuMessage, }, - mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, + mem::{ + address::{FromRaw, IntoRaw}, + device::DeviceMemoryIo, + PhysicalAddress, + }, task::Cpu, }; @@ -131,6 +142,7 @@ register_structs! { /// Per-processor local APIC interface pub struct LocalApic { regs: DeviceMemoryIo<'static, Regs>, + msi_vectors: IrqSafeSpinlock>, } unsafe impl Send for LocalApic {} @@ -142,14 +154,51 @@ impl Device for LocalApic { } } +impl MessageInterruptController for LocalApic { + fn handle_msi(&self, vector: usize) { + // TODO this is ugly + let mut i = 0; + + loop { + let table = self.msi_vectors.lock(); + let Some(&handler) = table.get(i) else { + break; + }; + drop(table); + + if handler.handle_msi(vector) { + break; + } + + i += 1; + } + } + + fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result { + // TODO only 1 ISR vector allocated for MSIs + let vector = 0; + let mut table = self.msi_vectors.lock(); + + table.push(handler); + + // TODO magic numbers + let apic_vector = 32 + APIC_MSI_OFFSET + vector; + + let value = apic_vector; + let address = Self::base(); + + Ok(MsiInfo { + address: address.into_raw(), + value, + vector: vector as _, + }) + } +} + impl LocalInterruptController for LocalApic { type IpiMessage = CpuMessage; - fn send_ipi( - &self, - target: device_api::interrupt::IpiDeliveryTarget, - msg: Self::IpiMessage, - ) -> Result<(), abi::error::Error> { + fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> { while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { core::hint::spin_loop(); } @@ -178,7 +227,7 @@ impl LocalInterruptController for LocalApic { } } - unsafe fn init_ap(&self) -> Result<(), abi::error::Error> { + unsafe fn init_ap(&self) -> Result<(), Error> { todo!() } } @@ -228,7 +277,10 @@ impl LocalApic { LocalVectorEntry::Mask::Masked + LocalVectorEntry::Vector.val(APIC_LINT1_VECTOR + 32), ); - Self { regs } + Self { + regs, + msi_vectors: IrqSafeSpinlock::new(Vec::new()), + } } /// Signals local APIC that we've handled the IRQ diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 29a03f15..c1c4f8cb 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -2,6 +2,8 @@ use core::arch::global_asm; +use static_assertions::{const_assert, const_assert_eq}; + use crate::{ arch::{x86_64::cpu::Cpu, Architecture}, task::thread::Thread, @@ -29,8 +31,17 @@ pub const APIC_IPI_VECTOR: u32 = 0x03; pub const APIC_SPURIOUS_VECTOR: u32 = 0xDF; /// Start of the I/O APIC IRQ range pub const APIC_EXTERNAL_OFFSET: u32 = 4; +/// Start of the MSI range +pub const APIC_MSI_OFFSET: u32 = APIC_EXTERNAL_OFFSET + MAX_EXTERNAL_VECTORS; /// Maximum number of APIC vectors allocated for handling IRQs from I/O APIC -pub const MAX_EXTERNAL_VECTORS: u32 = 16; +pub const MAX_EXTERNAL_VECTORS: u32 = APIC_SPURIOUS_VECTOR - APIC_EXTERNAL_OFFSET - MAX_MSI_VECTORS; +/// Number of I/O APIC IRQ vectors that are actually populated +pub const POPULATED_EXTERNAL_VECTORS: u32 = 16; +/// Maximum number of APIC vectors allocated for handling MSIs +pub const MAX_MSI_VECTORS: u32 = 16; + +const_assert!(POPULATED_EXTERNAL_VECTORS <= MAX_EXTERNAL_VECTORS); +const_assert_eq!(APIC_MSI_OFFSET + MAX_MSI_VECTORS, APIC_SPURIOUS_VECTOR); /// Fills the IDT with interrupt vectors for this APIC pub fn setup_vectors(idt: &mut [exception::Entry]) { @@ -49,6 +60,10 @@ pub fn setup_vectors(idt: &mut [exception::Entry]) { } unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { + if vector >= POPULATED_EXTERNAL_VECTORS as _ { + todo!("Got a weird IRQ with vector {}", vector); + } + let cpu = Cpu::local(); let frame = &mut *frame; @@ -62,6 +77,24 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { } } +unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) { + if vector != 0 { + todo!("Got a weird MSI with vector {}", vector); + } + + let cpu = Cpu::local(); + let frame = &mut *frame; + + ARCHITECTURE + .message_interrupt_controller() + .handle_msi(vector); + cpu.local_apic().clear_interrupt(); + + if let Some(thread) = Thread::get_current() { + thread.handle_pending_signals(frame); + } +} + unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { let frame = &mut *frame; let cpu = Cpu::local(); @@ -87,7 +120,12 @@ global_asm!( include_str!("vectors.S"), local_timer_irq_handler = sym local_timer_irq_handler, irq_handler = sym irq_handler, + msi_handler = sym msi_handler, ipi_handler = sym ipi_handler, dummy_irq_handler = sym dummy_irq_handler, + irq_vector_offset = const APIC_EXTERNAL_OFFSET, + irq_vector_count = const MAX_EXTERNAL_VECTORS, + msi_vector_offset = const APIC_MSI_OFFSET, + msi_vector_count = const MAX_MSI_VECTORS, options(att_syntax) ); diff --git a/src/arch/x86_64/apic/vectors.S b/src/arch/x86_64/apic/vectors.S index 6b90c713..2ff7a27b 100644 --- a/src/arch/x86_64/apic/vectors.S +++ b/src/arch/x86_64/apic/vectors.S @@ -123,6 +123,56 @@ irq_vector_\n: .endr .endm +.macro MSI_VECTOR, n +msi_vector_\n: + // %rsp + 0: %rip + // %rsp + 8: %cs + IRQ_SWAPGS_IF_NEEDED 8 + IRQ_SAVE_STATE + + // Force correct segment registers + mov $0x10, %ax + mov %ax, %ss + mov %ax, %ds + mov %ax, %es + + mov $\n, %rdi + mov %rbp, %rsi + call {msi_handler} + + IRQ_RESTORE_STATE + IRQ_SWAPGS_IF_NEEDED 8 + + iretq +.endm + +.macro MSI_VECTOR_ENTRY, n +.quad msi_vector_\n +.endm + +.macro MSI_VECTORS, start, end +.set i, 0 +.rept \end - \start + MSI_VECTOR %i + .set i, i+1 +.endr +.endm + +.macro MSI_VECTOR_ENTRIES, start, end +.set i, 0 +.rept \end - \start + MSI_VECTOR_ENTRY %i + .set i, i+1 +.endr +.endm + +.macro FILL_EMPTY_SPACE, start, end +.set i, 0 +.rept \end - \start +.quad dummy_vector +.endr +.endm + .section .text local_timer_vector: IRQ_SWAPGS_IF_NEEDED 8 @@ -149,7 +199,8 @@ dummy_vector: call {dummy_irq_handler} jmp . -IRQ_VECTORS 4, 255 +IRQ_VECTORS {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count} +MSI_VECTORS {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count} .section .rodata // 224 vectors: 256 - 32 (exceptions) @@ -163,8 +214,10 @@ __x86_64_apic_vectors: .quad dummy_vector // IPI vector: 3 .quad ipi_vector - // Regular IRQ vectors: 4..=222 - IRQ_VECTOR_ENTRIES 4, 223 + // Regular IRQ vectors: 4..207 + IRQ_VECTOR_ENTRIES {irq_vector_offset}, {irq_vector_offset} + {irq_vector_count} + // MSI vectors: 207..223 + MSI_VECTOR_ENTRIES {msi_vector_offset}, {msi_vector_offset} + {msi_vector_count} // Spurious interrupt vector: 223 .quad dummy_vector .size __x86_64_apic_vectors, . - __x86_64_apic_vectors diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 14b58c2e..0ca37092 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -5,8 +5,10 @@ use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use device_api::{ - input::KeyboardProducer, interrupt::ExternalInterruptController, - timer::MonotonicTimestampProviderDevice, Device, + input::KeyboardProducer, + interrupt::{ExternalInterruptController, MessageInterruptController}, + timer::MonotonicTimestampProviderDevice, + Device, }; use git_version::git_version; use kernel_util::{sync::SpinFence, util::OneTimeInit}; @@ -281,6 +283,10 @@ impl Architecture for X86_64 { self.ioapic.get() } + fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { + Cpu::local().local_apic() + } + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { self.timer.get() } diff --git a/src/device/bus/pci/capability.rs b/src/device/bus/pci/capability.rs new file mode 100644 index 00000000..73132c90 --- /dev/null +++ b/src/device/bus/pci/capability.rs @@ -0,0 +1,105 @@ +//! PCI capability structures and queries + +use abi::error::Error; +use device_api::interrupt::{MessageInterruptController, MsiHandler}; + +use crate::{ + device::bus::pci::PciBaseAddress, + mem::{address::FromRaw, device::DeviceMemoryIoMut, PhysicalAddress}, +}; + +use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; + +/// MSI-X capability query +pub struct MsiXCapability; + +/// Represents an entry in MSI-X vector table +#[repr(C)] +pub struct MsiXEntry { + /// Address to which the value is written on interrupt + pub address: u64, + /// Value which is written to trigger an interrupt + pub data: u32, + /// Vector control word + pub control: u32, +} + +/// MSI-X capability data structure +pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + +impl PciCapability for MsiXCapability { + const ID: PciCapabilityId = PciCapabilityId::MsiX; + type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>; + + fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( + space: &'s S, + offset: usize, + ) -> Self::CapabilityData<'s, S> { + MsiXData { space, offset } + } +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { + // TODO use pending bits as well + /// Maps and returns the vector table associated with the device's MSI-X capability + pub fn vector_table<'a>(&self) -> Result, Error> { + let w0 = self.space.read_u16(self.offset + 2); + let dw1 = self.space.read_u32(self.offset + 4); + + let table_size = (w0 as usize & 0x3FF) + 1; + let bir = dw1 as usize & 0x3; + let table_offset = dw1 as usize & !0x3; + + let Some(base) = self.space.bar(bir) else { + return Err(Error::DoesNotExist); + }; + let PciBaseAddress::Memory(base) = base else { + return Err(Error::InvalidOperation); + }; + + debugln!("MSI-X table address: {:#x}", base + table_offset); + + unsafe { + DeviceMemoryIoMut::map_slice(PhysicalAddress::from_raw(base + table_offset), table_size) + } + } + + /// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs + /// are not generated. + pub fn set_enabled(&mut self, enabled: bool) { + let mut w0 = self.space.read_u32(self.offset); + if enabled { + w0 |= 1 << 31; + } else { + w0 &= !(1 << 31); + } + self.space.write_u32(self.offset, w0); + } +} + +impl MsiXEntry { + /// If set, prevents the MSI-X interrupt from being delivered + pub fn set_masked(&mut self, masked: bool) { + if masked { + self.control |= 1; + } else { + self.control &= !1; + } + } + + /// Registers the MSI-X vector with the interrupt controller and enables it + pub fn register( + &mut self, + ic: &C, + handler: &'static dyn MsiHandler, + ) -> Result<(), Error> { + let info = ic.register_msi(handler)?; + self.address = info.address as _; + self.data = info.value as _; + self.set_masked(false); + Ok(()) + } +} diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index abb3f8c1..d699e412 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -8,6 +8,7 @@ use device_api::Device; use kernel_util::sync::IrqSafeSpinlock; use yggdrasil_abi::error::Error; +pub mod capability; mod space; pub use space::{ @@ -33,6 +34,14 @@ bitflags! { } } +bitflags! { + /// Status register of the PCI configuration space + pub struct PciStatusRegister: u16 { + /// Read-only. If set, the configuration space has a pointer to the capabilities list. + const CAPABILITIES_LIST = 1 << 4; + } +} + /// Represents the address of a single object on a bus (or the bus itself) #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct PciAddress { @@ -55,6 +64,33 @@ pub enum PciBaseAddress { Io(u16), } +/// Unique ID assigned to PCI capability structures +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] +#[repr(u8)] +pub enum PciCapabilityId { + /// MSI (32-bit or 64-bit) + Msi = 0x05, + /// MSI-X + MsiX = 0x11, + /// Unknown capability missing from this list + Unknown, +} + +/// Interface used for querying PCI capabilities +pub trait PciCapability { + /// Capability ID + const ID: PciCapabilityId; + /// Wrapper for accessing the capability data structure + type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a>; + + /// Constructs an access wrapper for this capability with given offset + fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( + space: &'s S, + offset: usize, + ) -> Self::CapabilityData<'s, S>; +} + /// Describes a PCI device #[derive(Debug)] pub struct PciDeviceInfo { diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs index 72a5018d..7a2e0977 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/src/device/bus/pci/space/mod.rs @@ -1,4 +1,6 @@ -use super::{PciAddress, PciBaseAddress, PciEcam}; +use crate::device::bus::pci::PciStatusRegister; + +use super::{PciAddress, PciBaseAddress, PciCapability, PciCapabilityId, PciEcam}; pub(super) mod ecam; @@ -72,6 +74,30 @@ pub enum PciConfigSpace { Ecam(PciEcam), } +pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> { + space: &'s S, + current: Option, +} + +impl<'s, S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'s, S> { + type Item = (PciCapabilityId, usize); + + fn next(&mut self) -> Option { + let offset = self.current? & !0x3; + + let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) }; + let next_pointer = self.space.read_u8(offset + 1); + + self.current = if next_pointer != 0 { + Some(next_pointer as usize) + } else { + None + }; + + Some((id, offset)) + } +} + /// Interface for accessing the configuration space of a device pub trait PciConfigurationSpace { /// Reads a 32-bit value from the device configuration space. @@ -120,7 +146,7 @@ pub trait PciConfigurationSpace { } /// Writes a byte to the device configuration space - fn write_u8(&self, offset: usize, value: u16) { + fn write_u8(&self, _offset: usize, _value: u16) { todo!() } @@ -185,6 +211,14 @@ pub trait PciConfigurationSpace { "#] secondary_bus ); + pci_config_field!( + 0x34 => u8, + #[doc = + r"Returns the offset within the configuration space where the Capabilities List + is located. Only valid if the corresponding Status Register bit is set" + ] + capability_pointer + ); /// Returns the value of the Base Address Register with given index. /// @@ -242,4 +276,38 @@ pub trait PciConfigurationSpace { } } } + + /// Returns an iterator over the PCI capabilities + fn capability_iter(&self) -> CapabilityIterator { + let status = PciStatusRegister::from_bits_retain(self.status()); + + let current = if status.contains(PciStatusRegister::CAPABILITIES_LIST) { + let ptr = self.capability_pointer() as usize; + + if ptr != 0 { + Some(self.capability_pointer() as usize) + } else { + None + } + } else { + // Return an empty iterator + None + }; + + CapabilityIterator { + space: self, + current, + } + } + + /// Locates a capability within this configuration space + fn capability<'s, C: PciCapability>(&'s self) -> Option> { + self.capability_iter().find_map(|(id, offset)| { + if id == C::ID { + Some(C::data(self, offset)) + } else { + None + } + }) + } } diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs index 9f6231f4..10fb324a 100644 --- a/src/device/nvme/command.rs +++ b/src/device/nvme/command.rs @@ -1,3 +1,5 @@ +#![allow(unused)] + use core::fmt::{self, Write}; use crate::{ @@ -47,6 +49,7 @@ pub struct IdentifyControllerRequest { pub struct CreateIoCompletionQueue { pub id: u32, pub size: usize, + pub vector: u32, pub data: PhysicalAddress, } @@ -110,9 +113,7 @@ impl Command for CreateIoCompletionQueue { sqe.command.set_opcode(0x05); sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw()); sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id; - sqe.command_specific[1] = 1; - // TODO ENABLE IRQS HERE - sqe.command_specific[2] = 0; + sqe.command_specific[1] = (self.vector << 16) | 3; } } diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index f11928da..cbaae53d 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -3,7 +3,7 @@ use core::{mem::size_of, time::Duration}; use abi::error::Error; use alloc::vec::Vec; -use device_api::Device; +use device_api::{interrupt::MsiHandler, Device}; use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -12,8 +12,11 @@ use tock_registers::{ }; use crate::{ + arch::{Architecture, ARCHITECTURE}, device::{ - bus::pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace}, + bus::pci::{ + capability::MsiXCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, + }, nvme::{ command::IdentifyControllerRequest, queue::{CompletionQueueEntry, SubmissionQueueEntry}, @@ -21,7 +24,7 @@ use crate::{ }, mem::{ address::{FromRaw, IntoRaw}, - device::DeviceMemoryIo, + device::{DeviceMemoryIo, DeviceMemoryIoMut}, PhysicalAddress, }, task::runtime, @@ -33,7 +36,7 @@ use self::{ queue::QueuePair, }; -use super::bus::pci::{FromPciBus, PciDeviceInfo}; +use super::bus::pci::{capability::MsiXEntry, FromPciBus, PciDeviceInfo}; mod command; mod error; @@ -109,6 +112,7 @@ pub struct NvmeController { regs: IrqSafeSpinlock>, admin_q: OneTimeInit>, ioqs: OneTimeInit>>, + vector_table: IrqSafeSpinlock>, doorbell_shift: usize, } @@ -123,9 +127,11 @@ impl Regs { impl NvmeController { async fn late_init(&'static self) -> Result<(), NvmeError> { - runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); + // runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); let admin_q = self.admin_q.get(); + infoln!("SETUP"); + // Request a CQ/SQ pair for I/O admin_q .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1)) @@ -133,7 +139,8 @@ impl NvmeController { // Allocate the queue let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) }; - let io_q = QueuePair::new(32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; + let io_q = + QueuePair::new(0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; // Identify the controller let identify = admin_q @@ -145,6 +152,7 @@ impl NvmeController { .request_no_data(CreateIoCompletionQueue { id: 1, size: 32, + vector: 0, data: io_q.cq_physical_pointer(), }) .await?; @@ -160,14 +168,6 @@ impl NvmeController { loop {} } - // TODO MSI(-X) or IRQ (ACPI currently broken) support for PCIe-based NVMe - async fn poll_task(&'static self) { - loop { - self.admin_q.get().process_completions(); - runtime::sleep(Duration::from_millis(100)).await; - } - } - unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) { let regs = self.regs.lock(); let sq_ptr = regs.doorbell_ptr(self.doorbell_shift, false, idx); @@ -176,12 +176,18 @@ impl NvmeController { } } +impl MsiHandler for NvmeController { + fn handle_msi(&self, vector: usize) -> bool { + debugln!("handle_msi {}", vector); + self.admin_q.get().process_completions() != 0 + } +} + impl Device for NvmeController { unsafe fn init(&'static self) -> Result<(), Error> { let regs = self.regs.lock(); let min_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMIN)); - let max_page_size = 1usize << (12 + regs.CAP.read(CAP::MPSMAX)); if min_page_size > 4096 { panic!(); @@ -205,8 +211,13 @@ impl Device for NvmeController { // Setup the admin queue (index 0) let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) }; let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) }; - let admin_q = - QueuePair::new(queue_slots as usize, admin_sq_doorbell, admin_cq_doorbell).unwrap(); + let admin_q = QueuePair::new( + 0, + queue_slots as usize, + admin_sq_doorbell, + admin_cq_doorbell, + ) + .unwrap(); regs.AQA .modify(AQA::ASQS.val(queue_slots as u32 - 1) + AQA::ACQS.val(queue_slots as u32 - 1)); @@ -239,6 +250,16 @@ impl Device for NvmeController { self.admin_q.init(admin_q); + // Register the IRQs (TODO: use multiple) + { + let mut vt = self.vector_table.lock(); + + // Register vector 0 + vt[0] + .register(ARCHITECTURE.message_interrupt_controller(), self) + .unwrap(); + } + // Schedule late_init task runtime::spawn(self.late_init())?; @@ -256,6 +277,15 @@ impl FromPciBus for NvmeController { panic!(); }; + // TODO also support MSI + let mut msix = info.config_space.capability::().unwrap(); + let mut vt = msix.vector_table()?; + + for vector in vt.iter_mut() { + vector.set_masked(true); + } + msix.set_enabled(true); + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; @@ -272,6 +302,7 @@ impl FromPciBus for NvmeController { regs: IrqSafeSpinlock::new(regs), admin_q: OneTimeInit::new(), ioqs: OneTimeInit::new(), + vector_table: IrqSafeSpinlock::new(vt), doorbell_shift, }) } diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index afab4c02..31a8ee14 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -1,6 +1,5 @@ use core::{ mem::size_of, - ops::DerefMut, pin::Pin, ptr::null_mut, task::{Context, Poll}, @@ -94,6 +93,8 @@ pub struct QueuePair<'a> { base: PhysicalAddress, page_count: usize, + vector: usize, + sq_base: PhysicalAddress, cq_base: PhysicalAddress, @@ -246,6 +247,7 @@ impl<'a, T> Queue<'a, T> { impl<'a> QueuePair<'a> { pub fn new( + vector: usize, capacity: usize, sq_doorbell: *mut u32, cq_doorbell: *mut u32, @@ -279,6 +281,7 @@ impl<'a> QueuePair<'a> { Ok(Self { completion_notify: QueueWaker::new(), + vector, base, page_count, sq_base, diff --git a/src/mem/device.rs b/src/mem/device.rs index 7a190134..45f5776b 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -1,12 +1,19 @@ //! Facilities for mapping devices to virtual address space -use core::{mem::size_of, ops::Deref}; +use core::{ + alloc::Layout, + mem::size_of, + ops::{Deref, DerefMut}, +}; use abi::error::Error; use alloc::sync::Arc; use crate::arch::{Architecture, ARCHITECTURE}; -use super::PhysicalAddress; +use super::{ + address::{AsPhysicalAddress, FromRaw}, + PhysicalAddress, +}; /// Describes a single device memory mapping #[derive(Debug)] @@ -37,6 +44,14 @@ pub struct DeviceMemoryIo<'a, T: ?Sized> { value: &'a T, } +/// Describes a single typed and mutable device memory mapping +#[derive(Debug)] +pub struct DeviceMemoryIoMut<'a, T: ?Sized> { + #[allow(unused)] + inner: RawDeviceMemoryMapping, + value: &'a mut T, +} + impl RawDeviceMemoryMapping { /// Maps a region of physical memory as device memory of given size. /// @@ -130,3 +145,42 @@ impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> { self.value } } + +impl AsPhysicalAddress for DeviceMemoryIo<'_, T> { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::from_raw(self.inner.base_address) + } +} + +impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> { + /// Maps a physical address as device memory to a slice `[T; len]` + /// + /// # Safety + /// + /// The caller must ensure the address actually points to a value of type `T`, as well as + /// proper access synchronization. The caller must also ensure the `len` is valid. + pub unsafe fn map_slice( + base: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let inner = RawDeviceMemoryMapping::map(base, layout.size())?; + let value = core::slice::from_raw_parts_mut(inner.address as *mut T, len); + + Ok(DeviceMemoryIoMut { inner, value }) + } +} + +impl<'a, T: ?Sized> Deref for DeviceMemoryIoMut<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for DeviceMemoryIoMut<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} From 352c68e31e794cff2347d5f11d399b9f57776bad Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 10 Dec 2023 12:51:53 +0200 Subject: [PATCH 110/211] dev/block: NVMe drive enumeration --- Cargo.toml | 1 + lib/device-api/Cargo.toml | 4 +- lib/device-api/macros/Cargo.toml | 2 +- src/debug.rs | 36 +++++--- src/device/bus/pci/space/mod.rs | 2 +- src/device/nvme/command.rs | 137 +++++++++++++++++++++++++++++- src/device/nvme/drive.rs | 87 +++++++++++++++++++ src/device/nvme/mod.rs | 139 +++++++++++++++++++++++++++---- src/device/nvme/queue.rs | 26 +++--- src/fs/devfs.rs | 20 ++++- src/init.rs | 1 - src/main.rs | 4 +- 12 files changed, 406 insertions(+), 53 deletions(-) create mode 100644 src/device/nvme/drive.rs diff --git a/Cargo.toml b/Cargo.toml index 960cd447..bbb1ec79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ device-api = { path = "lib/device-api", features = ["derive"] } kernel-util = { path = "lib/kernel-util" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } +device-api-macros = { path = "lib/device-api/macros" } atomic_enum = "0.2.0" bitflags = "2.3.3" diff --git a/lib/device-api/Cargo.toml b/lib/device-api/Cargo.toml index 814a018b..ff5aca07 100644 --- a/lib/device-api/Cargo.toml +++ b/lib/device-api/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -macros = { path = "macros", optional = true } +device-api-macros = { path = "macros", optional = true } [features] default = [] -derive = ["macros"] +derive = ["device-api-macros"] diff --git a/lib/device-api/macros/Cargo.toml b/lib/device-api/macros/Cargo.toml index aee949a6..8c3983aa 100644 --- a/lib/device-api/macros/Cargo.toml +++ b/lib/device-api/macros/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "macros" +name = "device-api-macros" version = "0.1.0" edition = "2021" diff --git a/src/debug.rs b/src/debug.rs index 4b6c7cf7..6ceb6113 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -200,21 +200,35 @@ pub static RING_LOGGER_SINK: RingLoggerSink = RingLoggerSink::new(); /// Prints a hex-dump of a slice, appending a virtual address offset to the output pub fn hex_dump(level: LogLevel, addr_offset: usize, data: &[u8]) { - for (i, b) in data.iter().enumerate() { - if i % 16 == 0 { - log_print_raw!(level, "{:X}: ", addr_offset + i) + const WINDOW_SIZE: usize = 16; + let window_count = (data.len() + WINDOW_SIZE) / WINDOW_SIZE; + + for iw in 0..window_count { + let off = iw * WINDOW_SIZE; + let len = core::cmp::min(data.len() - off, WINDOW_SIZE); + let window = &data[off..off + len]; + + log_print_raw!(level, "{:04X}: ", addr_offset + off); + for i in 0..WINDOW_SIZE { + if i < window.len() { + log_print_raw!(level, "{:02X}", window[i]); + } else { + log_print_raw!(level, " "); + } + + if i % 2 == 1 { + log_print_raw!(level, " "); + } } - log_print_raw!(level, "{:02X}", b); - - if i % 16 == 15 { - log_print_raw!(level, "\n"); - } else if i % 2 == 1 { - log_print_raw!(level, " "); + for &ch in window { + if ch.is_ascii_graphic() || ch == b' ' { + log_print_raw!(level, "{}", ch as char); + } else { + log_print_raw!(level, "."); + } } - } - if data.len() % 16 != 0 { log_print_raw!(level, "\n"); } } diff --git a/src/device/bus/pci/space/mod.rs b/src/device/bus/pci/space/mod.rs index 7a2e0977..51773391 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/src/device/bus/pci/space/mod.rs @@ -301,7 +301,7 @@ pub trait PciConfigurationSpace { } /// Locates a capability within this configuration space - fn capability<'s, C: PciCapability>(&'s self) -> Option> { + fn capability(&self) -> Option> { self.capability_iter().find_map(|(id, offset)| { if id == C::ID { Some(C::data(self, offset)) diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs index 10fb324a..efb875ae 100644 --- a/src/device/nvme/command.rs +++ b/src/device/nvme/command.rs @@ -2,6 +2,8 @@ use core::fmt::{self, Write}; +use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike}; + use crate::{ device::nvme::queue::PhysicalRegionPage, mem::{address::IntoRaw, PhysicalAddress}, @@ -33,6 +35,22 @@ pub enum ControllerType { Administrative, } +// I/O commands + +#[derive(Clone, Copy, Debug)] +pub struct IoRead { + pub nsid: u32, + pub lba: u64, + pub count: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct IoWrite { + pub nsid: u32, + pub lba: u64, + pub count: u32, +} + // Requests #[derive(Clone, Copy, Debug)] @@ -41,7 +59,15 @@ pub enum SetFeatureRequest { } #[derive(Clone, Copy, Debug)] -pub struct IdentifyControllerRequest { +pub struct IdentifyControllerRequest; + +#[derive(Clone, Copy, Debug)] +pub struct IdentifyActiveNamespaceIdListRequest { + pub start_id: u32, +} + +#[derive(Clone, Copy, Debug)] +pub struct IdentifyNamespaceRequest { pub nsid: u32, } @@ -81,11 +107,34 @@ pub struct IdentifyControllerResponse { pub cntrltype: ControllerType, } +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct IdentifyActiveNamespaceIdListResponse { + pub entries: [u32; 1024], +} + +register_structs! { + #[allow(non_snake_case)] + pub IdentifyNamespaceResponse { + (0 => NSZE: ReadOnly), + (8 => _0), + (25 => NLBAF: ReadOnly), + (26 => FLBAS: ReadOnly), + (27 => _1), + (128 => LBAFS: [ReadOnly; 64]), + (384 => _2), + (4096 => @END), + } +} + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct LbaFormat(u32); + impl Command for IdentifyControllerRequest { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { sqe.command.set_opcode(0x06); sqe.command_specific[0] = 0x01; - sqe.nsid = self.nsid; } } @@ -93,6 +142,62 @@ impl Request for IdentifyControllerRequest { type Response = IdentifyControllerResponse; } +impl Command for IdentifyActiveNamespaceIdListRequest { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + sqe.command.set_opcode(0x06); + sqe.command_specific[0] = 0x02; + sqe.nsid = self.start_id; + } +} + +impl Request for IdentifyActiveNamespaceIdListRequest { + type Response = IdentifyActiveNamespaceIdListResponse; +} + +impl Command for IdentifyNamespaceRequest { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + sqe.command.set_opcode(0x06); + sqe.command_specific[0] = 0x00; + sqe.nsid = self.nsid; + } +} + +impl Request for IdentifyNamespaceRequest { + type Response = IdentifyNamespaceResponse; +} + +impl IdentifyNamespaceResponse { + pub fn current_lba_fmt_idx(&self) -> usize { + let flbas = self.FLBAS.get(); + let mut index = flbas & 0xF; + if self.NLBAF.get() > 16 { + index |= (flbas & 0xE0) >> 1; + } + index as usize + } + + pub fn lba_fmt(&self, idx: usize) -> Option { + if idx >= self.NLBAF.get() as usize { + return None; + } + Some(LbaFormat(self.LBAFS[idx].get())) + } + + pub fn total_lba_count(&self) -> u64 { + self.NSZE.get() + } +} + +impl LbaFormat { + pub fn lba_data_size(&self) -> Option { + let lbads = (self.0 >> 16) & 0xFF; + if lbads < 9 { + return None; + } + Some(1 << lbads) + } +} + impl Command for SetFeatureRequest { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { sqe.command.set_opcode(0x09); @@ -111,7 +216,7 @@ impl Command for SetFeatureRequest { impl Command for CreateIoCompletionQueue { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { sqe.command.set_opcode(0x05); - sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw()); + sqe.data_pointer[0] = PhysicalRegionPage::with_addr(self.data); sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id; sqe.command_specific[1] = (self.vector << 16) | 3; } @@ -120,7 +225,7 @@ impl Command for CreateIoCompletionQueue { impl Command for CreateIoSubmissionQueue { fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { sqe.command.set_opcode(0x01); - sqe.data_pointer[1] = PhysicalRegionPage::with_addr(self.data.into_raw()); + sqe.data_pointer[0] = PhysicalRegionPage::with_addr(self.data); sqe.command_specific[0] = ((self.size as u32 - 1) << 16) | self.id; // Medium priority sqe.command_specific[1] = (self.cq_id << 16) | 1; @@ -140,3 +245,27 @@ impl fmt::Debug for String { Ok(()) } } + +impl Command for IoRead { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + assert!(self.count < 65536); + + sqe.command.set_opcode(0x02); + sqe.command_specific[0] = self.lba as u32; + sqe.command_specific[1] = (self.lba >> 32) as u32; + sqe.command_specific[2] = self.count; + sqe.nsid = self.nsid; + } +} + +impl Command for IoWrite { + fn fill_sqe(&self, sqe: &mut SubmissionQueueEntry) { + assert!(self.count < 65536); + + sqe.command.set_opcode(0x01); + sqe.command_specific[0] = self.lba as u32; + sqe.command_specific[1] = (self.lba >> 32) as u32; + sqe.command_specific[2] = self.count; + sqe.nsid = self.nsid; + } +} diff --git a/src/device/nvme/drive.rs b/src/device/nvme/drive.rs new file mode 100644 index 00000000..4914c8a9 --- /dev/null +++ b/src/device/nvme/drive.rs @@ -0,0 +1,87 @@ +use abi::{error::Error, io::DeviceRequest}; +use alloc::{boxed::Box, format}; +use vfs::BlockDevice; + +use crate::{device::nvme::command::IdentifyNamespaceRequest, fs::devfs}; + +use super::{error::NvmeError, NvmeController}; + +#[allow(unused)] +pub struct NvmeDrive { + controller: &'static NvmeController, + nsid: u32, + total_lba_count: u64, + lba_size: u64, +} + +impl NvmeDrive { + pub async fn create( + controller: &'static NvmeController, + nsid: u32, + ) -> Result<&'static NvmeDrive, NvmeError> { + let admin_q = controller.admin_q.get(); + let identify = admin_q.request(IdentifyNamespaceRequest { nsid })?.await?; + + let current_lba_format_idx = identify.current_lba_fmt_idx(); + let current_lba_format = identify.lba_fmt(current_lba_format_idx).unwrap(); + let lba_size = current_lba_format.lba_data_size().unwrap(); + let total_lba_count = identify.total_lba_count(); + + debugln!( + "ns = {}, lba = {}B, size = {}M", + nsid, + lba_size, + (total_lba_count * lba_size) / (1024 * 1024) + ); + + let dev = Box::leak(Box::new(NvmeDrive { + controller, + nsid, + total_lba_count, + lba_size, + })); + + // TODO add the drive as a block device + let node_name = format!("nvme{}n{}", controller.controller_id.get(), nsid); + devfs::add_named_block_device(dev, node_name).ok(); + + // TODO probe partitions + + Ok(dev) + } + + // TODO proper interface for reading/writing blocks + // pub async fn read_block( + // &self, + // lba: u64, + // block: &mut PhysicalRefMut<'_, [u8]>, + // ) -> Result<(), NvmeError> { + // self.controller.read_block(self.nsid, lba, block).await + // } + + // pub async fn write_block( + // &self, + // lba: u64, + // block: &PhysicalRefMut<'_, [u8]>, + // ) -> Result<(), NvmeError> { + // self.controller.write_block(self.nsid, lba, block).await + // } +} + +impl BlockDevice for NvmeDrive { + fn read(&'static self, _pos: u64, _buf: &mut [u8]) -> Result { + todo!() + } + + fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { + todo!() + } + + fn size(&self) -> Result { + Ok(self.lba_size * self.total_lba_count) + } + + fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { + todo!() + } +} diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index cbaae53d..6c23bf9d 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -2,7 +2,7 @@ use core::{mem::size_of, time::Duration}; use abi::error::Error; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, vec::Vec}; use device_api::{interrupt::MsiHandler, Device}; use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use tock_registers::{ @@ -18,13 +18,17 @@ use crate::{ capability::MsiXCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, }, nvme::{ - command::IdentifyControllerRequest, + command::{ + IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest, IoRead, IoWrite, + }, + drive::NvmeDrive, queue::{CompletionQueueEntry, SubmissionQueueEntry}, }, }, mem::{ - address::{FromRaw, IntoRaw}, + address::{AsPhysicalAddress, FromRaw, IntoRaw}, device::{DeviceMemoryIo, DeviceMemoryIoMut}, + pointer::PhysicalRefMut, PhysicalAddress, }, task::runtime, @@ -39,6 +43,7 @@ use self::{ use super::bus::pci::{capability::MsiXEntry, FromPciBus, PciDeviceInfo}; mod command; +mod drive; mod error; mod queue; @@ -113,6 +118,8 @@ pub struct NvmeController { admin_q: OneTimeInit>, ioqs: OneTimeInit>>, vector_table: IrqSafeSpinlock>, + drive_table: IrqSafeSpinlock>, + controller_id: OneTimeInit, doorbell_shift: usize, } @@ -120,17 +127,21 @@ pub struct NvmeController { impl Regs { unsafe fn doorbell_ptr(&self, shift: usize, completion: bool, queue_index: usize) -> *mut u32 { let doorbell_base = (self as *const Regs as *mut Regs).addr() + 0x1000; - let offset = (queue_index << shift) + completion as usize * 4; + let offset = ((queue_index << shift) + completion as usize) * 4; (doorbell_base + offset) as *mut u32 } } impl NvmeController { async fn late_init(&'static self) -> Result<(), NvmeError> { - // runtime::spawn(self.poll_task()).expect("Couldn't spawn NVMe poll task"); + register_nvme_controller(self); + let admin_q = self.admin_q.get(); - infoln!("SETUP"); + // Identify the controller + let _identify = admin_q.request(IdentifyControllerRequest)?.await?; + + // TODO do something with identify_controller // Request a CQ/SQ pair for I/O admin_q @@ -140,12 +151,7 @@ impl NvmeController { // Allocate the queue let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) }; let io_q = - QueuePair::new(0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; - - // Identify the controller - let identify = admin_q - .request(IdentifyControllerRequest { nsid: 0 })? - .await?; + QueuePair::new(1, 0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; // Create the queue on the device side admin_q @@ -165,7 +171,86 @@ impl NvmeController { }) .await?; - loop {} + self.ioqs.init(Vec::from_iter([io_q])); + + // Identify namespaces + self.enumerate_namespaces().await?; + + Ok(()) + } + + async fn enumerate_namespaces(&'static self) -> Result<(), NvmeError> { + let admin_q = self.admin_q.get(); + + let namespaces = admin_q + .request(IdentifyActiveNamespaceIdListRequest { start_id: 0 })? + .await?; + + let count = namespaces.entries.iter().position(|&x| x == 0).unwrap(); + let list = &namespaces.entries[..count]; + + for &nsid in list { + match NvmeDrive::create(self, nsid).await { + Ok(drive) => { + self.drive_table.lock().insert(nsid, drive); + } + Err(error) => { + warnln!("Could not create nvme drive, nsid={}: {:?}", nsid, error); + } + } + } + + Ok(()) + } + + pub async fn read_block( + &'static self, + nsid: u32, + lba: u64, + buffer: &mut PhysicalRefMut<'_, [u8]>, + ) -> Result<(), NvmeError> { + let ioq = &self.ioqs.get()[0]; + let buffer_address = unsafe { buffer.as_physical_address() }; + + debugln!("read nsid={}, lba={:#x}", nsid, lba); + let cmd_id = ioq.submit( + IoRead { + nsid, + lba, + count: 1, + }, + &[buffer_address], + true, + ); + + ioq.wait_for_completion(cmd_id, ()).await?; + + Ok(()) + } + + pub async fn write_block( + &'static self, + nsid: u32, + lba: u64, + buffer: &PhysicalRefMut<'_, [u8]>, + ) -> Result<(), NvmeError> { + let ioq = &self.ioqs.get()[0]; + let buffer_address = unsafe { buffer.as_physical_address() }; + + debugln!("write nsid={}, lba={:#x}", nsid, lba); + let cmd_id = ioq.submit( + IoWrite { + nsid, + lba, + count: 1, + }, + &[buffer_address], + true, + ); + + ioq.wait_for_completion(cmd_id, ()).await?; + + Ok(()) } unsafe fn doorbell_pair(&self, idx: usize) -> (*mut u32, *mut u32) { @@ -177,9 +262,15 @@ impl NvmeController { } impl MsiHandler for NvmeController { - fn handle_msi(&self, vector: usize) -> bool { - debugln!("handle_msi {}", vector); - self.admin_q.get().process_completions() != 0 + fn handle_msi(&self, _vector: usize) -> bool { + // TODO check MSI-X pending bits + self.admin_q.get().process_completions(); + if let Some(qs) = self.ioqs.try_get() { + for q in qs { + q.process_completions(); + } + } + true } } @@ -211,7 +302,9 @@ impl Device for NvmeController { // Setup the admin queue (index 0) let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) }; let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) }; + debugln!("sq_doorbell for adminq = {:p}", admin_sq_doorbell); let admin_q = QueuePair::new( + 0, 0, queue_slots as usize, admin_sq_doorbell, @@ -296,14 +389,26 @@ impl FromPciBus for NvmeController { // Disable the controller regs.CC.modify(CC::ENABLE::CLEAR); - let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 2; + let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1; Ok(Self { regs: IrqSafeSpinlock::new(regs), admin_q: OneTimeInit::new(), ioqs: OneTimeInit::new(), vector_table: IrqSafeSpinlock::new(vt), + drive_table: IrqSafeSpinlock::new(BTreeMap::new()), + controller_id: OneTimeInit::new(), doorbell_shift, }) } } + +static NVME_CONTROLLERS: IrqSafeSpinlock> = + IrqSafeSpinlock::new(Vec::new()); + +pub fn register_nvme_controller(ctrl: &'static NvmeController) { + let mut list = NVME_CONTROLLERS.lock(); + let id = list.len(); + list.push(ctrl); + ctrl.controller_id.init(id); +} diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 31a8ee14..88693f55 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -50,7 +50,7 @@ pub struct CommandDword0(u32); pub struct SubmissionQueueEntry { pub command: CommandDword0, // 0 pub nsid: u32, // 1 - _0: [u32; 2], // 2, 3 + pub io_data: [u32; 2], // 2, 3 pub metadata_pointer: u64, // 4, 5 pub data_pointer: [PhysicalRegionPage; 2], // 6, 7, 8, 9 pub command_specific: [u32; 6], // 10, 11, 12, 13, 14, 15 @@ -89,12 +89,13 @@ struct Inner<'a> { } // TODO PageBox? +#[allow(unused)] pub struct QueuePair<'a> { base: PhysicalAddress, page_count: usize, + id: u32, vector: usize, - sq_base: PhysicalAddress, cq_base: PhysicalAddress, @@ -110,8 +111,8 @@ impl PhysicalRegionPage { Self(0) } - pub const fn with_addr(addr: u64) -> Self { - Self(addr) + pub const fn with_addr(address: PhysicalAddress) -> Self { + Self(address.into_raw()) } } @@ -240,13 +241,14 @@ impl<'a, T> Queue<'a, T> { wrapped } - pub fn is_empty(&self) -> bool { - self.head == self.tail - } + // pub fn is_empty(&self) -> bool { + // self.head == self.tail + // } } impl<'a> QueuePair<'a> { pub fn new( + id: u32, vector: usize, capacity: usize, sq_doorbell: *mut u32, @@ -281,6 +283,7 @@ impl<'a> QueuePair<'a> { Ok(Self { completion_notify: QueueWaker::new(), + id, vector, base, page_count, @@ -350,7 +353,7 @@ impl<'a> QueuePair<'a> { match ranges.len() { 1 => { - sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0].into_raw()); + sqe.data_pointer[0] = PhysicalRegionPage::with_addr(ranges[0]); sqe.data_pointer[1] = PhysicalRegionPage::null(); } 0 => { @@ -397,7 +400,7 @@ impl<'a> QueuePair<'a> { 'r: 'a, { assert_ne!(size_of::(), 0); - assert!(size_of::() < 0x1000); + assert!(size_of::() <= 0x1000); let page = phys::alloc_page().map_err(NvmeError::MemoryError)?; // TODO PageBox @@ -424,8 +427,8 @@ impl<'a> QueuePair<'a> { n += 1; let sub_queue_id = cmp.sub_queue_id(); - // TODO support queues other than admin q - assert_eq!(sub_queue_id, 0); + // TODO allow several sqs receive completions through one cq? + assert_eq!(sub_queue_id, self.id); let sub_queue_head = cmp.sub_queue_head(); let cmp = *cmp; @@ -442,7 +445,6 @@ impl<'a> QueuePair<'a> { let command_id = cmp.command_id(); if inner.pending.remove(&command_id) { - debugln!("Insert completion: {}", command_id); inner.completed.insert(command_id, cmp); } } diff --git a/src/fs/devfs.rs b/src/fs/devfs.rs index c43a7869..74f9651d 100644 --- a/src/fs/devfs.rs +++ b/src/fs/devfs.rs @@ -6,7 +6,7 @@ use alloc::{format, string::String}; use kernel_util::util::OneTimeInit; use vfs::{ impls::{read_fn_node, MemoryDirectory}, - CharDevice, Node, NodeFlags, NodeRef, + BlockDevice, CharDevice, Node, NodeFlags, NodeRef, }; use crate::proc::random; @@ -37,7 +37,8 @@ pub fn root() -> &'static NodeRef { DEVFS_ROOT.get() } -fn _add_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Error> { +/// Adds a character device with a custom name +pub fn add_named_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Error> { infoln!("Add char device: {}", name); let node = Node::char(dev, NodeFlags::IN_MEMORY_PROPS); @@ -45,6 +46,19 @@ fn _add_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Er DEVFS_ROOT.get().add_child(name, node) } +/// Adds a block device with a custom name +pub fn add_named_block_device>( + dev: &'static dyn BlockDevice, + name: S, +) -> Result<(), Error> { + let name = name.into(); + infoln!("Add block device: {}", name); + + let node = Node::block(dev, NodeFlags::IN_MEMORY_PROPS); + + DEVFS_ROOT.get().add_child(name, node) +} + /// Adds a character device to the devfs pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Result<(), Error> { static TTY_COUNT: AtomicUsize = AtomicUsize::new(0); @@ -58,7 +72,7 @@ pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Re let value = count.fetch_add(1, Ordering::AcqRel); let name = format!("{}{}", prefix, value); - _add_char_device(dev, name) + add_named_char_device(dev, name) } /// Adds "pseudo"-devices to the filesystem (i.e. /dev/random) diff --git a/src/init.rs b/src/init.rs index be2265b3..9579d0ed 100644 --- a/src/init.rs +++ b/src/init.rs @@ -25,7 +25,6 @@ fn setup_root() -> Result { /// initialization has finished. pub fn kinit() -> Result<(), Error> { infoln!("In main"); - loop {} #[cfg(feature = "fb_console")] { diff --git a/src/main.rs b/src/main.rs index 4fdc7cb7..e73e5093 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,7 +19,9 @@ strict_provenance, slice_ptr_get, slice_split_once, - iter_collect_into + iter_collect_into, + iter_next_chunk, + exact_size_is_empty )] #![allow( clippy::new_without_default, From 4d23cc5c7409700e5c431732165e8c66ac46d2c9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 10 Dec 2023 18:52:33 +0200 Subject: [PATCH 111/211] refactor: move memory API outside of the main kernel --- lib/kernel-util/Cargo.toml | 1 + lib/kernel-util/src/api.rs | 18 +- lib/kernel-util/src/lib.rs | 5 +- lib/kernel-util/src/mem/address.rs | 157 ++++++++++++++++++ {src => lib/kernel-util/src}/mem/device.rs | 14 +- lib/kernel-util/src/mem/mod.rs | 3 + lib/kernel-util/src/mem/table.rs | 79 +++++++++ src/arch/mod.rs | 71 +++++--- src/arch/x86_64/acpi.rs | 11 +- src/arch/x86_64/apic/ioapic.rs | 13 +- src/arch/x86_64/apic/local.rs | 15 +- src/arch/x86_64/context.rs | 10 +- src/arch/x86_64/mem/mod.rs | 39 +++-- src/arch/x86_64/mem/process.rs | 30 ++-- src/arch/x86_64/mem/table.rs | 11 +- src/arch/x86_64/mod.rs | 32 ++-- src/arch/x86_64/smp.rs | 8 +- src/device/bus/pci/capability.rs | 9 +- src/device/bus/pci/mod.rs | 10 +- src/device/bus/pci/space/ecam.rs | 18 +- src/device/display/linear_fb.rs | 7 +- src/device/nvme/command.rs | 6 +- src/device/nvme/mod.rs | 16 +- src/device/nvme/queue.rs | 16 +- src/fs/mod.rs | 4 +- src/mem/address.rs | 184 +-------------------- src/mem/heap.rs | 6 +- src/mem/mod.rs | 6 +- src/mem/phys/manager.rs | 7 +- src/mem/phys/mod.rs | 10 +- src/mem/pointer.rs | 2 +- src/mem/table.rs | 33 +--- 32 files changed, 460 insertions(+), 391 deletions(-) create mode 100644 lib/kernel-util/src/mem/address.rs rename {src => lib/kernel-util/src}/mem/device.rs (94%) create mode 100644 lib/kernel-util/src/mem/mod.rs create mode 100644 lib/kernel-util/src/mem/table.rs diff --git a/lib/kernel-util/Cargo.toml b/lib/kernel-util/Cargo.toml index 496f8d0d..98dee9e8 100644 --- a/lib/kernel-util/Cargo.toml +++ b/lib/kernel-util/Cargo.toml @@ -6,3 +6,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index 5ead83e6..99aded18 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -1,7 +1,21 @@ +use yggdrasil_abi::error::Error; + +use crate::mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}; + extern "Rust" { pub fn __acquire_irq_guard() -> bool; pub fn __release_irq_guard(mask: bool); - pub fn __suspend(); - pub fn __yield(); + pub fn __allocate_2m_page() -> u64; + pub fn __allocate_page() -> u64; + pub fn __free_page(page: u64); + + pub fn __virtualize(phys: u64) -> usize; + pub fn __physicalize(virt: usize) -> u64; + + pub fn __map_device_pages( + base: PhysicalAddress, + count: usize, + ) -> Result; + pub fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping); } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index dc25946f..7a71340a 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -1,8 +1,11 @@ #![no_std] -#![feature(maybe_uninit_slice)] +#![feature(maybe_uninit_slice, step_trait, const_trait_impl, effects)] + +extern crate alloc; pub(crate) mod api; +pub mod mem; pub mod sync; pub mod util; diff --git a/lib/kernel-util/src/mem/address.rs b/lib/kernel-util/src/mem/address.rs new file mode 100644 index 00000000..e37584ce --- /dev/null +++ b/lib/kernel-util/src/mem/address.rs @@ -0,0 +1,157 @@ +use crate::api::{__physicalize, __virtualize}; +use core::{ + fmt, + iter::Step, + mem::align_of, + ops::{Add, Sub}, +}; + +/// Wrapper type to represent a physical memory address +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[repr(transparent)] +pub struct PhysicalAddress(pub(crate) u64); + +/// Interface for converting addresses from their raw values to more specific types +#[const_trait] +pub trait FromRaw { + /// Converts a raw value into the address wrapper type + fn from_raw(value: T) -> Self; +} + +/// Interface for converting wrapper types into their raw address representations +#[const_trait] +pub trait IntoRaw { + /// Converts a wrapper type value into its raw address + fn into_raw(self) -> T; +} + +/// Interface for obtaining physical addresses of values +pub trait AsPhysicalAddress { + /// Returns the value's physical address. + /// + /// # Safety + /// + /// The caller must ensure the value has been constructed and obtained through proper means. + unsafe fn as_physical_address(&self) -> PhysicalAddress; +} + +impl PhysicalAddress { + /// Physical address of zero + pub const ZERO: Self = Self(0); + + /// Maximum representable physical address + pub const MAX: Self = Self(u64::MAX); + /// Minumum representable physical address + pub const MIN: Self = Self(u64::MIN); + + /// Applies an offset to the address + pub const fn add(self, offset: usize) -> Self { + Self(self.0 + offset as u64) + } + + /// Returns `true` if the address is zero + #[inline(always)] + pub const fn is_zero(self) -> bool { + self.0 == 0 + } + + /// Returns `true` if the address is aligned to a boundary of a page at level `L` + #[inline] + pub const fn is_aligned_for(self) -> bool { + self.0 as usize % align_of::() == 0 + } + + /// Converts a previously virtualized physical address back into its physical form. + /// + /// # Safety + /// + /// The caller must ensure the function only receives addresses obtained through + /// [PhysicalAddress::virtualize_raw] or + /// [super::pointer::PhysicalRef]/[super::pointer::PhysicalRefMut] facilities. + pub unsafe fn from_virtualized(address: usize) -> Self { + Self(__physicalize(address)) + } + + /// Converts the physical address to a virtual one + pub fn virtualize_raw(self) -> usize { + unsafe { __virtualize(self.0) } + } +} + +impl Add for PhysicalAddress { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl Sub for PhysicalAddress { + type Output = usize; + + fn sub(self, rhs: Self) -> Self::Output { + (self.0 - rhs.0) as usize + } +} + +// Conversions + +impl const FromRaw for PhysicalAddress { + fn from_raw(value: u64) -> Self { + Self(value) + } +} + +impl const FromRaw for PhysicalAddress { + fn from_raw(value: usize) -> Self { + Self(value as u64) + } +} + +impl const IntoRaw for PhysicalAddress { + fn into_raw(self) -> u64 { + self.0 + } +} + +impl const IntoRaw for PhysicalAddress { + fn into_raw(self) -> usize { + self.0 as usize + } +} + +impl From for u64 { + fn from(addr: PhysicalAddress) -> u64 { + addr.0 + } +} + +impl From for usize { + fn from(addr: PhysicalAddress) -> usize { + addr.0 as usize + } +} + +// Ranges + +impl Step for PhysicalAddress { + fn steps_between(_start: &Self, _end: &Self) -> Option { + todo!() + } + + fn forward_checked(start: Self, count: usize) -> Option { + start.0.checked_add(count as u64).map(Self) + } + + fn backward_checked(_start: Self, _count: usize) -> Option { + todo!() + } +} + +// fmt + +impl fmt::LowerHex for PhysicalAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} diff --git a/src/mem/device.rs b/lib/kernel-util/src/mem/device.rs similarity index 94% rename from src/mem/device.rs rename to lib/kernel-util/src/mem/device.rs index 45f5776b..4e39b31e 100644 --- a/src/mem/device.rs +++ b/lib/kernel-util/src/mem/device.rs @@ -1,19 +1,15 @@ -//! Facilities for mapping devices to virtual address space use core::{ alloc::Layout, mem::size_of, ops::{Deref, DerefMut}, }; -use abi::error::Error; use alloc::sync::Arc; +use yggdrasil_abi::error::Error; -use crate::arch::{Architecture, ARCHITECTURE}; +use crate::api::{__map_device_pages, __unmap_device_pages}; -use super::{ - address::{AsPhysicalAddress, FromRaw}, - PhysicalAddress, -}; +use super::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; /// Describes a single device memory mapping #[derive(Debug)] @@ -60,7 +56,7 @@ impl RawDeviceMemoryMapping { /// The caller must ensure proper access synchronization, as well as the address' origin. #[inline] pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { - ARCHITECTURE.map_device_memory(base, size) + __map_device_pages(base, size) } /// Consumes the device mapping, leaking its address without deallocating the translation @@ -75,7 +71,7 @@ impl RawDeviceMemoryMapping { impl Drop for RawDeviceMemoryMapping { fn drop(&mut self) { unsafe { - ARCHITECTURE.unmap_device_memory(self); + __unmap_device_pages(self); } } } diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs new file mode 100644 index 00000000..81f4df8c --- /dev/null +++ b/lib/kernel-util/src/mem/mod.rs @@ -0,0 +1,3 @@ +pub mod address; +pub mod device; +pub mod table; diff --git a/lib/kernel-util/src/mem/table.rs b/lib/kernel-util/src/mem/table.rs new file mode 100644 index 00000000..15203094 --- /dev/null +++ b/lib/kernel-util/src/mem/table.rs @@ -0,0 +1,79 @@ +use super::address::PhysicalAddress; + +/// Interface for a single level of address translation +pub trait EntryLevel: Copy { + /// The right shift needed to obtain an index of an entry at this level from an address + const SHIFT: usize; + /// The size of a page at this entry level + const SIZE: usize = 1 << Self::SHIFT; +} + +#[const_trait] +pub trait EntryLevelExt: Sized { + fn page_index(self) -> usize; + fn page_offset(self) -> usize; + fn page_count(self) -> usize; + fn page_align_up(self) -> Self; + fn page_align_down(self) -> Self; + fn is_page_aligned_for(self) -> bool; +} + +#[const_trait] +trait AddressLike: Sized { + fn into_usize(self) -> usize; + fn from_usize(v: usize) -> Self; +} + +impl const AddressLike for usize { + #[inline(always)] + fn into_usize(self) -> usize { + self + } + + #[inline(always)] + fn from_usize(v: usize) -> Self { + v + } +} + +impl const AddressLike for PhysicalAddress { + fn from_usize(v: usize) -> Self { + Self(v as _) + } + + fn into_usize(self) -> usize { + self.0 as _ + } +} + +impl const EntryLevelExt for T { + #[inline(always)] + fn page_index(self) -> usize { + (self.into_usize() >> L::SHIFT) & 0x1FF + } + + #[inline(always)] + fn page_offset(self) -> usize { + self.into_usize() & (L::SIZE - 1) + } + + #[inline(always)] + fn page_count(self) -> usize { + (self.into_usize() + L::SIZE - 1) / L::SIZE + } + + #[inline(always)] + fn page_align_up(self) -> Self { + Self::from_usize((self.into_usize() + L::SIZE - 1) & !(L::SIZE - 1)) + } + + #[inline(always)] + fn page_align_down(self) -> Self { + Self::from_usize(self.into_usize() & !(L::SIZE - 1)) + } + + #[inline(always)] + fn is_page_aligned_for(self) -> bool { + self.page_offset::() == 0 + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index c021f3be..d599433d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -28,8 +28,11 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; +use kernel_util::mem::{ + address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, +}; -use crate::mem::{device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, PhysicalAddress}; +use crate::mem::phys::PhysicalMemoryRegion; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -45,23 +48,8 @@ cfg_if! { } } -// External API for architecture specifics - -#[no_mangle] -fn __acquire_irq_guard() -> bool { - let mask = ArchitectureImpl::interrupt_mask(); - unsafe { - ArchitectureImpl::set_interrupt_mask(true); - } - mask -} - -#[no_mangle] -fn __release_irq_guard(mask: bool) { - unsafe { - ArchitectureImpl::set_interrupt_mask(mask); - } -} +/// Architecture-specific lowest level of page mapping +pub type L3 = ::L3; // Architecture interfaces @@ -85,6 +73,9 @@ pub trait Architecture { /// IRQ number type associated with the architecture type IrqNumber; + /// Lowest page entry level, usually 4KiB pages + type L3: EntryLevel; + /// Starts up the application processors that may be present in the system. /// /// # Safety @@ -120,10 +111,10 @@ pub trait Architecture { ) -> Result<(), Error>; /// Converts a physical address to a virtual one, so it can be accessed by the kernel - fn virtualize(address: PhysicalAddress) -> Result; + fn virtualize(address: u64) -> usize; /// Converts a virtual address created by [Architecture::virtualize] back to its physical form - fn physicalize(address: usize) -> Result; + fn physicalize(address: usize) -> u64; // Architecture intrinsics @@ -233,3 +224,43 @@ pub trait Architecture { } } } + +// External API for architecture specifics + +#[no_mangle] +fn __acquire_irq_guard() -> bool { + let mask = ArchitectureImpl::interrupt_mask(); + unsafe { + ArchitectureImpl::set_interrupt_mask(true); + } + mask +} + +#[no_mangle] +fn __release_irq_guard(mask: bool) { + unsafe { + ArchitectureImpl::set_interrupt_mask(mask); + } +} + +#[no_mangle] +fn __virtualize(addr: u64) -> usize { + ArchitectureImpl::virtualize(addr) +} + +#[no_mangle] +fn __physicalize(addr: usize) -> u64 { + ArchitectureImpl::physicalize(addr) +} + +#[no_mangle] +fn __map_device_pages( + base: PhysicalAddress, + count: usize, +) -> Result { + unsafe { ARCHITECTURE.map_device_memory(base, count) } +} +#[no_mangle] +fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping) { + unsafe { ARCHITECTURE.unmap_device_memory(mapping) } +} diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 3112c622..68c23fe2 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -15,7 +15,11 @@ use device_api::{ interrupt::{InterruptHandler, IpiDeliveryTarget}, Device, }; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + mem::address::{FromRaw, PhysicalAddress}, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use yggdrasil_abi::error::Error; use crate::{ @@ -23,10 +27,7 @@ use crate::{ x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, Architecture, CpuMessage, ARCHITECTURE, }, - mem::{ - address::FromRaw, heap::GLOBAL_HEAP, pointer::PhysicalRef, read_memory, write_memory, - PhysicalAddress, - }, + mem::{heap::GLOBAL_HEAP, pointer::PhysicalRef, read_memory, write_memory}, }; use super::intrinsics; diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index a6be3d91..a0bd6fe8 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -8,17 +8,20 @@ use device_api::{ }, Device, }; -use kernel_util::sync::IrqSafeSpinlock; +use kernel_util::{ + mem::{ + address::{FromRaw, PhysicalAddress}, + device::DeviceMemoryIo, + }, + sync::IrqSafeSpinlock, +}; use tock_registers::{ interfaces::{Readable, Writeable}, register_structs, registers::{ReadWrite, WriteOnly}, }; -use crate::{ - arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}, - mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, -}; +use crate::arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}; use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS}; diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 6158a4c6..1a5b3894 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -10,7 +10,15 @@ use device_api::{ }, Device, }; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + mem::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::DeviceMemoryIo, + table::EntryLevelExt, + }, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -24,11 +32,6 @@ use crate::{ }, CpuMessage, }, - mem::{ - address::{FromRaw, IntoRaw}, - device::DeviceMemoryIo, - PhysicalAddress, - }, task::Cpu, }; diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index f3d79f86..195ea309 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -2,15 +2,9 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; +use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw}; -use crate::{ - arch::x86_64::mem::KERNEL_TABLES, - mem::{ - address::{AsPhysicalAddress, IntoRaw}, - phys, - }, - task::context::TaskContextImpl, -}; +use crate::{arch::x86_64::mem::KERNEL_TABLES, mem::phys, task::context::TaskContextImpl}; struct StackBuilder { base: usize, diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 6e20cc65..bd4d055c 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -5,7 +5,14 @@ use core::{ }; use abi::error::Error; -use kernel_util::util::OneTimeInit; +use kernel_util::{ + mem::{ + address::{FromRaw, PhysicalAddress}, + device::RawDeviceMemoryMapping, + table::EntryLevelExt, + }, + util::OneTimeInit, +}; use memtables::FixedTables; use static_assertions::{const_assert_eq, const_assert_ne}; @@ -14,12 +21,7 @@ pub mod table; use crate::{ arch::x86_64::{intrinsics, mem::table::PageAttributes, registers::CR3}, - mem::{ - address::{FromRaw, IntoRaw, KernelImageObject}, - device::RawDeviceMemoryMapping, - table::EntryLevel, - PhysicalAddress, KERNEL_VIRT_OFFSET, - }, + mem::{address::KernelImageObject, table::EntryLevel, KERNEL_VIRT_OFFSET}, }; use self::table::{PageEntry, PageTable, L0, L1, L2, L3}; @@ -28,9 +30,10 @@ const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; const KERNEL_PHYS_BASE: usize = 0x400000; // Mapped at compile time -const KERNEL_L0_INDEX: usize = L0::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); -const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); -const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE; +const KERNEL_L0_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); +const KERNEL_L1_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); +const KERNEL_START_L2_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); // Must not be zero, should be at 4MiB const_assert_ne!(KERNEL_START_L2_INDEX, 0); @@ -103,7 +106,7 @@ unsafe fn unmap_early_page(address: usize) { panic!("Tried to unmap invalid early mapping: {:#x}", address); } - let l3i = L3::index(address - EARLY_MAPPING_OFFSET); + let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); assert!(EARLY_MAPPING_L3[l3i].is_present()); EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; @@ -169,14 +172,14 @@ pub(super) unsafe fn map_device_memory( ) -> Result { // debugln!("Map {}B @ {:#x}", size, base); let l3_aligned = base.page_align_down::(); - let l3_offset = L3::page_offset(base.into_raw()); - let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE; + let l3_offset = base.page_offset::(); + let page_count = (l3_offset + size).page_count::(); if page_count > 256 { // Large mapping, use L2 mapping instead let l2_aligned = base.page_align_down::(); - let l2_offset = L2::page_offset(base.into_raw()); - let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE; + let l2_offset = base.page_offset::(); + let page_count = (l2_offset + size).page_count::(); let base_address = map_device_memory_l2(l2_aligned, page_count)?; let address = base_address + l2_offset; @@ -211,8 +214,8 @@ pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { L3::SIZE => { for i in 0..map.page_count { let page = map.base_address + i * L3::SIZE; - let l2i = L2::index(page); - let l3i = L3::index(page); + let l2i = page.page_index::(); + let l3i = page.page_index::(); assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; intrinsics::flush_tlb_entry(page); @@ -224,7 +227,7 @@ pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { } pub(super) unsafe fn map_heap_block(index: usize, page: PhysicalAddress) { - if L2::page_offset(page.into_raw()) != 0 { + if !page.is_page_aligned_for::() { panic!("Attempted to map a misaligned 2MiB page"); } assert!(index < 512); diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index b3c74439..71119e90 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,15 +1,17 @@ //! x86-64-specific process address space management functions +use kernel_util::mem::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + table::EntryLevelExt, +}; use yggdrasil_abi::error::Error; use crate::{ arch::x86_64::intrinsics, mem::{ - address::{AsPhysicalAddress, IntoRaw}, phys, pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, table::{EntryLevel, MapAttributes, NextPageTable}, - PhysicalAddress, }, }; @@ -77,10 +79,10 @@ impl ProcessAddressSpaceImpl { entry: PageEntry, overwrite: bool, ) -> Result<(), Error> { - let l0i = L0::index(virt); - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); + let l0i = virt.page_index::(); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); let mut l1 = self.l0.get_mut_or_alloc(l0i)?; let mut l2 = l1.get_mut_or_alloc(l1i)?; @@ -99,10 +101,10 @@ impl ProcessAddressSpaceImpl { } fn pop_l3_entry(&mut self, virt: usize) -> Result { - let l0i = L0::index(virt); - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); + let l0i = virt.page_index::(); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); // TODO somehow drop tables if they're known to be empty? let mut l1 = self.l0.get_mut(l0i).ok_or(Error::DoesNotExist)?; @@ -120,10 +122,10 @@ impl ProcessAddressSpaceImpl { } fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { - let l0i = L0::index(virt); - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); + let l0i = virt.page_index::(); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); let l1 = self.l0.get(l0i)?; let l2 = l1.get(l1i)?; diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index 3ff305b5..884caef2 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -6,13 +6,12 @@ use core::{ use abi::error::Error; use bitflags::bitflags; +use kernel_util::mem::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; use crate::mem::{ - address::{AsPhysicalAddress, FromRaw}, phys, pointer::{PhysicalRef, PhysicalRefMut}, table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, - PhysicalAddress, }; bitflags! { @@ -67,19 +66,19 @@ impl NonTerminalEntryLevel for L2 { type NextLevel = L3; } -impl const EntryLevel for L0 { +impl EntryLevel for L0 { const SHIFT: usize = 39; } -impl const EntryLevel for L1 { +impl EntryLevel for L1 { const SHIFT: usize = 30; } -impl const EntryLevel for L2 { +impl EntryLevel for L2 { const SHIFT: usize = 21; } -impl const EntryLevel for L3 { +impl EntryLevel for L3 { const SHIFT: usize = 12; } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 0ca37092..6c2d474f 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -11,7 +11,15 @@ use device_api::{ Device, }; use git_version::git_version; -use kernel_util::{sync::SpinFence, util::OneTimeInit}; +use kernel_util::{ + mem::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::RawDeviceMemoryMapping, + table::EntryLevelExt, + }, + sync::SpinFence, + util::OneTimeInit, +}; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; mod acpi; @@ -50,12 +58,9 @@ use crate::{ Initrd, INITRD_DATA, }, mem::{ - address::{FromRaw, IntoRaw}, - device::RawDeviceMemoryMapping, heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, table::EntryLevel, - PhysicalAddress, }, }; @@ -117,6 +122,7 @@ pub static ARCHITECTURE: X86_64 = X86_64 { impl Architecture for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; type IrqNumber = IrqNumber; + type L3 = mem::table::L3; unsafe fn start_application_processors(&self) { if let Some(acpi) = self.acpi.try_get() { @@ -256,25 +262,23 @@ impl Architecture for X86_64 { } #[inline] - fn virtualize(address: PhysicalAddress) -> Result { - let raw: usize = address.into_raw(); - if raw < *mem::MEMORY_LIMIT.get() { - Ok(raw + RAM_MAPPING_OFFSET) + fn virtualize(address: u64) -> usize { + let address = address as usize; + if address < *mem::MEMORY_LIMIT.get() { + address + RAM_MAPPING_OFFSET } else { - errorln!("Invalid physical address: {:#x}", address); - Err(Error::InvalidMemoryOperation) + panic!("Invalid physical address: {:#x}", address); } } #[inline] - fn physicalize(address: usize) -> Result { + fn physicalize(address: usize) -> u64 { if address < RAM_MAPPING_OFFSET || address - RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() { - errorln!("Not a virtualized physical address: {:#x}", address); - return Err(Error::InvalidMemoryOperation); + panic!("Not a virtualized physical address: {:#x}", address); } - Ok(PhysicalAddress::from_raw(address - RAM_MAPPING_OFFSET)) + (address - RAM_MAPPING_OFFSET) as _ } fn external_interrupt_controller( diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index 970d776f..11ccfd99 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -2,6 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; +use kernel_util::mem::address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}; use crate::{ arch::{ @@ -15,12 +16,7 @@ use crate::{ }, Architecture, ArchitectureImpl, }, - mem::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw}, - phys, - pointer::PhysicalRefMut, - PhysicalAddress, - }, + mem::{phys, pointer::PhysicalRefMut}, task::Cpu, }; diff --git a/src/device/bus/pci/capability.rs b/src/device/bus/pci/capability.rs index 73132c90..d9e71baa 100644 --- a/src/device/bus/pci/capability.rs +++ b/src/device/bus/pci/capability.rs @@ -2,12 +2,13 @@ use abi::error::Error; use device_api::interrupt::{MessageInterruptController, MsiHandler}; - -use crate::{ - device::bus::pci::PciBaseAddress, - mem::{address::FromRaw, device::DeviceMemoryIoMut, PhysicalAddress}, +use kernel_util::mem::{ + address::{FromRaw, PhysicalAddress}, + device::DeviceMemoryIoMut, }; +use crate::device::bus::pci::PciBaseAddress; + use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; /// MSI-X capability query diff --git a/src/device/bus/pci/mod.rs b/src/device/bus/pci/mod.rs index d699e412..360589d8 100644 --- a/src/device/bus/pci/mod.rs +++ b/src/device/bus/pci/mod.rs @@ -5,7 +5,10 @@ use acpi_lib::mcfg::McfgEntry; use alloc::{boxed::Box, rc::Rc, vec::Vec}; use bitflags::bitflags; use device_api::Device; -use kernel_util::sync::IrqSafeSpinlock; +use kernel_util::{ + mem::address::{FromRaw, PhysicalAddress}, + sync::IrqSafeSpinlock, +}; use yggdrasil_abi::error::Error; pub mod capability; @@ -15,10 +18,7 @@ pub use space::{ ecam::PciEcam, PciConfigSpace, PciConfigurationSpace, PciLegacyConfigurationSpace, }; -use crate::{ - device::nvme, - mem::{address::FromRaw, PhysicalAddress}, -}; +use crate::device::nvme; bitflags! { /// Command register of the PCI configuration space diff --git a/src/device/bus/pci/space/ecam.rs b/src/device/bus/pci/space/ecam.rs index 7fac9d45..5f948e20 100644 --- a/src/device/bus/pci/space/ecam.rs +++ b/src/device/bus/pci/space/ecam.rs @@ -1,8 +1,7 @@ //! PCI Express ECAM interface +use kernel_util::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; use yggdrasil_abi::error::Error; -use crate::mem::{device::DeviceMemoryMapping, PhysicalAddress}; - use super::{PciAddress, PciConfigurationSpace}; /// PCI Express Enhanced Configuration Access Mechanism @@ -57,20 +56,5 @@ impl PciEcam { let this = Self::map(phys_addr)?; Ok(if this.is_valid() { Some(this) } else { None }) - - // if phys_addr + 0xFFF < 0x100000000 { - // // Probe without allocating a mapping - // let raw = PciRawEcam { - // virt_addr: phys_addr.virtualize(), - // }; - - // if !raw.is_valid() { - // return Ok(None); - // } - - // Self::map(phys_addr).map(Some) - // } else { - // todo!() - // } } } diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 843af4dd..f7e045e2 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -4,9 +4,10 @@ use core::ops::{Index, IndexMut}; use abi::error::Error; use device_api::Device; -use kernel_util::sync::IrqSafeSpinlock; - -use crate::mem::{device::RawDeviceMemoryMapping, PhysicalAddress}; +use kernel_util::{ + mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}, + sync::IrqSafeSpinlock, +}; use super::{DisplayDevice, DisplayDimensions}; diff --git a/src/device/nvme/command.rs b/src/device/nvme/command.rs index efb875ae..0759de59 100644 --- a/src/device/nvme/command.rs +++ b/src/device/nvme/command.rs @@ -2,12 +2,10 @@ use core::fmt::{self, Write}; +use kernel_util::mem::address::PhysicalAddress; use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike}; -use crate::{ - device::nvme::queue::PhysicalRegionPage, - mem::{address::IntoRaw, PhysicalAddress}, -}; +use crate::device::nvme::queue::PhysicalRegionPage; use super::queue::SubmissionQueueEntry; diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index 6c23bf9d..73447a92 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -4,7 +4,14 @@ use core::{mem::size_of, time::Duration}; use abi::error::Error; use alloc::{collections::BTreeMap, vec::Vec}; use device_api::{interrupt::MsiHandler, Device}; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + mem::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + device::{DeviceMemoryIo, DeviceMemoryIoMut}, + }, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, @@ -25,12 +32,7 @@ use crate::{ queue::{CompletionQueueEntry, SubmissionQueueEntry}, }, }, - mem::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw}, - device::{DeviceMemoryIo, DeviceMemoryIoMut}, - pointer::PhysicalRefMut, - PhysicalAddress, - }, + mem::pointer::PhysicalRefMut, task::runtime, }; diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 88693f55..925da5bd 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -12,12 +12,18 @@ use alloc::{ }; use bytemuck::{Pod, Zeroable}; use futures_util::Future; -use kernel_util::sync::IrqSafeSpinlock; +use kernel_util::{ + mem::{ + address::{IntoRaw, PhysicalAddress}, + table::EntryLevelExt, + }, + sync::IrqSafeSpinlock, +}; use static_assertions::const_assert; use crate::{ - arch::x86_64::mem::table::L3, - mem::{address::IntoRaw, phys, pointer::PhysicalRefMut, table::EntryLevel, PhysicalAddress}, + arch::L3, + mem::{phys, pointer::PhysicalRefMut}, task::runtime::QueueWaker, }; @@ -257,11 +263,11 @@ impl<'a> QueuePair<'a> { let sq_size = capacity * size_of::(); let cq_size = capacity * size_of::(); - let page_count = L3::page_count(sq_size) + L3::page_count(cq_size); + let page_count = sq_size.page_count::() + cq_size.page_count::(); let base = phys::alloc_pages_contiguous(page_count)?; let sq_base = base; - let cq_base = base.add(L3::align_up(sq_size)); + let cq_base = base.add(sq_size.page_align_up::()); debugln!( "Allocated queue pair: sq={:x?}, cq={:x?} ({} pages)", diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 3bd2d748..994a4012 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -2,12 +2,12 @@ use core::ptr::NonNull; -use kernel_util::util::OneTimeInit; +use kernel_util::{mem::address::PhysicalAddress, util::OneTimeInit}; use memfs::block::{self, BlockAllocator}; use vfs::NodeRef; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::mem::{phys, PhysicalAddress}; +use crate::mem::phys; pub mod devfs; pub mod sysfs; diff --git a/src/mem/address.rs b/src/mem/address.rs index d4d32cda..dfbf9f90 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -1,15 +1,9 @@ //! Address manipulation interfaces and utilities -use core::{ - fmt, - iter::Step, - mem::align_of, - ops::{Add, Deref, DerefMut, Sub}, -}; +use kernel_util::mem::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; -use crate::arch::{Architecture, ArchitectureImpl}; - -use super::{table::EntryLevel, KERNEL_VIRT_OFFSET}; +use super::KERNEL_VIRT_OFFSET; +use core::ops::{Deref, DerefMut}; /// Wrapper type to represent an object residing within the kernel #[repr(transparent)] @@ -17,35 +11,6 @@ pub struct KernelImageObject { inner: T, } -/// Wrapper type to represent a physical memory address -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -#[repr(transparent)] -pub struct PhysicalAddress(u64); - -/// Interface for converting addresses from their raw values to more specific types -#[const_trait] -pub trait FromRaw { - /// Converts a raw value into the address wrapper type - fn from_raw(value: T) -> Self; -} - -/// Interface for converting wrapper types into their raw address representations -#[const_trait] -pub trait IntoRaw { - /// Converts a wrapper type value into its raw address - fn into_raw(self) -> T; -} - -/// Interface for obtaining physical addresses of values -pub trait AsPhysicalAddress { - /// Returns the value's physical address. - /// - /// # Safety - /// - /// The caller must ensure the value has been constructed and obtained through proper means. - unsafe fn as_physical_address(&self) -> PhysicalAddress; -} - // KernelImageObject wrapper for objects inside the kernel impl KernelImageObject { @@ -78,146 +43,3 @@ impl DerefMut for KernelImageObject { &mut self.inner } } - -// - -impl PhysicalAddress { - /// Physical address of zero - pub const ZERO: Self = Self(0); - - /// Maximum representable physical address - pub const MAX: Self = Self(u64::MAX); - /// Minumum representable physical address - pub const MIN: Self = Self(u64::MIN); - - /// Applies an offset to the address - pub const fn add(self, offset: usize) -> Self { - Self(self.0 + offset as u64) - } - - /// Returns `true` if the address is zero - #[inline(always)] - pub const fn is_zero(self) -> bool { - self.0 == 0 - } - - /// Returns the offset this address has within a page of level `L` - pub const fn page_offset(self) -> usize { - L::page_offset(self.0 as usize) - } - - /// Aligns the address down to a boundary of a page of level `L` - pub const fn page_align_down(self) -> Self { - Self(self.0 & !(L::SIZE as u64 - 1)) - } - - /// Aligns the address up to a boundary of a page of level `L` - pub const fn page_align_up(self) -> Self { - Self((self.0 + L::SIZE as u64 - 1) & !(L::SIZE as u64 - 1)) - } - - /// Returns the page index this address has at level `L` - pub const fn page_index(self) -> usize { - L::index(self.0 as usize) - } - - /// Returns `true` if the address is aligned to a boundary of a page at level `L` - #[inline] - pub const fn is_aligned_for(self) -> bool { - self.0 as usize % align_of::() == 0 - } - - /// Converts a previously virtualized physical address back into its physical form. - /// - /// # Safety - /// - /// The caller must ensure the function only receives addresses obtained through - /// [PhysicalAddress::virtualize_raw] or - /// [super::pointer::PhysicalRef]/[super::pointer::PhysicalRefMut] facilities. - pub unsafe fn from_virtualized(address: usize) -> Self { - ArchitectureImpl::physicalize(address).unwrap() - } - - /// Converts the physical address to a virtual one - pub fn virtualize_raw(self) -> usize { - ArchitectureImpl::virtualize(self).unwrap() - } -} - -impl Add for PhysicalAddress { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -impl Sub for PhysicalAddress { - type Output = usize; - - fn sub(self, rhs: Self) -> Self::Output { - (self.0 - rhs.0) as usize - } -} - -// Conversions - -impl const FromRaw for PhysicalAddress { - fn from_raw(value: u64) -> Self { - Self(value) - } -} - -impl const FromRaw for PhysicalAddress { - fn from_raw(value: usize) -> Self { - Self(value as u64) - } -} - -impl const IntoRaw for PhysicalAddress { - fn into_raw(self) -> u64 { - self.0 - } -} - -impl const IntoRaw for PhysicalAddress { - fn into_raw(self) -> usize { - self.0 as usize - } -} - -impl From for u64 { - fn from(addr: PhysicalAddress) -> u64 { - addr.0 - } -} - -impl From for usize { - fn from(addr: PhysicalAddress) -> usize { - addr.0 as usize - } -} - -// Ranges - -impl Step for PhysicalAddress { - fn steps_between(_start: &Self, _end: &Self) -> Option { - todo!() - } - - fn forward_checked(start: Self, count: usize) -> Option { - start.0.checked_add(count as u64).map(Self) - } - - fn backward_checked(_start: Self, _count: usize) -> Option { - todo!() - } -} - -// fmt - -impl fmt::LowerHex for PhysicalAddress { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::LowerHex::fmt(&self.0, f) - } -} diff --git a/src/mem/heap.rs b/src/mem/heap.rs index fca4f7ce..5a7a146e 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -5,18 +5,18 @@ use core::{ ptr::{null_mut, NonNull}, }; +use kernel_util::sync::IrqSafeSpinlock; use linked_list_allocator::Heap; -use spinning_top::Spinlock; /// Kernel heap manager pub struct KernelAllocator { - inner: Spinlock, + inner: IrqSafeSpinlock, } impl KernelAllocator { const fn empty() -> Self { Self { - inner: Spinlock::new(Heap::empty()), + inner: IrqSafeSpinlock::new(Heap::empty()), } } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 3ee99e87..4c607c57 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -7,20 +7,18 @@ use core::{ }; use abi::error::Error; +use kernel_util::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; use crate::arch::{Architecture, ArchitectureImpl}; pub mod address; -pub mod device; pub mod heap; pub mod phys; pub mod pointer; pub mod process; pub mod table; -pub use address::PhysicalAddress; - -use self::{device::DeviceMemoryMapping, process::ProcessAddressSpace}; +use self::process::ProcessAddressSpace; /// Offset applied to the physical kernel image when translating it into the virtual address space pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 17a943bd..3cde108f 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -1,11 +1,8 @@ //! Physical memory manager implementation use abi::error::Error; +use kernel_util::mem::address::{FromRaw, IntoRaw, PhysicalAddress}; -use crate::mem::{ - address::{FromRaw, IntoRaw}, - pointer::PhysicalRefMut, - PhysicalAddress, -}; +use crate::mem::pointer::PhysicalRefMut; pub type BitmapWord = u64; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 3a0db3f8..b1837c28 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -3,11 +3,15 @@ use core::ops::Range; use abi::error::Error; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + mem::address::{FromRaw, IntoRaw, PhysicalAddress}, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use crate::{ arch::{Architecture, ARCHITECTURE}, - mem::{address::IntoRaw, phys::reserved::is_reserved}, + mem::phys::reserved::is_reserved, }; use self::{ @@ -15,8 +19,6 @@ use self::{ reserved::reserve_region, }; -use super::{address::FromRaw, PhysicalAddress}; - // 8 * 4096 bits per page, 1 page per bit const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PAGE_LIMIT * 4096); diff --git a/src/mem/pointer.rs b/src/mem/pointer.rs index 4ab32f3a..3d3bc55e 100644 --- a/src/mem/pointer.rs +++ b/src/mem/pointer.rs @@ -5,7 +5,7 @@ use core::{ ops::{Deref, DerefMut}, }; -use super::{address::AsPhysicalAddress, PhysicalAddress}; +use kernel_util::mem::address::{AsPhysicalAddress, PhysicalAddress}; /// Wrapper for immutably accessing a value at a physical address #[repr(transparent)] diff --git a/src/mem/table.rs b/src/mem/table.rs index 20b857b4..1555bdd3 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -3,6 +3,7 @@ use core::ops::{Deref, DerefMut}; use abi::error::Error; use bitflags::bitflags; +pub use kernel_util::mem::table::EntryLevel; // TODO EXECUTABLE bitflags! { @@ -36,38 +37,6 @@ pub trait NextPageTable { fn get(&self, index: usize) -> Option; } -/// Interface for a single level of address translation -#[const_trait] -pub trait EntryLevel: Copy { - /// The right shift needed to obtain an index of an entry at this level from an address - const SHIFT: usize; - /// The size of a page at this entry level - const SIZE: usize = 1 << Self::SHIFT; - - /// Returns the index into a page table for a given address - #[inline] - fn index(addr: usize) -> usize { - (addr >> Self::SHIFT) & 0x1FF - } - /// Returns the offset of an address from the page start at current level - #[inline] - fn page_offset(addr: usize) -> usize { - addr & (Self::SIZE - 1) - } - - /// Aligns the `addr` up to the level page boundary - #[inline] - fn align_up(addr: usize) -> usize { - (addr + Self::SIZE - 1) & !(Self::SIZE - 1) - } - - /// Returns the page count needed to fully contain a block of size `addr` - #[inline] - fn page_count(addr: usize) -> usize { - (addr + Self::SIZE - 1) / Self::SIZE - } -} - /// Tag trait to mark that the page table level may point to a next-level table pub trait NonTerminalEntryLevel: EntryLevel { /// Tag type of the level this entry level may point to From 6aa18a1fa202768f6ce366778e1bd2a4bc410571 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 10 Dec 2023 20:54:15 +0200 Subject: [PATCH 112/211] mem: implement PageBox --- lib/kernel-util/Cargo.toml | 1 + lib/kernel-util/src/api.rs | 7 +- lib/kernel-util/src/lib.rs | 9 +- lib/kernel-util/src/mem/mod.rs | 218 ++++++++++++++++++++ {src => lib/kernel-util/src}/mem/pointer.rs | 4 +- lib/vfs/src/lib.rs | 8 +- src/arch/x86_64/acpi.rs | 7 +- src/arch/x86_64/mem/process.rs | 2 +- src/arch/x86_64/mem/table.rs | 6 +- src/arch/x86_64/smp.rs | 7 +- src/device/nvme/drive.rs | 19 +- src/device/nvme/mod.rs | 80 ++++--- src/device/nvme/queue.rs | 106 +++------- src/mem/mod.rs | 1 - src/mem/phys/manager.rs | 7 +- src/mem/phys/mod.rs | 10 + src/proc/elf.rs | 8 +- src/proc/exec.rs | 10 +- tools/gentables/src/x86_64.rs | 9 +- 19 files changed, 342 insertions(+), 177 deletions(-) rename {src => lib/kernel-util/src}/mem/pointer.rs (97%) diff --git a/lib/kernel-util/Cargo.toml b/lib/kernel-util/Cargo.toml index 98dee9e8..81ac1c42 100644 --- a/lib/kernel-util/Cargo.toml +++ b/lib/kernel-util/Cargo.toml @@ -7,3 +7,4 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +log = "0.4.20" diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index 99aded18..aa43441f 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -6,9 +6,10 @@ extern "Rust" { pub fn __acquire_irq_guard() -> bool; pub fn __release_irq_guard(mask: bool); - pub fn __allocate_2m_page() -> u64; - pub fn __allocate_page() -> u64; - pub fn __free_page(page: u64); + pub fn __allocate_2m_page() -> Result; + pub fn __allocate_page() -> Result; + pub fn __allocate_contiguous_pages(count: usize) -> Result; + pub fn __free_page(page: PhysicalAddress); pub fn __virtualize(phys: u64) -> usize; pub fn __physicalize(virt: usize) -> u64; diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 7a71340a..7f83491a 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -1,5 +1,12 @@ #![no_std] -#![feature(maybe_uninit_slice, step_trait, const_trait_impl, effects)] +#![feature( + maybe_uninit_slice, + step_trait, + const_trait_impl, + effects, + slice_ptr_get, + strict_provenance +)] extern crate alloc; diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 81f4df8c..3140f216 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -1,3 +1,221 @@ +use core::{ + alloc::Layout, + fmt, + mem::{size_of, MaybeUninit}, + ops::{Deref, DerefMut}, +}; + +use yggdrasil_abi::error::Error; + +use crate::api::{__allocate_contiguous_pages, __free_page, __physicalize}; + +use self::address::{AsPhysicalAddress, PhysicalAddress}; + pub mod address; pub mod device; +pub mod pointer; pub mod table; + +pub struct PageBox { + value: *mut T, + page_count: usize, +} + +impl PageBox { + #[inline] + fn alloc_slice(count: usize) -> Result<(PhysicalAddress, usize), Error> { + // TODO hardcoded page sizes + let layout = Layout::array::(count).unwrap(); + let page_count = (layout.size() + 0xFFF) / 0x1000; + Ok(( + unsafe { __allocate_contiguous_pages(page_count) }?, + page_count, + )) + } + + #[inline] + fn alloc() -> Result<(PhysicalAddress, usize), Error> { + let page_count = (size_of::() + 0xFFF) / 0x1000; + Ok(( + unsafe { __allocate_contiguous_pages(page_count) }?, + page_count, + )) + } + + pub fn new(init: T) -> Result, Error> { + let (base, page_count) = Self::alloc()?; + let value = base.virtualize_raw() as *mut T; + + unsafe { + value.write(init); + } + + let result = PageBox { value, page_count }; + result.trace_created(); + Ok(result) + } + + pub fn new_slice(item: T, count: usize) -> Result, Error> + where + T: Copy, + { + let (base, page_count) = Self::alloc_slice(count)?; + let base_virt_ptr = base.virtualize_raw() as *mut T; + let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); + + for i in 0..count { + unsafe { + value.get_unchecked_mut(i).write(item); + } + } + + let result = PageBox { value, page_count }; + result.trace_created(); + Ok(result) + } + + pub fn new_uninit() -> Result>, Error> { + let (base, page_count) = PageBox::>::alloc()?; + let value = base.virtualize_raw() as *mut MaybeUninit; + let result = PageBox { value, page_count }; + result.trace_created(); + Ok(result) + } + + pub fn new_uninit_slice(count: usize) -> Result]>, Error> { + let (base, page_count) = PageBox::>::alloc_slice(count)?; + let base_virt_ptr = base.virtualize_raw() as *mut MaybeUninit; + let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); + let result = PageBox { value, page_count }; + result.trace_created(); + Ok(result) + } +} + +impl PageBox { + #[inline] + pub fn as_ptr(&self) -> *const T { + self.value as _ + } + + #[inline] + fn trace_created(&self) { + log::trace!( + "Alloc PageBox<{}> @ {:p}, {}", + core::any::type_name::(), + self.value, + self.page_count + ); + } + + #[inline] + fn trace_dropped(&self) { + log::trace!( + "Free PageBox<{}> @ {:p}, {}", + core::any::type_name::(), + self.value, + self.page_count + ); + } +} + +impl PageBox> { + pub unsafe fn assume_init(self) -> PageBox { + // SAFETY: Memory-safe, as: + // 1. MaybeUninit is transparent + // 2. self.value still points to the same memory and is not deallocated + let page_count = self.page_count; + let value = MaybeUninit::assume_init_mut(&mut *self.value); + + // Prevent deallocation of the PageBox with MaybeUninit + core::mem::forget(self); + + PageBox { value, page_count } + } +} + +impl PageBox<[MaybeUninit]> { + pub unsafe fn assume_init_slice(self) -> PageBox<[T]> { + // SAFETY: Memory-safe, as: + // 1. MaybeUninit is transparent + // 2. self.value still points to the same memory and is not deallocated + let page_count = self.page_count; + let value = MaybeUninit::slice_assume_init_mut(&mut *self.value); + + core::mem::forget(self); + + PageBox { value, page_count } + } + + pub unsafe fn assume_init_slice_ref(&self) -> &[T] { + MaybeUninit::slice_assume_init_ref(self.deref()) + } + + pub unsafe fn assume_init_slice_mut(&mut self) -> &mut [T] { + MaybeUninit::slice_assume_init_mut(self.deref_mut()) + } +} + +impl AsPhysicalAddress for PageBox { + #[inline] + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress(__physicalize(self.value.addr())) + } +} + +impl Deref for PageBox { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + unsafe { &*self.value } + } +} + +impl DerefMut for PageBox { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.value } + } +} + +impl Drop for PageBox { + fn drop(&mut self) { + self.trace_dropped(); + unsafe { + core::ptr::drop_in_place(self.value); + } + // SAFETY: Safe, pointer obtained through "virtualize" + let base = PhysicalAddress(unsafe { __physicalize(self.value.addr()) }); + for i in 0..self.page_count { + // SAFETY: Safe, page allocated only by this PageBox + unsafe { + __free_page(base.add(0x1000 * i)); + } + } + } +} + +impl fmt::Pointer for PageBox { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.value.fmt(f) + } +} + +impl fmt::Debug for PageBox { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(self.deref(), f) + } +} + +impl fmt::Display for PageBox { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self.deref(), f) + } +} + +unsafe impl Send for PageBox {} +unsafe impl Sync for PageBox {} diff --git a/src/mem/pointer.rs b/lib/kernel-util/src/mem/pointer.rs similarity index 97% rename from src/mem/pointer.rs rename to lib/kernel-util/src/mem/pointer.rs index 3d3bc55e..db89f6df 100644 --- a/src/mem/pointer.rs +++ b/lib/kernel-util/src/mem/pointer.rs @@ -1,11 +1,9 @@ -//! Pointer utilities and interfaces - use core::{ fmt, ops::{Deref, DerefMut}, }; -use kernel_util::mem::address::{AsPhysicalAddress, PhysicalAddress}; +use super::address::{AsPhysicalAddress, PhysicalAddress}; /// Wrapper for immutably accessing a value at a physical address #[repr(transparent)] diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 33704419..7b265746 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -3,13 +3,7 @@ #![cfg_attr(not(test), no_std)] #![allow(clippy::new_ret_no_self)] #![deny(missing_docs)] -#![feature( - trait_upcasting, - if_let_guard, - maybe_uninit_slice, - trait_alias, - let_chains -)] +#![feature(if_let_guard, maybe_uninit_slice, trait_alias, let_chains)] #[cfg(test)] extern crate hosted_tests; diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 68c23fe2..e0d2ef95 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -16,7 +16,10 @@ use device_api::{ Device, }; use kernel_util::{ - mem::address::{FromRaw, PhysicalAddress}, + mem::{ + address::{FromRaw, PhysicalAddress}, + pointer::PhysicalRef, + }, sync::IrqSafeSpinlock, util::OneTimeInit, }; @@ -27,7 +30,7 @@ use crate::{ x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, Architecture, CpuMessage, ARCHITECTURE, }, - mem::{heap::GLOBAL_HEAP, pointer::PhysicalRef, read_memory, write_memory}, + mem::{heap::GLOBAL_HEAP, read_memory, write_memory}, }; use super::intrinsics; diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index 71119e90..aba47e2e 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,6 +1,7 @@ //! x86-64-specific process address space management functions use kernel_util::mem::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + pointer::PhysicalRefMut, table::EntryLevelExt, }; use yggdrasil_abi::error::Error; @@ -9,7 +10,6 @@ use crate::{ arch::x86_64::intrinsics, mem::{ phys, - pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, table::{EntryLevel, MapAttributes, NextPageTable}, }, diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index 884caef2..f6db3d8d 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -6,11 +6,13 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -use kernel_util::mem::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; +use kernel_util::mem::{ + address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, + pointer::{PhysicalRef, PhysicalRefMut}, +}; use crate::mem::{ phys, - pointer::{PhysicalRef, PhysicalRefMut}, table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, }; diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index 11ccfd99..fa69e85f 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -2,7 +2,10 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; -use kernel_util::mem::address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}; +use kernel_util::mem::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + pointer::PhysicalRefMut, +}; use crate::{ arch::{ @@ -16,7 +19,7 @@ use crate::{ }, Architecture, ArchitectureImpl, }, - mem::{phys, pointer::PhysicalRefMut}, + mem::phys, task::Cpu, }; diff --git a/src/device/nvme/drive.rs b/src/device/nvme/drive.rs index 4914c8a9..be2daf60 100644 --- a/src/device/nvme/drive.rs +++ b/src/device/nvme/drive.rs @@ -20,7 +20,7 @@ impl NvmeDrive { nsid: u32, ) -> Result<&'static NvmeDrive, NvmeError> { let admin_q = controller.admin_q.get(); - let identify = admin_q.request(IdentifyNamespaceRequest { nsid })?.await?; + let identify = admin_q.request(IdentifyNamespaceRequest { nsid }).await?; let current_lba_format_idx = identify.current_lba_fmt_idx(); let current_lba_format = identify.lba_fmt(current_lba_format_idx).unwrap(); @@ -49,23 +49,6 @@ impl NvmeDrive { Ok(dev) } - - // TODO proper interface for reading/writing blocks - // pub async fn read_block( - // &self, - // lba: u64, - // block: &mut PhysicalRefMut<'_, [u8]>, - // ) -> Result<(), NvmeError> { - // self.controller.read_block(self.nsid, lba, block).await - // } - - // pub async fn write_block( - // &self, - // lba: u64, - // block: &PhysicalRefMut<'_, [u8]>, - // ) -> Result<(), NvmeError> { - // self.controller.write_block(self.nsid, lba, block).await - // } } impl BlockDevice for NvmeDrive { diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index 73447a92..fa705501 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -6,7 +6,7 @@ use alloc::{collections::BTreeMap, vec::Vec}; use device_api::{interrupt::MsiHandler, Device}; use kernel_util::{ mem::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, DeviceMemoryIoMut}, }, sync::IrqSafeSpinlock, @@ -32,7 +32,6 @@ use crate::{ queue::{CompletionQueueEntry, SubmissionQueueEntry}, }, }, - mem::pointer::PhysicalRefMut, task::runtime, }; @@ -117,8 +116,8 @@ register_structs! { pub struct NvmeController { regs: IrqSafeSpinlock>, - admin_q: OneTimeInit>, - ioqs: OneTimeInit>>, + admin_q: OneTimeInit, + ioqs: OneTimeInit>, vector_table: IrqSafeSpinlock>, drive_table: IrqSafeSpinlock>, controller_id: OneTimeInit, @@ -126,6 +125,12 @@ pub struct NvmeController { doorbell_shift: usize, } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IoDirection { + Read, + Write, +} + impl Regs { unsafe fn doorbell_ptr(&self, shift: usize, completion: bool, queue_index: usize) -> *mut u32 { let doorbell_base = (self as *const Regs as *mut Regs).addr() + 0x1000; @@ -141,7 +146,7 @@ impl NvmeController { let admin_q = self.admin_q.get(); // Identify the controller - let _identify = admin_q.request(IdentifyControllerRequest)?.await?; + let _identify = admin_q.request(IdentifyControllerRequest).await?; // TODO do something with identify_controller @@ -185,7 +190,7 @@ impl NvmeController { let admin_q = self.admin_q.get(); let namespaces = admin_q - .request(IdentifyActiveNamespaceIdListRequest { start_id: 0 })? + .request(IdentifyActiveNamespaceIdListRequest { start_id: 0 }) .await?; let count = namespaces.entries.iter().position(|&x| x == 0).unwrap(); @@ -205,50 +210,37 @@ impl NvmeController { Ok(()) } - pub async fn read_block( + // TODO sane methods for IO + pub async fn perform_io( &'static self, nsid: u32, lba: u64, - buffer: &mut PhysicalRefMut<'_, [u8]>, + buffer_address: PhysicalAddress, + direction: IoDirection, ) -> Result<(), NvmeError> { let ioq = &self.ioqs.get()[0]; - let buffer_address = unsafe { buffer.as_physical_address() }; - debugln!("read nsid={}, lba={:#x}", nsid, lba); - let cmd_id = ioq.submit( - IoRead { - nsid, - lba, - count: 1, - }, - &[buffer_address], - true, - ); - - ioq.wait_for_completion(cmd_id, ()).await?; - - Ok(()) - } - - pub async fn write_block( - &'static self, - nsid: u32, - lba: u64, - buffer: &PhysicalRefMut<'_, [u8]>, - ) -> Result<(), NvmeError> { - let ioq = &self.ioqs.get()[0]; - let buffer_address = unsafe { buffer.as_physical_address() }; - - debugln!("write nsid={}, lba={:#x}", nsid, lba); - let cmd_id = ioq.submit( - IoWrite { - nsid, - lba, - count: 1, - }, - &[buffer_address], - true, - ); + debugln!("{:?} nsid={}, lba={:#x}", direction, nsid, lba); + let cmd_id = match direction { + IoDirection::Read => ioq.submit( + IoRead { + nsid, + lba, + count: 1, + }, + &[buffer_address], + true, + ), + IoDirection::Write => ioq.submit( + IoWrite { + nsid, + lba, + count: 1, + }, + &[buffer_address], + true, + ), + }; ioq.wait_for_completion(cmd_id, ()).await?; diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 925da5bd..3d027954 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -14,18 +14,14 @@ use bytemuck::{Pod, Zeroable}; use futures_util::Future; use kernel_util::{ mem::{ - address::{IntoRaw, PhysicalAddress}, - table::EntryLevelExt, + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + PageBox, }, sync::IrqSafeSpinlock, }; use static_assertions::const_assert; -use crate::{ - arch::L3, - mem::{phys, pointer::PhysicalRefMut}, - task::runtime::QueueWaker, -}; +use crate::task::runtime::QueueWaker; use super::{ command::{Command, Request}, @@ -74,9 +70,8 @@ pub struct CompletionQueueEntry { dw: [u32; 4], } -pub struct Queue<'a, T> { - data: PhysicalRefMut<'a, [T]>, - +pub struct Queue { + data: PageBox<[T]>, mask: usize, head: usize, tail: usize, @@ -86,28 +81,26 @@ pub struct Queue<'a, T> { tail_doorbell: *mut u32, } -struct Inner<'a> { - sq: Queue<'a, SubmissionQueueEntry>, - cq: Queue<'a, CompletionQueueEntry>, +struct Inner { + sq: Queue, + cq: Queue, completed: BTreeMap, pending: BTreeSet, } -// TODO PageBox? -#[allow(unused)] -pub struct QueuePair<'a> { - base: PhysicalAddress, - page_count: usize, +pub struct QueuePair { id: u32, + #[allow(unused)] vector: usize, + sq_base: PhysicalAddress, cq_base: PhysicalAddress, completion_notify: QueueWaker, - inner: IrqSafeSpinlock>, + inner: IrqSafeSpinlock, } const_assert!(size_of::().is_power_of_two()); @@ -166,25 +159,23 @@ impl CompletionQueueEntry { } } -impl<'a, T> Queue<'a, T> { - pub unsafe fn from_raw_parts( - base: PhysicalAddress, - len: usize, +impl Queue { + pub fn new( + data: PageBox<[T]>, head_doorbell: *mut u32, tail_doorbell: *mut u32, phase: bool, ) -> Self { - // Submission queues have tail doorbells, completion queues have head doorbells assert!( (head_doorbell.is_null() && !tail_doorbell.is_null()) || (!head_doorbell.is_null() && tail_doorbell.is_null()) ); Self { - data: PhysicalRefMut::map_slice(base, len), - mask: len - 1, + mask: data.len() - 1, head: 0, tail: 0, + data, head_doorbell, tail_doorbell, phase, @@ -246,13 +237,9 @@ impl<'a, T> Queue<'a, T> { wrapped } - - // pub fn is_empty(&self) -> bool { - // self.head == self.tail - // } } -impl<'a> QueuePair<'a> { +impl QueuePair { pub fn new( id: u32, vector: usize, @@ -260,24 +247,16 @@ impl<'a> QueuePair<'a> { sq_doorbell: *mut u32, cq_doorbell: *mut u32, ) -> Result { - let sq_size = capacity * size_of::(); - let cq_size = capacity * size_of::(); + let sq_data = PageBox::new_slice(SubmissionQueueEntry::zeroed(), capacity)?; + let cq_data = PageBox::new_slice(CompletionQueueEntry::zeroed(), capacity)?; - let page_count = sq_size.page_count::() + cq_size.page_count::(); - let base = phys::alloc_pages_contiguous(page_count)?; + let sq_base = unsafe { sq_data.as_physical_address() }; + let cq_base = unsafe { cq_data.as_physical_address() }; - let sq_base = base; - let cq_base = base.add(sq_size.page_align_up::()); + debugln!("Allocated queue pair: sq={:p}, cq={:p}", sq_data, cq_data); - debugln!( - "Allocated queue pair: sq={:x?}, cq={:x?} ({} pages)", - sq_base..sq_base.add(sq_size), - cq_base..cq_base.add(cq_size), - page_count - ); - - let sq = unsafe { Queue::from_raw_parts(sq_base, capacity, null_mut(), sq_doorbell, true) }; - let cq = unsafe { Queue::from_raw_parts(cq_base, capacity, cq_doorbell, null_mut(), true) }; + let sq = Queue::new(sq_data, null_mut(), sq_doorbell, true); + let cq = Queue::new(cq_data, cq_doorbell, null_mut(), true); let inner = IrqSafeSpinlock::new(Inner { sq, @@ -291,8 +270,6 @@ impl<'a> QueuePair<'a> { id, vector, - base, - page_count, sq_base, cq_base, inner, @@ -313,12 +290,9 @@ impl<'a> QueuePair<'a> { &'r self, command_id: u32, result: T, - ) -> impl Future> + 'r - where - 'r: 'a, - { + ) -> impl Future> + 'r { struct Fut<'r, R: Unpin + 'r> { - this: &'r QueuePair<'r>, + this: &'r QueuePair, response: Option, command_id: u32, } @@ -386,34 +360,22 @@ impl<'a> QueuePair<'a> { pub fn request_no_data<'r, C: Command>( &'r self, req: C, - ) -> impl Future> + 'r - where - 'r: 'a, - { + ) -> impl Future> + 'r { let command_id = self.submit(req, &[], true); self.wait_for_completion(command_id, ()) } - pub fn request<'r, R: Request>( + pub async fn request<'r, R: Request>( &'r self, req: R, - ) -> Result< - impl Future, CommandError>>, - NvmeError, - > + ) -> Result, NvmeError> where R::Response: 'r, - 'r: 'a, { - assert_ne!(size_of::(), 0); - assert!(size_of::() <= 0x1000); - - let page = phys::alloc_page().map_err(NvmeError::MemoryError)?; - // TODO PageBox - let response = unsafe { PhysicalRefMut::map(page) }; - - let command_id = self.submit(req, &[page], true); - Ok(self.wait_for_completion(command_id, response)) + let response = PageBox::new_uninit().map_err(NvmeError::MemoryError)?; + let command_id = self.submit(req, &[unsafe { response.as_physical_address() }], true); + let result = self.wait_for_completion(command_id, response).await?; + Ok(unsafe { result.assume_init() }) } pub fn process_completions(&self) -> usize { diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 4c607c57..307c9cb5 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -14,7 +14,6 @@ use crate::arch::{Architecture, ArchitectureImpl}; pub mod address; pub mod heap; pub mod phys; -pub mod pointer; pub mod process; pub mod table; diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 3cde108f..9e8106ec 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -1,8 +1,9 @@ //! Physical memory manager implementation use abi::error::Error; -use kernel_util::mem::address::{FromRaw, IntoRaw, PhysicalAddress}; - -use crate::mem::pointer::PhysicalRefMut; +use kernel_util::mem::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + pointer::PhysicalRefMut, +}; pub type BitmapWord = u64; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index b1837c28..283e829a 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -208,3 +208,13 @@ fn kernel_physical_memory_region() -> PhysicalMemoryRegion { PhysicalMemoryRegion { base, size } } + +#[no_mangle] +fn __allocate_contiguous_pages(count: usize) -> Result { + alloc_pages_contiguous(count) +} + +#[no_mangle] +unsafe fn __free_page(page: PhysicalAddress) { + free_page(page) +} diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 633b8cef..50193b1b 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -7,16 +7,12 @@ use elf::{ segment::ProgramHeader, ElfStream, ParseError, }; +use kernel_util::mem::pointer::{PhysicalRef, PhysicalRefMut}; use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ - mem::{ - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - process::ProcessAddressSpace, - table::MapAttributes, - }, + mem::{phys, process::ProcessAddressSpace, table::MapAttributes}, task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 24d88b44..dacc0ca3 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -9,13 +9,11 @@ use abi::{ process::ProgramArgumentInner, }; use alloc::{string::String, sync::Arc, vec::Vec}; +use kernel_util::mem::pointer::PhysicalRefMut; use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ - mem::{ - phys, pointer::PhysicalRefMut, process::ProcessAddressSpace, table::MapAttributes, - ForeignPointer, - }, + mem::{phys, process::ProcessAddressSpace, table::MapAttributes, ForeignPointer}, proc, task::{ context::TaskContextImpl, @@ -190,7 +188,9 @@ pub fn load>( let count = file.read(&mut head)?; let head = &head[..count]; - if let Some(shebang) = head.strip_prefix(b"#!") && let Some((shebang, _)) = shebang.split_once(|&ch| ch == b'\n') { + if let Some(shebang) = head.strip_prefix(b"#!") + && let Some((shebang, _)) = shebang.split_once(|&ch| ch == b'\n') + { let shebang = core::str::from_utf8(shebang).map_err(|_| Error::InvalidFile)?; let mut shebang_args = shebang.split(' ').collect::>(); if shebang_args.is_empty() || shebang_args.len() >= 8 { diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs index 725636e3..87aa5c45 100644 --- a/tools/gentables/src/x86_64.rs +++ b/tools/gentables/src/x86_64.rs @@ -5,12 +5,7 @@ use std::{ }; use bitflags::bitflags; -use bytemuck::Zeroable; -use elf::{ - abi::{PF_W, PF_X, PT_LOAD}, - endian::AnyEndian, - ElfStream, -}; +use elf::{abi::PT_LOAD, endian::AnyEndian, ElfStream}; use memtables::x86_64::{FixedTables, KERNEL_L3_COUNT}; use crate::{GenData, GenError}; @@ -36,7 +31,7 @@ pub struct X8664Builder { } impl PageFlags { - fn from_elf(flags: u32) -> Self { + fn from_elf(_flags: u32) -> Self { let mut out = Self::empty(); // if flags & PF_W != 0 { out |= Self::WRITABLE; From 2da98eaaa8e07639f3c2538c4180ec1f66887124 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 10 Dec 2023 21:25:54 +0200 Subject: [PATCH 113/211] dev/pci: make PCI driver an external crate --- Cargo.toml | 3 + driver/bus/pci/Cargo.toml | 16 +++ .../pci => driver/bus/pci/src}/capability.rs | 7 +- .../pci/mod.rs => driver/bus/pci/src/lib.rs | 89 ++++++++++++--- .../pci => driver/bus/pci/src}/space/ecam.rs | 0 .../pci => driver/bus/pci/src}/space/mod.rs | 3 +- src/arch/x86_64/mod.rs | 13 ++- src/device/bus/mod.rs | 2 - src/device/nvme/mod.rs | 102 +++++++++--------- 9 files changed, 158 insertions(+), 77 deletions(-) create mode 100644 driver/bus/pci/Cargo.toml rename {src/device/bus/pci => driver/bus/pci/src}/capability.rs (95%) rename src/device/bus/pci/mod.rs => driver/bus/pci/src/lib.rs (79%) rename {src/device/bus/pci => driver/bus/pci/src}/space/ecam.rs (100%) rename {src/device/bus/pci => driver/bus/pci/src}/space/mod.rs (99%) diff --git a/Cargo.toml b/Cargo.toml index bbb1ec79..cfb3d84b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,9 @@ memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } +# Drivers +ygg_driver_pci = { path = "driver/bus/pci" } + atomic_enum = "0.2.0" bitflags = "2.3.3" linked_list_allocator = "0.10.5" diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml new file mode 100644 index 00000000..5268dc89 --- /dev/null +++ b/driver/bus/pci/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ygg_driver_pci" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } +kernel-util = { path = "../../../lib/kernel-util" } + +log = "0.4.20" +bitflags = "2.3.3" + +acpi = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" } diff --git a/src/device/bus/pci/capability.rs b/driver/bus/pci/src/capability.rs similarity index 95% rename from src/device/bus/pci/capability.rs rename to driver/bus/pci/src/capability.rs index d9e71baa..ddd324bd 100644 --- a/src/device/bus/pci/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -1,15 +1,14 @@ //! PCI capability structures and queries -use abi::error::Error; use device_api::interrupt::{MessageInterruptController, MsiHandler}; use kernel_util::mem::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIoMut, }; - -use crate::device::bus::pci::PciBaseAddress; +use yggdrasil_abi::error::Error; use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; +use crate::PciBaseAddress; /// MSI-X capability query pub struct MsiXCapability; @@ -61,7 +60,7 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { return Err(Error::InvalidOperation); }; - debugln!("MSI-X table address: {:#x}", base + table_offset); + log::debug!("MSI-X table address: {:#x}", base + table_offset); unsafe { DeviceMemoryIoMut::map_slice(PhysicalAddress::from_raw(base + table_offset), table_size) diff --git a/src/device/bus/pci/mod.rs b/driver/bus/pci/src/lib.rs similarity index 79% rename from src/device/bus/pci/mod.rs rename to driver/bus/pci/src/lib.rs index 360589d8..b2ae181e 100644 --- a/src/device/bus/pci/mod.rs +++ b/driver/bus/pci/src/lib.rs @@ -1,8 +1,12 @@ //! PCI/PCIe bus interfaces +#![no_std] + +extern crate alloc; + use core::fmt; -use acpi_lib::mcfg::McfgEntry; -use alloc::{boxed::Box, rc::Rc, vec::Vec}; +use acpi::mcfg::McfgEntry; +use alloc::{rc::Rc, vec::Vec}; use bitflags::bitflags; use device_api::Device; use kernel_util::{ @@ -18,8 +22,6 @@ pub use space::{ ecam::PciEcam, PciConfigSpace, PciConfigurationSpace, PciLegacyConfigurationSpace, }; -use crate::device::nvme; - bitflags! { /// Command register of the PCI configuration space pub struct PciCommandRegister: u16 { @@ -100,10 +102,15 @@ pub struct PciDeviceInfo { pub config_space: PciConfigSpace, } -/// Interface for "taking" PCI devices from the bus -pub trait FromPciBus: Sized { - /// Constructs an instance of a driver for the device using the information provided - fn from_pci_bus(info: &PciDeviceInfo) -> Result; +pub enum PciMatch { + Generic(fn(&PciDeviceInfo) -> bool), + Class(u8, Option, Option), +} + +pub struct PciDriver { + name: &'static str, + check: PciMatch, + probe: fn(&PciDeviceInfo) -> Result<&'static dyn Device, Error>, } /// Used to store PCI bus devices which were enumerated by the kernel @@ -296,28 +303,76 @@ fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> { let config = &device.info.config_space; - debugln!( + log::debug!( "{}: {:04x}:{:04x}", device.info.address, config.vendor_id(), config.device_id() ); - // Match by class - match (config.class_code(), config.subclass(), config.prog_if()) { - (0x01, 0x08, 0x02) => { - let dev = Box::leak(Box::new(nvme::NvmeController::from_pci_bus(&device.info)?)); + let class = config.class_code(); + let subclass = config.subclass(); + let prog_if = config.prog_if(); + + let drivers = PCI_DRIVERS.lock(); + for driver in drivers.iter() { + if driver + .check + .check_device(&device.info, class, subclass, prog_if) + { + // TODO add the device to the bus + log::debug!(" -> {:?}", driver.name); + let device = (driver.probe)(&device.info)?; unsafe { - dev.init()?; + device.init()?; } - } - _ => { - debugln!(" -> No driver"); + } else { + log::debug!(" -> No driver"); } } Ok(()) } +impl PciMatch { + pub fn check_device(&self, info: &PciDeviceInfo, class: u8, subclass: u8, prog_if: u8) -> bool { + match self { + Self::Generic(f) => f(info), + &Self::Class(class_, Some(subclass_), Some(prog_if_)) => { + class_ == class && subclass_ == subclass && prog_if_ == prog_if + } + &Self::Class(class_, Some(subclass_), _) => class_ == class && subclass_ == subclass, + &Self::Class(class_, _, _) => class_ == class, + } + } +} + +pub fn register_class_driver( + name: &'static str, + class: u8, + subclass: Option, + prog_if: Option, + probe: fn(&PciDeviceInfo) -> Result<&'static dyn Device, Error>, +) { + PCI_DRIVERS.lock().push(PciDriver { + name, + check: PciMatch::Class(class, subclass, prog_if), + probe, + }); +} + +pub fn register_generic_driver( + name: &'static str, + check: fn(&PciDeviceInfo) -> bool, + probe: fn(&PciDeviceInfo) -> Result<&'static dyn Device, Error>, +) { + PCI_DRIVERS.lock().push(PciDriver { + name, + check: PciMatch::Generic(check), + probe, + }); +} + +static PCI_DRIVERS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); static PCI_MANAGER: IrqSafeSpinlock = IrqSafeSpinlock::new(PciBusManager::new()); diff --git a/src/device/bus/pci/space/ecam.rs b/driver/bus/pci/src/space/ecam.rs similarity index 100% rename from src/device/bus/pci/space/ecam.rs rename to driver/bus/pci/src/space/ecam.rs diff --git a/src/device/bus/pci/space/mod.rs b/driver/bus/pci/src/space/mod.rs similarity index 99% rename from src/device/bus/pci/space/mod.rs rename to driver/bus/pci/src/space/mod.rs index 51773391..182f911f 100644 --- a/src/device/bus/pci/space/mod.rs +++ b/driver/bus/pci/src/space/mod.rs @@ -1,6 +1,5 @@ -use crate::device::bus::pci::PciStatusRegister; - use super::{PciAddress, PciBaseAddress, PciCapability, PciCapabilityId, PciEcam}; +use crate::PciStatusRegister; pub(super) mod ecam; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 6c2d474f..5834257c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -21,6 +21,7 @@ use kernel_util::{ util::OneTimeInit, }; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; +use ygg_driver_pci::PciBusManager; mod acpi; mod apic; @@ -49,8 +50,8 @@ use crate::{ debug::{self, LogLevel}, device::{ self, - bus::pci::PciBusManager, display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, + nvme, tty::CombinedTerminal, }, fs::{ @@ -384,6 +385,16 @@ impl X86_64 { Cpu::init_local(LocalApic::new(), cpu_id as _); if cpu_id == 0 { + // Register the PCI drivers + // TODO make this implicit init + ygg_driver_pci::register_class_driver( + "NVMe Host Controller", + 0x01, + Some(0x08), + Some(0x02), + nvme::probe, + ); + match self.boot_data.get() { &BootData::YBoot(data) => { let start = PhysicalAddress::from_raw(data.initrd_address); diff --git a/src/device/bus/mod.rs b/src/device/bus/mod.rs index e6810506..42007f9c 100644 --- a/src/device/bus/mod.rs +++ b/src/device/bus/mod.rs @@ -2,5 +2,3 @@ #[cfg(feature = "device-tree")] pub mod simple_bus; - -pub mod pci; diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index fa705501..0e0ac415 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -2,7 +2,7 @@ use core::{mem::size_of, time::Duration}; use abi::error::Error; -use alloc::{collections::BTreeMap, vec::Vec}; +use alloc::{boxed::Box, collections::BTreeMap, vec::Vec}; use device_api::{interrupt::MsiHandler, Device}; use kernel_util::{ mem::{ @@ -17,20 +17,19 @@ use tock_registers::{ register_bitfields, register_structs, registers::{ReadOnly, ReadWrite, WriteOnly}, }; +use ygg_driver_pci::{ + capability::{MsiXCapability, MsiXEntry}, + PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, +}; use crate::{ arch::{Architecture, ARCHITECTURE}, - device::{ - bus::pci::{ - capability::MsiXCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, - }, - nvme::{ - command::{ - IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest, IoRead, IoWrite, - }, - drive::NvmeDrive, - queue::{CompletionQueueEntry, SubmissionQueueEntry}, + device::nvme::{ + command::{ + IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest, IoRead, IoWrite, }, + drive::NvmeDrive, + queue::{CompletionQueueEntry, SubmissionQueueEntry}, }, task::runtime, }; @@ -41,8 +40,6 @@ use self::{ queue::QueuePair, }; -use super::bus::pci::{capability::MsiXEntry, FromPciBus, PciDeviceInfo}; - mod command; mod drive; mod error; @@ -358,48 +355,51 @@ impl Device for NvmeController { } } -impl FromPciBus for NvmeController { - fn from_pci_bus(info: &PciDeviceInfo) -> Result { - let PciBaseAddress::Memory(bar0) = info.config_space.bar(0).unwrap() else { - panic!(); - }; - - // TODO also support MSI - let mut msix = info.config_space.capability::().unwrap(); - let mut vt = msix.vector_table()?; - - for vector in vt.iter_mut() { - vector.set_masked(true); - } - msix.set_enabled(true); - - let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); - cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); - cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; - info.config_space.set_command(cmd.bits()); - - let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar0)) }?; - - // Disable the controller - regs.CC.modify(CC::ENABLE::CLEAR); - - let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1; - - Ok(Self { - regs: IrqSafeSpinlock::new(regs), - admin_q: OneTimeInit::new(), - ioqs: OneTimeInit::new(), - vector_table: IrqSafeSpinlock::new(vt), - drive_table: IrqSafeSpinlock::new(BTreeMap::new()), - controller_id: OneTimeInit::new(), - doorbell_shift, - }) - } -} +// impl FromPciBus for NvmeController { +// fn from_pci_bus(info: &PciDeviceInfo) -> Result { +// } +// } static NVME_CONTROLLERS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); +pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { + let PciBaseAddress::Memory(bar0) = info.config_space.bar(0).unwrap() else { + panic!(); + }; + + // TODO also support MSI + let mut msix = info.config_space.capability::().unwrap(); + let mut vt = msix.vector_table()?; + + for vector in vt.iter_mut() { + vector.set_masked(true); + } + msix.set_enabled(true); + + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); + cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); + cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; + info.config_space.set_command(cmd.bits()); + + let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar0)) }?; + + // Disable the controller + regs.CC.modify(CC::ENABLE::CLEAR); + + let doorbell_shift = regs.CAP.read(CAP::DSTRD) as usize + 1; + + Ok(Box::leak(Box::new(NvmeController { + regs: IrqSafeSpinlock::new(regs), + admin_q: OneTimeInit::new(), + ioqs: OneTimeInit::new(), + vector_table: IrqSafeSpinlock::new(vt), + drive_table: IrqSafeSpinlock::new(BTreeMap::new()), + controller_id: OneTimeInit::new(), + doorbell_shift, + }))) +} + pub fn register_nvme_controller(ctrl: &'static NvmeController) { let mut list = NVME_CONTROLLERS.lock(); let id = list.len(); From e78784d96d80a36d1f5a994d345fdb1530b40c38 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 10 Dec 2023 23:01:39 +0200 Subject: [PATCH 114/211] refactor: move async runtime into kernel-util --- lib/kernel-util/Cargo.toml | 3 + lib/kernel-util/src/api.rs | 24 ++++- lib/kernel-util/src/lib.rs | 6 +- lib/kernel-util/src/runtime/executor.rs | 65 ++++++++++++ lib/kernel-util/src/runtime/macros.rs | 8 ++ lib/kernel-util/src/runtime/mod.rs | 54 ++++++++++ lib/kernel-util/src/runtime/task.rs | 46 ++++++++ lib/kernel-util/src/runtime/task_queue.rs | 70 +++++++++++++ lib/kernel-util/src/runtime/waker.rs | 72 +++++++++++++ lib/kernel-util/src/thread.rs | 102 ++++++++++++++++++ src/arch/mod.rs | 7 ++ src/arch/x86_64/boot/mod.rs | 2 +- src/arch/x86_64/peripherals/i8253.rs | 15 ++- src/device/display/console.rs | 4 +- src/device/nvme/mod.rs | 2 +- src/device/nvme/queue.rs | 3 +- src/device/timer.rs | 9 -- src/device/tty.rs | 3 +- src/init.rs | 3 +- src/syscall/mod.rs | 14 +-- src/task/context.rs | 35 +------ src/task/mod.rs | 5 +- src/task/process.rs | 3 +- src/task/runtime/mod.rs | 122 +++++++++------------- src/task/sync.rs | 3 +- src/task/thread.rs | 45 +++++++- src/util/queue.rs | 3 +- src/util/ring.rs | 4 +- 28 files changed, 571 insertions(+), 161 deletions(-) create mode 100644 lib/kernel-util/src/runtime/executor.rs create mode 100644 lib/kernel-util/src/runtime/macros.rs create mode 100644 lib/kernel-util/src/runtime/mod.rs create mode 100644 lib/kernel-util/src/runtime/task.rs create mode 100644 lib/kernel-util/src/runtime/task_queue.rs create mode 100644 lib/kernel-util/src/runtime/waker.rs create mode 100644 lib/kernel-util/src/thread.rs diff --git a/lib/kernel-util/Cargo.toml b/lib/kernel-util/Cargo.toml index 81ac1c42..781b9149 100644 --- a/lib/kernel-util/Cargo.toml +++ b/lib/kernel-util/Cargo.toml @@ -7,4 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } + log = "0.4.20" +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } +crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index aa43441f..b391d89c 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -1,6 +1,12 @@ -use yggdrasil_abi::error::Error; +use core::time::Duration; -use crate::mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}; +use alloc::{string::String, sync::Arc}; +use yggdrasil_abi::{error::Error, process::ExitCode}; + +use crate::{ + mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}, + thread::{CurrentThread, Thread}, +}; extern "Rust" { pub fn __acquire_irq_guard() -> bool; @@ -19,4 +25,18 @@ extern "Rust" { count: usize, ) -> Result; pub fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping); + + // SAFETY: Both the kernel-side and api-side Threads are sized and Arc<___> has the same value + // for them + pub fn __create_kthread( + name: String, + func: extern "C" fn(usize) -> !, + arg: usize, + ) -> Result, Error>; + pub fn __enqueue(t: &Arc); + pub fn __current_thread() -> CurrentThread; + pub fn __suspend_current(t: &CurrentThread) -> Result<(), Error>; + pub fn __exit_current(t: &CurrentThread, code: ExitCode) -> !; + + pub fn __monotonic_timestamp() -> Result; } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 7f83491a..573a8066 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -5,7 +5,9 @@ const_trait_impl, effects, slice_ptr_get, - strict_provenance + strict_provenance, + never_type, + let_chains )] extern crate alloc; @@ -13,7 +15,9 @@ extern crate alloc; pub(crate) mod api; pub mod mem; +pub mod runtime; pub mod sync; +pub mod thread; pub mod util; #[repr(C)] diff --git a/lib/kernel-util/src/runtime/executor.rs b/lib/kernel-util/src/runtime/executor.rs new file mode 100644 index 00000000..cd3295d1 --- /dev/null +++ b/lib/kernel-util/src/runtime/executor.rs @@ -0,0 +1,65 @@ +use core::task::{Context, Poll}; + +use alloc::{boxed::Box, format, sync::Arc}; +use futures_util::{task::waker_ref, Future}; +use yggdrasil_abi::error::Error; + +use crate::thread::Thread; + +use super::{ + task::{Task, Termination}, + task_queue, +}; + +/// Pushes a task into the executor's queue +pub fn enqueue(task: Arc) -> Result<(), Error> { + task_queue::push_task(task) +} + +/// Spawns a background worker to execute the tasks from the global queue +pub fn spawn_async_worker(index: usize) -> Result<(), Error> { + let name = format!("[async-worker-{}]", index); + + Thread::spawn(name, move || { + loop { + let task = task_queue::pop_task().unwrap(); // queue.block_pop().unwrap(); + let mut future_slot = task.future.lock(); + + if let Some(mut future) = future_slot.take() { + let waker = waker_ref(&task); + let context = &mut Context::from_waker(&waker); + + if future.as_mut().poll(context).is_pending() { + *future_slot = Some(future); + } + } + } + }) +} + +/// Creates a new task for the [Future] and queues it for execution in background +pub fn spawn + Send + 'static>( + future: F, +) -> Result<(), Error> { + enqueue(Task::new(future)) +} + +/// Runs a [Future] to its completion on the current thread +pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { + let thread = Thread::current(); + let mut future = Box::pin(future); + + loop { + let waker = waker_ref(&thread); + let context = &mut Context::from_waker(&waker); + + match future.as_mut().poll(context) { + Poll::Ready(value) => break Ok(value), + Poll::Pending => { + if let Err(error) = thread.suspend() { + break Err(error); + } + } + } + } +} diff --git a/lib/kernel-util/src/runtime/macros.rs b/lib/kernel-util/src/runtime/macros.rs new file mode 100644 index 00000000..00c7da9b --- /dev/null +++ b/lib/kernel-util/src/runtime/macros.rs @@ -0,0 +1,8 @@ +#[macro_export] +macro_rules! block { + ($($stmt:tt)*) => { + $crate::runtime::run_to_completion(alloc::boxed::Box::pin(async move { + $($stmt)* + })) + }; +} diff --git a/lib/kernel-util/src/runtime/mod.rs b/lib/kernel-util/src/runtime/mod.rs new file mode 100644 index 00000000..d9e93dd0 --- /dev/null +++ b/lib/kernel-util/src/runtime/mod.rs @@ -0,0 +1,54 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +#[macro_use] +mod macros; + +mod executor; +mod task; +mod task_queue; +mod waker; + +pub use executor::{run_to_completion, spawn, spawn_async_worker}; +use futures_util::Future; +pub use task_queue::init_task_queue; +pub use waker::QueueWaker; + +use crate::api; + +static SLEEP_WAKER: QueueWaker = QueueWaker::new(); + +/// Suspends the task until given duration passes +pub fn sleep(duration: Duration) -> impl Future { + struct SleepFuture { + deadline: Duration, + } + + impl Future for SleepFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + SLEEP_WAKER.register(cx.waker()); + let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + if now >= self.deadline { + SLEEP_WAKER.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + } + + let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + let deadline = now + duration; + + SleepFuture { deadline } +} + +/// Updates the runtime's time +pub fn tick(_now: Duration) { + SLEEP_WAKER.wake_all(); +} diff --git a/lib/kernel-util/src/runtime/task.rs b/lib/kernel-util/src/runtime/task.rs new file mode 100644 index 00000000..1d5602a1 --- /dev/null +++ b/lib/kernel-util/src/runtime/task.rs @@ -0,0 +1,46 @@ +use core::fmt; + +use alloc::sync::Arc; +use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt}; + +use crate::sync::IrqSafeSpinlock; + +use super::executor; + +pub trait Termination { + fn print(&self); +} + +pub struct Task { + pub(super) future: IrqSafeSpinlock>>, +} + +impl ArcWake for Task { + fn wake_by_ref(arc_self: &Arc) { + executor::enqueue(arc_self.clone()).unwrap(); + } +} + +impl Task { + pub fn new + Send + 'static>(future: F) -> Arc { + let future = IrqSafeSpinlock::new(Some( + async move { + future.await.print(); + } + .boxed(), + )); + Arc::new(Self { future }) + } +} + +impl Termination for () { + fn print(&self) {} +} + +impl Termination for Result { + fn print(&self) { + if let Err(error) = self { + log::error!("A task finished with an error: {:?}", error); + } + } +} diff --git a/lib/kernel-util/src/runtime/task_queue.rs b/lib/kernel-util/src/runtime/task_queue.rs new file mode 100644 index 00000000..18e036f2 --- /dev/null +++ b/lib/kernel-util/src/runtime/task_queue.rs @@ -0,0 +1,70 @@ +use alloc::sync::Arc; +use crossbeam_queue::ArrayQueue; +use yggdrasil_abi::error::Error; + +use crate::{sync::IrqGuard, thread::Thread, util::OneTimeInit}; + +use super::task::Task; + +pub(super) static TASK_QUEUE: OneTimeInit = OneTimeInit::new(); + +pub(super) struct TaskQueue { + // Queue of workers waiting for an item + pending_workers: ArrayQueue>, + task_queue: ArrayQueue>, +} + +impl TaskQueue { + pub fn new(task_capacity: usize) -> Self { + assert!(task_capacity > 0); + Self { + pending_workers: ArrayQueue::new(16), + task_queue: ArrayQueue::new(task_capacity), + } + } + + fn wakeup_one(&self) { + if let Some(worker) = self.pending_workers.pop() { + worker.enqueue(); + } + } + + pub fn enqueue(&self, task: Arc) -> Result<(), Error> { + let _irq = IrqGuard::acquire(); + if self.task_queue.push(task).is_err() { + todo!(); + } + self.wakeup_one(); + Ok(()) + } + + pub fn dequeue(&self) -> Result, Error> { + let thread = Thread::current(); + // assert!(ArchitectureImpl::interrupt_mask()); + loop { + if let Some(task) = self.task_queue.pop() { + return Ok(task); + } + + if self.pending_workers.push(thread.clone()).is_err() { + panic!("Pending worker queue overflow"); + } + + // This must not fail. Signals must not be raised. + thread.suspend().unwrap(); + } + } +} + +/// Initializes the global async/await task queue +pub fn init_task_queue() { + TASK_QUEUE.init(TaskQueue::new(128)); +} + +pub(super) fn push_task(task: Arc) -> Result<(), Error> { + TASK_QUEUE.get().enqueue(task) +} + +pub(super) fn pop_task() -> Result, Error> { + TASK_QUEUE.get().dequeue() +} diff --git a/lib/kernel-util/src/runtime/waker.rs b/lib/kernel-util/src/runtime/waker.rs new file mode 100644 index 00000000..24b3df55 --- /dev/null +++ b/lib/kernel-util/src/runtime/waker.rs @@ -0,0 +1,72 @@ +use core::task::Waker; + +use alloc::collections::VecDeque; + +use crate::sync::IrqSafeSpinlock; + +/// Async/await primitive to suspend and wake up tasks waiting on some shared resource +pub struct QueueWaker { + queue: IrqSafeSpinlock>, +} + +impl QueueWaker { + /// Constructs an empty [QueueWaker] + pub const fn new() -> Self { + Self { + queue: IrqSafeSpinlock::new(VecDeque::new()), + } + } + + /// Registers a [Waker] reference to be waken up by this [QueueWaker] + pub fn register(&self, waker: &Waker) { + let mut queue = self.queue.lock(); + + if queue.iter().any(|other| other.will_wake(waker)) { + return; + } + + queue.push_back(waker.clone()); + } + + /// Removes a [Waker] reference from this [QueueWaker] + pub fn remove(&self, waker: &Waker) -> bool { + let mut queue = self.queue.lock(); + let mut index = 0; + let mut removed = false; + + while index < queue.len() { + if queue[index].will_wake(waker) { + removed = true; + queue.remove(index); + } + index += 1; + } + + removed + } + + /// Wakes up up to `limit` tasks waiting on this queue + pub fn wake_some(&self, limit: usize) -> usize { + let mut queue = self.queue.lock(); + let mut count = 0; + + while count < limit + && let Some(item) = queue.pop_front() + { + item.wake(); + count += 1; + } + + count + } + + /// Wakes up a single task waiting on this queue + pub fn wake_one(&self) -> bool { + self.wake_some(1) != 0 + } + + /// Wakes up all tasks waiting on this queue + pub fn wake_all(&self) -> usize { + self.wake_some(usize::MAX) + } +} diff --git a/lib/kernel-util/src/thread.rs b/lib/kernel-util/src/thread.rs new file mode 100644 index 00000000..384fb36d --- /dev/null +++ b/lib/kernel-util/src/thread.rs @@ -0,0 +1,102 @@ +use core::{fmt, ops::Deref}; + +use alloc::{boxed::Box, string::String, sync::Arc}; +use futures_util::task::ArcWake; +use yggdrasil_abi::{error::Error, process::ExitCode}; + +use crate::{api, sync::IrqGuard}; + +#[repr(C)] +pub(crate) struct Thread(!); +#[repr(C)] +pub(crate) struct CurrentThread(Arc, IrqGuard); + +/// Conversion trait to allow multiple kernel closure return types +pub trait Termination { + /// Converts the closure return type into [ExitCode] + fn into_exit_code(self) -> ExitCode; +} + +impl Termination for Result { + fn into_exit_code(self) -> ExitCode { + match self { + Ok(_) => ExitCode::SUCCESS, + Err(err) => { + log::warn!("Kernel thread failed: {:?}", err); + ExitCode::Exited(1) + } + } + } +} + +impl Termination for ExitCode { + fn into_exit_code(self) -> ExitCode { + self + } +} + +impl Termination for () { + fn into_exit_code(self) -> ExitCode { + ExitCode::SUCCESS + } +} + +impl Thread { + pub fn spawn, F: FnOnce() + Send + 'static>( + name: S, + f: F, + ) -> Result<(), Error> { + extern "C" fn closure_wrapper(closure_addr: usize) -> ! { + let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; + let result = closure(); + Thread::current().exit(result.into_exit_code()); + unreachable!(); + } + + let closure = Box::new(f); + log::debug!("closure: {:p}", closure); + let thread = unsafe { + api::__create_kthread( + name.into(), + closure_wrapper::, + Box::into_raw(closure) as usize, + ) + }?; + + thread.enqueue(); + + Ok(()) + } + + pub fn current() -> CurrentThread { + unsafe { api::__current_thread() } + } + + pub fn enqueue(self: &Arc) { + unsafe { api::__enqueue(self) } + } +} + +impl CurrentThread { + pub fn suspend(&self) -> Result<(), Error> { + unsafe { api::__suspend_current(self) } + } + + pub fn exit(&self, code: ExitCode) { + unsafe { api::__exit_current(self, code) } + } +} + +impl ArcWake for Thread { + fn wake_by_ref(arc_self: &Arc) { + arc_self.clone().enqueue() + } +} + +impl Deref for CurrentThread { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index d599433d..f16783a4 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,5 +1,7 @@ //! Provides architecture/platform-specific implementation details +use core::time::Duration; + use abi::error::Error; /// Returns an absolute address to the given symbol @@ -264,3 +266,8 @@ fn __map_device_pages( fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping) { unsafe { ARCHITECTURE.unmap_device_memory(mapping) } } + +#[no_mangle] +fn __monotonic_timestamp() -> Result { + ARCHITECTURE.monotonic_timer().monotonic_timestamp() +} diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index c15f29eb..c41b561b 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,6 +1,7 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; +use kernel_util::runtime; use tock_registers::interfaces::Writeable; use yboot_proto::{ v1::{FramebufferOption, MemoryMap}, @@ -12,7 +13,6 @@ use crate::{ fs::devfs, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, - task::runtime, }; use super::{cpuid::init_cpuid, exception, ARCHITECTURE}; diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index 3eab2a46..3a94e1cc 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -2,17 +2,14 @@ use core::time::Duration; use abi::error::Error; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; -use kernel_util::sync::IrqSafeSpinlock; +use kernel_util::{runtime, sync::IrqSafeSpinlock}; -use crate::{ - arch::{ - x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - IrqNumber, - }, - Architecture, ARCHITECTURE, +use crate::arch::{ + x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + IrqNumber, }, - task::runtime, + Architecture, ARCHITECTURE, }; const FREQUENCY: u32 = 1193180; diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 68548e22..0823aadb 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -5,9 +5,9 @@ use core::time::Duration; use abi::{error::Error, primitive_enum}; use alloc::{vec, vec::Vec}; use bitflags::bitflags; -use kernel_util::{sync::IrqSafeSpinlock, util::StaticVector}; +use kernel_util::{runtime, sync::IrqSafeSpinlock, util::StaticVector}; -use crate::{debug::DebugSink, task::runtime}; +use crate::debug::DebugSink; const CONSOLE_ROW_LEN: usize = 80; const MAX_CSI_ARGS: usize = 8; diff --git a/src/device/nvme/mod.rs b/src/device/nvme/mod.rs index 0e0ac415..f54a3288 100644 --- a/src/device/nvme/mod.rs +++ b/src/device/nvme/mod.rs @@ -9,6 +9,7 @@ use kernel_util::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, DeviceMemoryIoMut}, }, + runtime, sync::IrqSafeSpinlock, util::OneTimeInit, }; @@ -31,7 +32,6 @@ use crate::{ drive::NvmeDrive, queue::{CompletionQueueEntry, SubmissionQueueEntry}, }, - task::runtime, }; use self::{ diff --git a/src/device/nvme/queue.rs b/src/device/nvme/queue.rs index 3d027954..1381c481 100644 --- a/src/device/nvme/queue.rs +++ b/src/device/nvme/queue.rs @@ -17,12 +17,11 @@ use kernel_util::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, PageBox, }, + runtime::QueueWaker, sync::IrqSafeSpinlock, }; use static_assertions::const_assert; -use crate::task::runtime::QueueWaker; - use super::{ command::{Command, Request}, error::NvmeError, diff --git a/src/device/timer.rs b/src/device/timer.rs index a24002bc..17f1690a 100644 --- a/src/device/timer.rs +++ b/src/device/timer.rs @@ -1,10 +1 @@ //! Timer device utilities - -use core::time::Duration; - -use crate::task::runtime; - -/// Global system timer tick handler -pub fn tick(now: Duration) { - runtime::tick(now); -} diff --git a/src/device/tty.rs b/src/device/tty.rs index 826d942f..dc2b5da9 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -15,12 +15,13 @@ use crate::{ #[cfg(feature = "fb_console")] pub mod combined { //! Combined console + keyboard terminal device - use crate::{block, task::process::ProcessId}; + use crate::task::process::ProcessId; use abi::{ error::Error, io::{DeviceRequest, TerminalSize}, }; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; + use kernel_util::block; use vfs::CharDevice; // use vfs::CharDevice; diff --git a/src/init.rs b/src/init.rs index 9579d0ed..619de99a 100644 --- a/src/init.rs +++ b/src/init.rs @@ -28,7 +28,8 @@ pub fn kinit() -> Result<(), Error> { #[cfg(feature = "fb_console")] { - use crate::{device::display::console::update_consoles_task, task::runtime}; + use crate::device::display::console::update_consoles_task; + use kernel_util::runtime; runtime::spawn(async move { update_consoles_task().await; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 7d8b459d..a881a1b5 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -8,19 +8,17 @@ use abi::{ syscall::SyscallFunction, }; use alloc::sync::Arc; -use kernel_util::sync::IrqSafeSpinlockGuard; +use kernel_util::{block, runtime, sync::IrqSafeSpinlockGuard}; use vfs::{IoContext, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ - block, debug::LogLevel, fs, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, - runtime, thread::Thread, }, }; @@ -134,10 +132,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let file = io.ioctx().open(at, path, opts, mode)?; // TODO NO_CTTY? - if process.session_terminal().is_none() && - let Some(node) = file.node() && node.is_terminal() { - debugln!("Session terminal set for #{}: {}", process.id(), path); - process.set_session_terminal(node.clone()); + if process.session_terminal().is_none() + && let Some(node) = file.node() + && node.is_terminal() + { + debugln!("Session terminal set for #{}: {}", process.id(), path); + process.set_session_terminal(node.clone()); } let fd = io.place_file(file)?; diff --git a/src/task/context.rs b/src/task/context.rs index ecac1854..e3a7e23f 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -1,10 +1,9 @@ //! Platform-specific task context manipulation interfaces -use core::fmt; - -use abi::{arch::SavedFrame, error::Error, process::ExitCode}; +use abi::{arch::SavedFrame, error::Error}; use alloc::boxed::Box; use cfg_if::cfg_if; +use kernel_util::thread::Termination; use crate::task::thread::Thread; @@ -18,36 +17,6 @@ cfg_if! { } } -/// Conversion trait to allow multiple kernel closure return types -pub trait Termination { - /// Converts the closure return type into [ExitCode] - fn into_exit_code(self) -> ExitCode; -} - -impl Termination for Result { - fn into_exit_code(self) -> ExitCode { - match self { - Ok(_) => ExitCode::SUCCESS, - Err(err) => { - warnln!("Kernel thread failed: {:?}", err); - ExitCode::Exited(1) - } - } - } -} - -impl Termination for ExitCode { - fn into_exit_code(self) -> ExitCode { - self - } -} - -impl Termination for () { - fn into_exit_code(self) -> ExitCode { - ExitCode::SUCCESS - } -} - /// Interface for task state save/restore mechanisms pub trait TaskFrame { /// Creates a "snapshot" of a exception/syscall frame diff --git a/src/task/mod.rs b/src/task/mod.rs index 09a24826..6e21f0b4 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -4,18 +4,17 @@ use abi::error::Error; use alloc::{string::String, vec::Vec}; -use kernel_util::sync::SpinFence; +use kernel_util::{runtime, sync::SpinFence, thread::Termination}; use crate::{ arch::{Architecture, ArchitectureImpl}, task::{sched::CpuQueue, thread::Thread}, }; -use self::context::{TaskContextImpl, Termination}; +use self::context::TaskContextImpl; pub mod context; pub mod process; -pub mod runtime; pub mod sched; pub mod sync; pub mod thread; diff --git a/src/task/process.rs b/src/task/process.rs index 55897ace..261e02e8 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -14,7 +14,7 @@ use abi::{ }; use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; use futures_util::Future; -use kernel_util::sync::IrqSafeSpinlock; +use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; use vfs::NodeRef; use crate::{ @@ -24,7 +24,6 @@ use crate::{ }; use super::{ - runtime::QueueWaker, sync::UserspaceMutex, thread::{Thread, ThreadId}, TaskContext, diff --git a/src/task/runtime/mod.rs b/src/task/runtime/mod.rs index b20bfa02..e40af2a1 100644 --- a/src/task/runtime/mod.rs +++ b/src/task/runtime/mod.rs @@ -1,5 +1,51 @@ //! Async/await runtime implementation +// use core::{ +// pin::Pin, +// task::{Context, Poll}, +// time::Duration, +// }; +// +// use futures_util::Future; +// +// use crate::arch::{Architecture, ARCHITECTURE}; +// +// mod executor; +// mod macros; +// mod task; +// mod task_queue; +// mod waker; +// +// pub use executor::{run_to_completion, spawn, spawn_async_worker}; +// pub use task_queue::init_task_queue; +// pub use waker::QueueWaker; +// +// /// [Future] implementation that wraps a poll-function +// pub struct PollFn { +// f: F, +// } +// +// impl Future for PollFn +// where +// F: FnMut(&mut Context) -> Poll, +// { +// type Output = T; +// +// fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { +// #[allow(clippy::needless_borrow)] +// (unsafe { &mut self.get_unchecked_mut().f })(cx) +// } +// } +// +// /// Constructs a [PollFn] from given poll-function +// pub fn poll_fn(f: F) -> PollFn +// where +// F: FnMut(&mut Context) -> Poll, +// { +// PollFn { f } +// } +// + use core::{ pin::Pin, task::{Context, Poll}, @@ -7,80 +53,6 @@ use core::{ }; use futures_util::Future; +use kernel_util::runtime::QueueWaker; use crate::arch::{Architecture, ARCHITECTURE}; - -mod executor; -mod macros; -mod task; -mod task_queue; -mod waker; - -pub use executor::{run_to_completion, spawn, spawn_async_worker}; -pub use task_queue::init_task_queue; -pub use waker::QueueWaker; - -/// [Future] implementation that wraps a poll-function -pub struct PollFn { - f: F, -} - -impl Future for PollFn -where - F: FnMut(&mut Context) -> Poll, -{ - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - #[allow(clippy::needless_borrow)] - (unsafe { &mut self.get_unchecked_mut().f })(cx) - } -} - -/// Constructs a [PollFn] from given poll-function -pub fn poll_fn(f: F) -> PollFn -where - F: FnMut(&mut Context) -> Poll, -{ - PollFn { f } -} - -static SLEEP_WAKER: QueueWaker = QueueWaker::new(); - -/// Suspends the task until given duration passes -pub fn sleep(duration: Duration) -> impl Future { - struct SleepFuture { - deadline: Duration, - } - - impl Future for SleepFuture { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - SLEEP_WAKER.register(cx.waker()); - let now = ARCHITECTURE - .monotonic_timer() - .monotonic_timestamp() - .unwrap(); - if now >= self.deadline { - SLEEP_WAKER.remove(cx.waker()); - Poll::Ready(()) - } else { - Poll::Pending - } - } - } - - let now = ARCHITECTURE - .monotonic_timer() - .monotonic_timestamp() - .unwrap(); - let deadline = now + duration; - - SleepFuture { deadline } -} - -/// Updates the runtime's time -pub fn tick(_now: Duration) { - SLEEP_WAKER.wake_all(); -} diff --git a/src/task/sync.rs b/src/task/sync.rs index fcc0fb5a..3776b154 100644 --- a/src/task/sync.rs +++ b/src/task/sync.rs @@ -8,8 +8,7 @@ use core::{ use alloc::sync::Arc; use futures_util::Future; - -use super::runtime::QueueWaker; +use kernel_util::runtime::QueueWaker; /// User-space mutex (like BSD/Linux's futex) data structure pub struct UserspaceMutex { diff --git a/src/task/thread.rs b/src/task/thread.rs index 9aa9d51b..cfc9b0d0 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -20,17 +20,18 @@ use alloc::{ }; use atomic_enum::atomic_enum; use futures_util::{task::ArcWake, Future}; -use kernel_util::sync::{IrqGuard, IrqSafeSpinlock}; +use kernel_util::{ + block, + runtime::QueueWaker, + sync::{IrqGuard, IrqSafeSpinlock}, +}; use crate::{ - block, mem::{process::ProcessAddressSpace, ForeignPointer}, task::{context::TaskContextImpl, Cpu}, }; -use super::{ - context::TaskFrame, process::Process, runtime::QueueWaker, sched::CpuQueue, TaskContext, -}; +use super::{context::TaskFrame, process::Process, sched::CpuQueue, TaskContext}; /// Represents the states a thread can be at some point in time #[atomic_enum] @@ -56,6 +57,7 @@ pub enum ThreadId { } /// Wrapper which guarantees the thread referred to is the current one on the current CPU +#[repr(C)] pub struct CurrentThread(Arc, IrqGuard); struct SignalEntry { @@ -545,3 +547,36 @@ impl fmt::Display for ThreadId { } } } + +// External API + +#[no_mangle] +fn __create_kthread( + name: String, + func: extern "C" fn(usize) -> !, + arg: usize, +) -> Result, Error> { + let context = TaskContext::kernel(func, arg)?; + Ok(Thread::new_kthread(name, context)) +} + +#[no_mangle] +fn __enqueue(t: &Arc) { + t.enqueue_somewhere(); +} + +#[no_mangle] +fn __current_thread() -> CurrentThread { + Thread::current() +} + +#[no_mangle] +fn __suspend_current(t: &CurrentThread) -> Result<(), Error> { + t.suspend() +} + +#[no_mangle] +fn __exit_current(t: &CurrentThread, code: ExitCode) -> ! { + t.exit(code); + unreachable!(); +} diff --git a/src/util/queue.rs b/src/util/queue.rs index fe888221..67969503 100644 --- a/src/util/queue.rs +++ b/src/util/queue.rs @@ -8,8 +8,7 @@ use core::{ use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; use futures_util::Future; - -use crate::task::runtime::QueueWaker; +use kernel_util::runtime::QueueWaker; /// Asynchronous queue pub struct AsyncQueue { diff --git a/src/util/ring.rs b/src/util/ring.rs index 15ad752a..c9605d44 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -8,9 +8,7 @@ use core::{ use abi::error::Error; use alloc::sync::Arc; use futures_util::Future; -use kernel_util::sync::IrqSafeSpinlock; - -use crate::task::runtime::QueueWaker; +use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; /// Ring buffer base pub struct RingBuffer { From 8b7a7b9295f38c61673b24fca01266ea7c5a4673 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 10 Dec 2023 23:22:21 +0200 Subject: [PATCH 115/211] block/nvme: move nvme driver to its own crate --- Cargo.toml | 3 ++ driver/block/nvme/Cargo.toml | 21 ++++++++++++ .../nvme => driver/block/nvme/src}/command.rs | 2 +- .../nvme => driver/block/nvme/src}/drive.rs | 8 ++--- .../nvme => driver/block/nvme/src}/error.rs | 2 +- .../mod.rs => driver/block/nvme/src/lib.rs | 33 ++++++++++--------- .../nvme => driver/block/nvme/src}/queue.rs | 4 +-- driver/fs/kernel-fs/Cargo.toml | 13 ++++++++ {src/fs => driver/fs/kernel-fs/src}/devfs.rs | 26 +++------------ driver/fs/kernel-fs/src/lib.rs | 5 +++ lib/kernel-util/Cargo.toml | 1 + lib/kernel-util/src/api.rs | 3 ++ lib/kernel-util/src/lib.rs | 7 ++++ src/arch/mod.rs | 5 +++ src/arch/x86_64/boot/mod.rs | 2 +- src/arch/x86_64/mod.rs | 9 ++--- src/device/mod.rs | 2 -- src/fs/mod.rs | 20 +++++++++-- src/init.rs | 3 +- src/main.rs | 4 +-- 20 files changed, 112 insertions(+), 61 deletions(-) create mode 100644 driver/block/nvme/Cargo.toml rename {src/device/nvme => driver/block/nvme/src}/command.rs (99%) rename {src/device/nvme => driver/block/nvme/src}/drive.rs (90%) rename {src/device/nvme => driver/block/nvme/src}/error.rs (88%) rename src/device/nvme/mod.rs => driver/block/nvme/src/lib.rs (93%) rename {src/device/nvme => driver/block/nvme/src}/queue.rs (98%) create mode 100644 driver/fs/kernel-fs/Cargo.toml rename {src/fs => driver/fs/kernel-fs/src}/devfs.rs (76%) create mode 100644 driver/fs/kernel-fs/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index cfb3d84b..40c95020 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ opt-level = 3 [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } +# TODO move to drivers memfs = { path = "lib/memfs" } device-api = { path = "lib/device-api", features = ["derive"] } kernel-util = { path = "lib/kernel-util" } @@ -20,6 +21,8 @@ device-api-macros = { path = "lib/device-api/macros" } # Drivers ygg_driver_pci = { path = "driver/bus/pci" } +ygg_driver_nvme = { path = "driver/block/nvme" } +kernel-fs = { path = "driver/fs/kernel-fs" } atomic_enum = "0.2.0" bitflags = "2.3.3" diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml new file mode 100644 index 00000000..ca5721ce --- /dev/null +++ b/driver/block/nvme/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "ygg_driver_nvme" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } +vfs = { path = "../../../lib/vfs" } + +ygg_driver_pci = { path = "../../bus/pci" } +kernel-fs = { path = "../../fs/kernel-fs" } + +log = "0.4.20" +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } +static_assertions = "1.1.0" +tock-registers = "0.8.1" +bytemuck = { version = "1.14.0", features = ["derive"] } diff --git a/src/device/nvme/command.rs b/driver/block/nvme/src/command.rs similarity index 99% rename from src/device/nvme/command.rs rename to driver/block/nvme/src/command.rs index 0759de59..d3eb069e 100644 --- a/src/device/nvme/command.rs +++ b/driver/block/nvme/src/command.rs @@ -5,7 +5,7 @@ use core::fmt::{self, Write}; use kernel_util::mem::address::PhysicalAddress; use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike}; -use crate::device::nvme::queue::PhysicalRegionPage; +use crate::queue::PhysicalRegionPage; use super::queue::SubmissionQueueEntry; diff --git a/src/device/nvme/drive.rs b/driver/block/nvme/src/drive.rs similarity index 90% rename from src/device/nvme/drive.rs rename to driver/block/nvme/src/drive.rs index be2daf60..559d61c0 100644 --- a/src/device/nvme/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -1,8 +1,9 @@ -use abi::{error::Error, io::DeviceRequest}; use alloc::{boxed::Box, format}; +use kernel_fs::devfs; use vfs::BlockDevice; +use yggdrasil_abi::{error::Error, io::DeviceRequest}; -use crate::{device::nvme::command::IdentifyNamespaceRequest, fs::devfs}; +use crate::command::IdentifyNamespaceRequest; use super::{error::NvmeError, NvmeController}; @@ -27,7 +28,7 @@ impl NvmeDrive { let lba_size = current_lba_format.lba_data_size().unwrap(); let total_lba_count = identify.total_lba_count(); - debugln!( + log::debug!( "ns = {}, lba = {}B, size = {}M", nsid, lba_size, @@ -41,7 +42,6 @@ impl NvmeDrive { lba_size, })); - // TODO add the drive as a block device let node_name = format!("nvme{}n{}", controller.controller_id.get(), nsid); devfs::add_named_block_device(dev, node_name).ok(); diff --git a/src/device/nvme/error.rs b/driver/block/nvme/src/error.rs similarity index 88% rename from src/device/nvme/error.rs rename to driver/block/nvme/src/error.rs index 86553300..e933f039 100644 --- a/src/device/nvme/error.rs +++ b/driver/block/nvme/src/error.rs @@ -1,4 +1,4 @@ -use abi::error::Error; +use yggdrasil_abi::error::Error; use super::queue::CommandError; diff --git a/src/device/nvme/mod.rs b/driver/block/nvme/src/lib.rs similarity index 93% rename from src/device/nvme/mod.rs rename to driver/block/nvme/src/lib.rs index f54a3288..42013079 100644 --- a/src/device/nvme/mod.rs +++ b/driver/block/nvme/src/lib.rs @@ -1,15 +1,21 @@ +#![feature(strict_provenance, const_trait_impl)] #![allow(missing_docs)] +#![no_std] + +extern crate alloc; + use core::{mem::size_of, time::Duration}; -use abi::error::Error; use alloc::{boxed::Box, collections::BTreeMap, vec::Vec}; +use command::{IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest}; use device_api::{interrupt::MsiHandler, Device}; +use drive::NvmeDrive; use kernel_util::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, DeviceMemoryIoMut}, }, - runtime, + message_interrupt_controller, runtime, sync::IrqSafeSpinlock, util::OneTimeInit, }; @@ -22,16 +28,11 @@ use ygg_driver_pci::{ capability::{MsiXCapability, MsiXEntry}, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, }; +use yggdrasil_abi::error::Error; use crate::{ - arch::{Architecture, ARCHITECTURE}, - device::nvme::{ - command::{ - IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest, IoRead, IoWrite, - }, - drive::NvmeDrive, - queue::{CompletionQueueEntry, SubmissionQueueEntry}, - }, + command::{IoRead, IoWrite}, + queue::{CompletionQueueEntry, SubmissionQueueEntry}, }; use self::{ @@ -199,7 +200,7 @@ impl NvmeController { self.drive_table.lock().insert(nsid, drive); } Err(error) => { - warnln!("Could not create nvme drive, nsid={}: {:?}", nsid, error); + log::warn!("Could not create nvme drive, nsid={}: {:?}", nsid, error); } } } @@ -217,7 +218,7 @@ impl NvmeController { ) -> Result<(), NvmeError> { let ioq = &self.ioqs.get()[0]; - debugln!("{:?} nsid={}, lba={:#x}", direction, nsid, lba); + log::debug!("{:?} nsid={}, lba={:#x}", direction, nsid, lba); let cmd_id = match direction { IoDirection::Read => ioq.submit( IoRead { @@ -276,7 +277,7 @@ impl Device for NvmeController { } let timeout = Duration::from_millis(regs.CAP.read(CAP::TO) * 500); - debugln!("Worst-case timeout: {:?}", timeout); + log::debug!("Worst-case timeout: {:?}", timeout); while regs.CSTS.matches_any(CSTS::RDY::SET) { core::hint::spin_loop(); @@ -293,7 +294,7 @@ impl Device for NvmeController { // Setup the admin queue (index 0) let admin_sq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, false, 0) }; let admin_cq_doorbell = unsafe { regs.doorbell_ptr(self.doorbell_shift, true, 0) }; - debugln!("sq_doorbell for adminq = {:p}", admin_sq_doorbell); + log::debug!("sq_doorbell for adminq = {:p}", admin_sq_doorbell); let admin_q = QueuePair::new( 0, 0, @@ -322,7 +323,7 @@ impl Device for NvmeController { // Enable the controller regs.CC.modify(CC::ENABLE::SET); - debugln!("Reset the controller"); + log::debug!("Reset the controller"); while !regs.CSTS.matches_any(CSTS::RDY::SET + CSTS::CFS::SET) { core::hint::spin_loop(); @@ -340,7 +341,7 @@ impl Device for NvmeController { // Register vector 0 vt[0] - .register(ARCHITECTURE.message_interrupt_controller(), self) + .register(message_interrupt_controller(), self) .unwrap(); } diff --git a/src/device/nvme/queue.rs b/driver/block/nvme/src/queue.rs similarity index 98% rename from src/device/nvme/queue.rs rename to driver/block/nvme/src/queue.rs index 1381c481..0688bec2 100644 --- a/src/device/nvme/queue.rs +++ b/driver/block/nvme/src/queue.rs @@ -5,7 +5,6 @@ use core::{ task::{Context, Poll}, }; -use abi::error::Error; use alloc::{ collections::{BTreeMap, BTreeSet}, vec::Vec, @@ -21,6 +20,7 @@ use kernel_util::{ sync::IrqSafeSpinlock, }; use static_assertions::const_assert; +use yggdrasil_abi::error::Error; use super::{ command::{Command, Request}, @@ -252,7 +252,7 @@ impl QueuePair { let sq_base = unsafe { sq_data.as_physical_address() }; let cq_base = unsafe { cq_data.as_physical_address() }; - debugln!("Allocated queue pair: sq={:p}, cq={:p}", sq_data, cq_data); + log::debug!("Allocated queue pair: sq={:p}, cq={:p}", sq_data, cq_data); let sq = Queue::new(sq_data, null_mut(), sq_doorbell, true); let cq = Queue::new(cq_data, cq_doorbell, null_mut(), true); diff --git a/driver/fs/kernel-fs/Cargo.toml b/driver/fs/kernel-fs/Cargo.toml new file mode 100644 index 00000000..59a737ee --- /dev/null +++ b/driver/fs/kernel-fs/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "kernel-fs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +vfs = { path = "../../../lib/vfs" } +kernel-util = { path = "../../../lib/kernel-util" } + +log = "0.4.20" diff --git a/src/fs/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs similarity index 76% rename from src/fs/devfs.rs rename to driver/fs/kernel-fs/src/devfs.rs index 74f9651d..8cdd6955 100644 --- a/src/fs/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -1,15 +1,10 @@ //! Device virtual file system use core::sync::atomic::{AtomicUsize, Ordering}; -use abi::error::Error; use alloc::{format, string::String}; use kernel_util::util::OneTimeInit; -use vfs::{ - impls::{read_fn_node, MemoryDirectory}, - BlockDevice, CharDevice, Node, NodeFlags, NodeRef, -}; - -use crate::proc::random; +use vfs::{impls::MemoryDirectory, BlockDevice, CharDevice, Node, NodeFlags, NodeRef}; +use yggdrasil_abi::error::Error; /// Describes the kind of a character device #[derive(Debug)] @@ -39,7 +34,7 @@ pub fn root() -> &'static NodeRef { /// Adds a character device with a custom name pub fn add_named_char_device(dev: &'static dyn CharDevice, name: String) -> Result<(), Error> { - infoln!("Add char device: {}", name); + log::info!("Add char device: {}", name); let node = Node::char(dev, NodeFlags::IN_MEMORY_PROPS); @@ -52,7 +47,7 @@ pub fn add_named_block_device>( name: S, ) -> Result<(), Error> { let name = name.into(); - infoln!("Add block device: {}", name); + log::info!("Add block device: {}", name); let node = Node::block(dev, NodeFlags::IN_MEMORY_PROPS); @@ -74,16 +69,3 @@ pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Re add_named_char_device(dev, name) } - -/// Adds "pseudo"-devices to the filesystem (i.e. /dev/random) -pub fn add_pseudo_devices() -> Result<(), Error> { - let random = read_fn_node(move |_, buf| { - random::read(buf); - Ok(buf.len()) - }); - - let root = root(); - root.add_child("random", random)?; - - Ok(()) -} diff --git a/driver/fs/kernel-fs/src/lib.rs b/driver/fs/kernel-fs/src/lib.rs new file mode 100644 index 00000000..22b952ff --- /dev/null +++ b/driver/fs/kernel-fs/src/lib.rs @@ -0,0 +1,5 @@ +#![no_std] + +extern crate alloc; + +pub mod devfs; diff --git a/lib/kernel-util/Cargo.toml b/lib/kernel-util/Cargo.toml index 781b9149..c953eeaf 100644 --- a/lib/kernel-util/Cargo.toml +++ b/lib/kernel-util/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../device-api", features = ["derive"] } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index b391d89c..0a3c3b2f 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -1,6 +1,7 @@ use core::time::Duration; use alloc::{string::String, sync::Arc}; +use device_api::interrupt::MessageInterruptController; use yggdrasil_abi::{error::Error, process::ExitCode}; use crate::{ @@ -39,4 +40,6 @@ extern "Rust" { pub fn __exit_current(t: &CurrentThread, code: ExitCode) -> !; pub fn __monotonic_timestamp() -> Result; + + pub fn __message_interrupt_controller() -> &'static dyn MessageInterruptController; } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 573a8066..be09a9bc 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -10,6 +10,8 @@ let_chains )] +use device_api::interrupt::MessageInterruptController; + extern crate alloc; pub(crate) mod api; @@ -20,6 +22,11 @@ pub mod sync; pub mod thread; pub mod util; +#[inline] +pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController { + unsafe { api::__message_interrupt_controller() } +} + #[repr(C)] pub struct AlignedTo { pub align: [Align; 0], diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f16783a4..2b259669 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -271,3 +271,8 @@ fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping) { fn __monotonic_timestamp() -> Result { ARCHITECTURE.monotonic_timer().monotonic_timestamp() } + +#[no_mangle] +fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { + ARCHITECTURE.message_interrupt_controller() +} diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index c41b561b..f7b891ca 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,6 +1,7 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; +use kernel_fs::devfs; use kernel_util::runtime; use tock_registers::interfaces::Writeable; use yboot_proto::{ @@ -10,7 +11,6 @@ use yboot_proto::{ use crate::{ arch::x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, - fs::devfs, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, }; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5834257c..6e315029 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -11,6 +11,7 @@ use device_api::{ Device, }; use git_version::git_version; +use kernel_fs::devfs::{self, CharDeviceType}; use kernel_util::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, @@ -51,13 +52,9 @@ use crate::{ device::{ self, display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, - nvme, tty::CombinedTerminal, }, - fs::{ - devfs::{self, CharDeviceType}, - Initrd, INITRD_DATA, - }, + fs::{Initrd, INITRD_DATA}, mem::{ heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, @@ -392,7 +389,7 @@ impl X86_64 { 0x01, Some(0x08), Some(0x02), - nvme::probe, + ygg_driver_nvme::probe, ); match self.boot_data.get() { diff --git a/src/device/mod.rs b/src/device/mod.rs index e5c73988..7fa4d9e5 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -10,8 +10,6 @@ pub mod devtree; #[cfg(not(target_arch = "aarch64"))] pub mod bus; -pub mod nvme; - pub mod display; pub mod power; pub mod serial; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 994a4012..22811030 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -2,14 +2,15 @@ use core::ptr::NonNull; +use kernel_fs::devfs; use kernel_util::{mem::address::PhysicalAddress, util::OneTimeInit}; use memfs::block::{self, BlockAllocator}; -use vfs::NodeRef; +use vfs::{impls::read_fn_node, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::mem::phys; +use crate::{mem::phys, proc::random}; -pub mod devfs; +// pub mod devfs; pub mod sysfs; /// Describes in-memory filesystem image used as initial root @@ -53,3 +54,16 @@ pub fn create_filesystem(options: &MountOptions) -> Result { _ => todo!(), } } + +/// Adds "pseudo"-devices to the filesystem (i.e. /dev/random) +pub fn add_pseudo_devices() -> Result<(), Error> { + let random = read_fn_node(move |_, buf| { + random::read(buf); + Ok(buf.len()) + }); + + let root = devfs::root(); + root.add_child("random", random)?; + + Ok(()) +} diff --git a/src/init.rs b/src/init.rs index 619de99a..dd41b086 100644 --- a/src/init.rs +++ b/src/init.rs @@ -3,11 +3,12 @@ use abi::{ error::Error, io::{OpenOptions, RawFd}, }; +use kernel_fs::devfs; use memfs::MemoryFilesystem; use vfs::{Action, IoContext, NodeRef}; use crate::{ - fs::{devfs, FileBlockAllocator, INITRD_DATA}, + fs::{FileBlockAllocator, INITRD_DATA}, proc, }; diff --git a/src/main.rs b/src/main.rs index e73e5093..69bafe83 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,7 +39,7 @@ use kernel_util::sync::SpinFence; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, - fs::{devfs, sysfs}, + fs::sysfs, mem::heap, proc::random, task::{spawn_kernel_closure, Cpu}, @@ -96,7 +96,7 @@ pub fn kernel_main() -> ! { // Setup the sysfs sysfs::init(); - devfs::add_pseudo_devices().unwrap(); + fs::add_pseudo_devices().unwrap(); unsafe { ARCHITECTURE.start_application_processors(); From 5ffd4ca4e2e0561cde7b03fdd5f8b3cd7309a383 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 11 Dec 2023 00:58:11 +0200 Subject: [PATCH 116/211] dev/block: add a simple block subsystem --- Cargo.toml | 1 + driver/block/block/Cargo.toml | 12 +++++++ driver/block/block/src/cache.rs | 57 ++++++++++++++++++++++++++++++ driver/block/block/src/lib.rs | 39 +++++++++++++++++++++ driver/block/nvme/Cargo.toml | 1 + driver/block/nvme/src/drive.rs | 60 +++++++++++++++++++++++++++++--- driver/fs/kernel-fs/Cargo.toml | 2 ++ driver/fs/kernel-fs/src/devfs.rs | 3 +- lib/vfs/Cargo.toml | 2 ++ lib/vfs/src/device.rs | 33 +----------------- lib/vfs/src/lib.rs | 2 +- lib/vfs/src/node/mod.rs | 5 +-- 12 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 driver/block/block/Cargo.toml create mode 100644 driver/block/block/src/cache.rs create mode 100644 driver/block/block/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 40c95020..c8447ba7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ device-api-macros = { path = "lib/device-api/macros" } # Drivers ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_nvme = { path = "driver/block/nvme" } +ygg_driver_block = { path = "driver/block/block" } kernel-fs = { path = "driver/fs/kernel-fs" } atomic_enum = "0.2.0" diff --git a/driver/block/block/Cargo.toml b/driver/block/block/Cargo.toml new file mode 100644 index 00000000..0513182a --- /dev/null +++ b/driver/block/block/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ygg_driver_block" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } + +log = "0.4.20" diff --git a/driver/block/block/src/cache.rs b/driver/block/block/src/cache.rs new file mode 100644 index 00000000..cd724305 --- /dev/null +++ b/driver/block/block/src/cache.rs @@ -0,0 +1,57 @@ +use core::{fmt, mem::MaybeUninit}; + +use alloc::collections::BTreeMap; +use kernel_util::mem::PageBox; +use yggdrasil_abi::error::Error; + +// TODO LRU +pub struct BlockCache { + table: BTreeMap, + block_size: usize, +} + +pub type CachedBlock = PageBox<[u8]>; + +trait TryGetOrInsertWith { + fn try_get_or_insert_with Result>( + &mut self, + key: K, + f: F, + ) -> Result<&mut V, E>; +} + +impl BlockCache { + pub fn new(block_size: usize) -> Self { + log::debug!("Block cache created with bs={}", block_size); + Self { + table: BTreeMap::new(), + block_size, + } + } + + pub fn get_or_fetch_with< + E: From, + F: FnOnce(&mut PageBox<[MaybeUninit]>) -> Result<(), E>, + >( + &mut self, + index: K, + f: F, + ) -> Result<&mut CachedBlock, E> + where + K: Copy + fmt::Display, + { + if !self.table.contains_key(&index) { + let mut block = PageBox::new_uninit_slice(self.block_size)?; + log::debug!("Missed block with index {}, fetching", index); + f(&mut block)?; + self.table + .insert(index, unsafe { block.assume_init_slice() }); + } + + Ok(self.table.get_mut(&index).unwrap()) + } + + pub fn evict(&mut self, index: K) { + self.table.remove(&index); + } +} diff --git a/driver/block/block/src/lib.rs b/driver/block/block/src/lib.rs new file mode 100644 index 00000000..d89f6fb4 --- /dev/null +++ b/driver/block/block/src/lib.rs @@ -0,0 +1,39 @@ +#![no_std] + +extern crate alloc; + +use yggdrasil_abi::{error::Error, io::DeviceRequest}; + +pub mod cache; + +/// Block device interface +#[allow(unused)] +pub trait BlockDevice: Sync { + /// Reads data frmo the given offset of the device + fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { + Err(Error::NotImplemented) + } + /// Writes the data to the given offset of the device + fn write(&'static self, pos: u64, buf: &[u8]) -> Result { + Err(Error::NotImplemented) + } + + /// Returns the size of the block device in bytes + fn size(&self) -> Result { + Err(Error::NotImplemented) + } + + /// Returns `true` if the device can be read from + fn is_readable(&self) -> bool { + true + } + /// Returns `true` if the device can be written to + fn is_writable(&self) -> bool { + true + } + + /// Performs a device-specific function + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + Err(Error::NotImplemented) + } +} diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml index ca5721ce..725a668e 100644 --- a/driver/block/nvme/Cargo.toml +++ b/driver/block/nvme/Cargo.toml @@ -12,6 +12,7 @@ device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } ygg_driver_pci = { path = "../../bus/pci" } +ygg_driver_block = { path = "../../block/block" } kernel-fs = { path = "../../fs/kernel-fs" } log = "0.4.20" diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index 559d61c0..cf788e62 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -1,15 +1,23 @@ +use core::mem::MaybeUninit; + use alloc::{boxed::Box, format}; use kernel_fs::devfs; -use vfs::BlockDevice; +use kernel_util::{ + block, + mem::{address::AsPhysicalAddress, PageBox}, + sync::IrqSafeSpinlock, +}; +use ygg_driver_block::{cache::BlockCache, BlockDevice}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; -use crate::command::IdentifyNamespaceRequest; +use crate::{command::IdentifyNamespaceRequest, IoDirection}; use super::{error::NvmeError, NvmeController}; #[allow(unused)] pub struct NvmeDrive { controller: &'static NvmeController, + cache: IrqSafeSpinlock>, nsid: u32, total_lba_count: u64, lba_size: u64, @@ -37,6 +45,7 @@ impl NvmeDrive { let dev = Box::leak(Box::new(NvmeDrive { controller, + cache: IrqSafeSpinlock::new(BlockCache::new(lba_size as _)), nsid, total_lba_count, lba_size, @@ -49,11 +58,54 @@ impl NvmeDrive { Ok(dev) } + + async fn read_block( + &self, + lba: u64, + block: &mut PageBox<[MaybeUninit]>, + ) -> Result<(), NvmeError> { + self.controller + .perform_io( + self.nsid, + lba, + unsafe { block.as_physical_address() }, + IoDirection::Read, + ) + .await + } } impl BlockDevice for NvmeDrive { - fn read(&'static self, _pos: u64, _buf: &mut [u8]) -> Result { - todo!() + fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { + let mut cache = self.cache.lock(); + let mut rem = buf.len(); + let mut off = 0; + + while rem != 0 { + let lba = pos / self.lba_size; + + if lba == self.total_lba_count { + break; + } + + let block_offset = (pos % self.lba_size) as usize; + let count = core::cmp::min(self.lba_size as usize - block_offset, rem); + + let block = cache.get_or_fetch_with(lba, |block| { + block! { + self.read_block(lba, block).await + }? + .map_err(|_| Error::InvalidOperation) + })?; + + buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); + + rem -= count; + off += count; + pos += count as u64; + } + + Ok(off) } fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { diff --git a/driver/fs/kernel-fs/Cargo.toml b/driver/fs/kernel-fs/Cargo.toml index 59a737ee..d63a945f 100644 --- a/driver/fs/kernel-fs/Cargo.toml +++ b/driver/fs/kernel-fs/Cargo.toml @@ -10,4 +10,6 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "../../../lib/vfs" } kernel-util = { path = "../../../lib/kernel-util" } +ygg_driver_block = { path = "../../block/block" } + log = "0.4.20" diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 8cdd6955..5d9bb7f7 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -3,7 +3,8 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::{format, string::String}; use kernel_util::util::OneTimeInit; -use vfs::{impls::MemoryDirectory, BlockDevice, CharDevice, Node, NodeFlags, NodeRef}; +use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; +use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; /// Describes the kind of a character device diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index e4d187d8..22f37bae 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -9,6 +9,8 @@ edition = "2021" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } kernel-util = { path = "../kernel-util" } +ygg_driver_block = { path = "../../driver/block/block" } + log = "0.4.20" [dev-dependencies] diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs index b76c6d6c..1015ed99 100644 --- a/lib/vfs/src/device.rs +++ b/lib/vfs/src/device.rs @@ -1,39 +1,8 @@ +use ygg_driver_block::BlockDevice; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::node::{CommonImpl, NodeRef}; -/// Block device interface -#[allow(unused)] -pub trait BlockDevice: Sync { - /// Reads data frmo the given offset of the device - fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { - Err(Error::NotImplemented) - } - /// Writes the data to the given offset of the device - fn write(&'static self, pos: u64, buf: &[u8]) -> Result { - Err(Error::NotImplemented) - } - - /// Returns the size of the block device in bytes - fn size(&self) -> Result { - Err(Error::NotImplemented) - } - - /// Returns `true` if the device can be read from - fn is_readable(&self) -> bool { - true - } - /// Returns `true` if the device can be written to - fn is_writable(&self) -> bool { - true - } - - /// Performs a device-specific function - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - Err(Error::NotImplemented) - } -} - /// Character device interface #[allow(unused)] pub trait CharDevice: Sync { diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 7b265746..8b836512 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -17,7 +17,7 @@ pub(crate) mod node; pub(crate) mod path; pub(crate) mod traits; -pub use device::{BlockDevice, CharDevice}; +pub use device::CharDevice; pub use file::{DirectoryOpenPosition, File, FileRef, InstanceData}; pub use ioctx::{Action, IoContext}; pub use node::{ diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index 9e5a76e4..2030bd52 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -2,6 +2,7 @@ use core::{any::Any, fmt}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use kernel_util::sync::IrqSafeSpinlock; +use ygg_driver_block::BlockDevice; use yggdrasil_abi::{ bitflags, error::Error, @@ -19,7 +20,7 @@ mod tree; pub use access::AccessToken; pub use traits::{CommonImpl, DirectoryImpl, RegularImpl, SymlinkImpl}; -use crate::device::{BlockDevice, BlockDeviceWrapper, CharDevice, CharDeviceWrapper}; +use crate::device::{BlockDeviceWrapper, CharDevice, CharDeviceWrapper}; /// Wrapper type for a [Node] shared reference pub type NodeRef = Arc; @@ -295,7 +296,7 @@ mod tests { use core::any::Any; use std::sync::Arc; - use crate::{node::NodeFlags, NodeRef}; + use crate::node::NodeFlags; use super::{CommonImpl, DirectoryImpl, Node, RegularImpl}; From 4ce7a57c4a3907d4d625222eb025d96416b02520 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 11 Dec 2023 03:04:49 +0200 Subject: [PATCH 117/211] refactor: move drivers + add authors --- Cargo.toml | 6 +++--- driver/block/{block => core}/Cargo.toml | 1 + driver/block/{block => core}/src/cache.rs | 0 driver/block/{block => core}/src/lib.rs | 0 driver/block/nvme/Cargo.toml | 3 ++- driver/bus/pci/Cargo.toml | 1 + driver/fs/kernel-fs/Cargo.toml | 3 ++- {lib => driver/fs}/memfs/Cargo.toml | 5 +++-- {lib => driver/fs}/memfs/src/block.rs | 0 {lib => driver/fs}/memfs/src/bvec.rs | 0 {lib => driver/fs}/memfs/src/dir.rs | 0 {lib => driver/fs}/memfs/src/file.rs | 0 {lib => driver/fs}/memfs/src/lib.rs | 0 {lib => driver/fs}/memfs/src/tar.rs | 0 {lib => driver/fs}/memfs/test/dir1/test1.txt | 0 {lib => driver/fs}/memfs/test/test1.txt | 0 {lib => driver/fs}/memfs/test/test_image.tar | Bin lib/device-api/Cargo.toml | 1 + lib/device-api/macros/Cargo.toml | 1 + lib/hosted-tests/Cargo.toml | 1 + lib/kernel-util/Cargo.toml | 1 + lib/memtables/Cargo.toml | 1 + lib/vfs/Cargo.toml | 3 ++- lib/vmalloc/Cargo.toml | 1 + tools/gentables/Cargo.toml | 1 + 25 files changed, 21 insertions(+), 8 deletions(-) rename driver/block/{block => core}/Cargo.toml (88%) rename driver/block/{block => core}/src/cache.rs (100%) rename driver/block/{block => core}/src/lib.rs (100%) rename {lib => driver/fs}/memfs/Cargo.toml (71%) rename {lib => driver/fs}/memfs/src/block.rs (100%) rename {lib => driver/fs}/memfs/src/bvec.rs (100%) rename {lib => driver/fs}/memfs/src/dir.rs (100%) rename {lib => driver/fs}/memfs/src/file.rs (100%) rename {lib => driver/fs}/memfs/src/lib.rs (100%) rename {lib => driver/fs}/memfs/src/tar.rs (100%) rename {lib => driver/fs}/memfs/test/dir1/test1.txt (100%) rename {lib => driver/fs}/memfs/test/test1.txt (100%) rename {lib => driver/fs}/memfs/test/test_image.tar (100%) diff --git a/Cargo.toml b/Cargo.toml index c8447ba7..86719329 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ name = "yggdrasil-kernel" version = "0.1.0" edition = "2021" build = "build.rs" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [profile.dev.package.tock-registers] @@ -11,8 +12,6 @@ opt-level = 3 [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } -# TODO move to drivers -memfs = { path = "lib/memfs" } device-api = { path = "lib/device-api", features = ["derive"] } kernel-util = { path = "lib/kernel-util" } memtables = { path = "lib/memtables" } @@ -22,8 +21,9 @@ device-api-macros = { path = "lib/device-api/macros" } # Drivers ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_nvme = { path = "driver/block/nvme" } -ygg_driver_block = { path = "driver/block/block" } +ygg_driver_block = { path = "driver/block/core" } kernel-fs = { path = "driver/fs/kernel-fs" } +memfs = { path = "driver/fs/memfs" } atomic_enum = "0.2.0" bitflags = "2.3.3" diff --git a/driver/block/block/Cargo.toml b/driver/block/core/Cargo.toml similarity index 88% rename from driver/block/block/Cargo.toml rename to driver/block/core/Cargo.toml index 0513182a..afded29d 100644 --- a/driver/block/block/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -2,6 +2,7 @@ name = "ygg_driver_block" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/driver/block/block/src/cache.rs b/driver/block/core/src/cache.rs similarity index 100% rename from driver/block/block/src/cache.rs rename to driver/block/core/src/cache.rs diff --git a/driver/block/block/src/lib.rs b/driver/block/core/src/lib.rs similarity index 100% rename from driver/block/block/src/lib.rs rename to driver/block/core/src/lib.rs diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml index 725a668e..467cc2d4 100644 --- a/driver/block/nvme/Cargo.toml +++ b/driver/block/nvme/Cargo.toml @@ -2,6 +2,7 @@ name = "ygg_driver_nvme" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -12,7 +13,7 @@ device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } ygg_driver_pci = { path = "../../bus/pci" } -ygg_driver_block = { path = "../../block/block" } +ygg_driver_block = { path = "../../block/core" } kernel-fs = { path = "../../fs/kernel-fs" } log = "0.4.20" diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index 5268dc89..8ef36ab4 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -2,6 +2,7 @@ name = "ygg_driver_pci" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/driver/fs/kernel-fs/Cargo.toml b/driver/fs/kernel-fs/Cargo.toml index d63a945f..7aa8d3b6 100644 --- a/driver/fs/kernel-fs/Cargo.toml +++ b/driver/fs/kernel-fs/Cargo.toml @@ -2,6 +2,7 @@ name = "kernel-fs" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -10,6 +11,6 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "../../../lib/vfs" } kernel-util = { path = "../../../lib/kernel-util" } -ygg_driver_block = { path = "../../block/block" } +ygg_driver_block = { path = "../../block/core" } log = "0.4.20" diff --git a/lib/memfs/Cargo.toml b/driver/fs/memfs/Cargo.toml similarity index 71% rename from lib/memfs/Cargo.toml rename to driver/fs/memfs/Cargo.toml index fae1b5db..444de5de 100644 --- a/lib/memfs/Cargo.toml +++ b/driver/fs/memfs/Cargo.toml @@ -2,13 +2,14 @@ name = "memfs" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../kernel-util" } -vfs = { path = "../vfs" } +kernel-util = { path = "../../../lib/kernel-util" } +vfs = { path = "../../../lib/vfs" } static_assertions = "1.1.0" log = "0.4.20" diff --git a/lib/memfs/src/block.rs b/driver/fs/memfs/src/block.rs similarity index 100% rename from lib/memfs/src/block.rs rename to driver/fs/memfs/src/block.rs diff --git a/lib/memfs/src/bvec.rs b/driver/fs/memfs/src/bvec.rs similarity index 100% rename from lib/memfs/src/bvec.rs rename to driver/fs/memfs/src/bvec.rs diff --git a/lib/memfs/src/dir.rs b/driver/fs/memfs/src/dir.rs similarity index 100% rename from lib/memfs/src/dir.rs rename to driver/fs/memfs/src/dir.rs diff --git a/lib/memfs/src/file.rs b/driver/fs/memfs/src/file.rs similarity index 100% rename from lib/memfs/src/file.rs rename to driver/fs/memfs/src/file.rs diff --git a/lib/memfs/src/lib.rs b/driver/fs/memfs/src/lib.rs similarity index 100% rename from lib/memfs/src/lib.rs rename to driver/fs/memfs/src/lib.rs diff --git a/lib/memfs/src/tar.rs b/driver/fs/memfs/src/tar.rs similarity index 100% rename from lib/memfs/src/tar.rs rename to driver/fs/memfs/src/tar.rs diff --git a/lib/memfs/test/dir1/test1.txt b/driver/fs/memfs/test/dir1/test1.txt similarity index 100% rename from lib/memfs/test/dir1/test1.txt rename to driver/fs/memfs/test/dir1/test1.txt diff --git a/lib/memfs/test/test1.txt b/driver/fs/memfs/test/test1.txt similarity index 100% rename from lib/memfs/test/test1.txt rename to driver/fs/memfs/test/test1.txt diff --git a/lib/memfs/test/test_image.tar b/driver/fs/memfs/test/test_image.tar similarity index 100% rename from lib/memfs/test/test_image.tar rename to driver/fs/memfs/test/test_image.tar diff --git a/lib/device-api/Cargo.toml b/lib/device-api/Cargo.toml index ff5aca07..bab8d602 100644 --- a/lib/device-api/Cargo.toml +++ b/lib/device-api/Cargo.toml @@ -2,6 +2,7 @@ name = "device-api" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/device-api/macros/Cargo.toml b/lib/device-api/macros/Cargo.toml index 8c3983aa..6d7dd27f 100644 --- a/lib/device-api/macros/Cargo.toml +++ b/lib/device-api/macros/Cargo.toml @@ -2,6 +2,7 @@ name = "device-api-macros" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/hosted-tests/Cargo.toml b/lib/hosted-tests/Cargo.toml index 2699c8a0..2cdf5853 100644 --- a/lib/hosted-tests/Cargo.toml +++ b/lib/hosted-tests/Cargo.toml @@ -2,6 +2,7 @@ name = "hosted-tests" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/kernel-util/Cargo.toml b/lib/kernel-util/Cargo.toml index c953eeaf..217d5daa 100644 --- a/lib/kernel-util/Cargo.toml +++ b/lib/kernel-util/Cargo.toml @@ -2,6 +2,7 @@ name = "kernel-util" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/memtables/Cargo.toml b/lib/memtables/Cargo.toml index 2697e627..061db54a 100644 --- a/lib/memtables/Cargo.toml +++ b/lib/memtables/Cargo.toml @@ -2,6 +2,7 @@ name = "memtables" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 22f37bae..bdb44b29 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -2,6 +2,7 @@ name = "vfs" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -9,7 +10,7 @@ edition = "2021" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } kernel-util = { path = "../kernel-util" } -ygg_driver_block = { path = "../../driver/block/block" } +ygg_driver_block = { path = "../../driver/block/core" } log = "0.4.20" diff --git a/lib/vmalloc/Cargo.toml b/lib/vmalloc/Cargo.toml index a5062b55..24ba6152 100644 --- a/lib/vmalloc/Cargo.toml +++ b/lib/vmalloc/Cargo.toml @@ -2,6 +2,7 @@ name = "vmalloc" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/tools/gentables/Cargo.toml b/tools/gentables/Cargo.toml index 2e0a20e3..6598260b 100644 --- a/tools/gentables/Cargo.toml +++ b/tools/gentables/Cargo.toml @@ -2,6 +2,7 @@ name = "gentables" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 61f217ab56e728cb9fed356c2143ab2e0df0fc64 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 11 Dec 2023 21:13:33 +0200 Subject: [PATCH 118/211] block/nvme: multi-queue + multi-MSI --- driver/block/core/src/cache.rs | 7 +- driver/block/nvme/src/lib.rs | 202 +++++++++++++++++++----------- driver/block/nvme/src/queue.rs | 6 +- driver/bus/pci/src/capability.rs | 80 +++++++++--- lib/device-api/src/interrupt.rs | 32 ++++- lib/device-api/src/lib.rs | 1 + lib/kernel-util/src/api.rs | 3 + lib/kernel-util/src/lib.rs | 10 ++ lib/kernel-util/src/mem/device.rs | 20 +++ lib/kernel-util/src/mem/mod.rs | 20 +++ lib/kernel-util/src/mem/table.rs | 26 ++-- lib/kernel-util/src/sync.rs | 9 +- lib/kernel-util/src/thread.rs | 8 +- src/arch/mod.rs | 12 +- src/arch/x86_64/apic/local.rs | 66 ++++++---- src/arch/x86_64/apic/mod.rs | 2 +- 16 files changed, 367 insertions(+), 137 deletions(-) diff --git a/driver/block/core/src/cache.rs b/driver/block/core/src/cache.rs index cd724305..69dba62a 100644 --- a/driver/block/core/src/cache.rs +++ b/driver/block/core/src/cache.rs @@ -1,6 +1,6 @@ use core::{fmt, mem::MaybeUninit}; -use alloc::collections::BTreeMap; +use alloc::collections::{btree_map::Entry, BTreeMap}; use kernel_util::mem::PageBox; use yggdrasil_abi::error::Error; @@ -40,12 +40,11 @@ impl BlockCache { where K: Copy + fmt::Display, { - if !self.table.contains_key(&index) { + if let Entry::Vacant(entry) = self.table.entry(index) { let mut block = PageBox::new_uninit_slice(self.block_size)?; log::debug!("Missed block with index {}, fetching", index); f(&mut block)?; - self.table - .insert(index, unsafe { block.assume_init_slice() }); + entry.insert(unsafe { block.assume_init_slice() }); } Ok(self.table.get_mut(&index).unwrap()) diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index 42013079..d70bb98e 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -1,22 +1,30 @@ -#![feature(strict_provenance, const_trait_impl)] +#![feature(strict_provenance, const_trait_impl, let_chains)] #![allow(missing_docs)] #![no_std] extern crate alloc; -use core::{mem::size_of, time::Duration}; +use core::{ + mem::size_of, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, +}; use alloc::{boxed::Box, collections::BTreeMap, vec::Vec}; use command::{IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest}; -use device_api::{interrupt::MsiHandler, Device}; +use device_api::{ + interrupt::{InterruptAffinity, MsiHandler}, + Device, +}; use drive::NvmeDrive; use kernel_util::{ + cpu_count, cpu_index, mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, - device::{DeviceMemoryIo, DeviceMemoryIoMut}, + device::DeviceMemoryIo, }, message_interrupt_controller, runtime, - sync::IrqSafeSpinlock, + sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; use tock_registers::{ @@ -25,7 +33,7 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; use ygg_driver_pci::{ - capability::{MsiXCapability, MsiXEntry}, + capability::{MsiXCapability, MsiXVectorTable}, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, }; use yggdrasil_abi::error::Error; @@ -116,10 +124,12 @@ pub struct NvmeController { regs: IrqSafeSpinlock>, admin_q: OneTimeInit, ioqs: OneTimeInit>, - vector_table: IrqSafeSpinlock>, + io_queue_count: AtomicUsize, drive_table: IrqSafeSpinlock>, controller_id: OneTimeInit, + vector_table: IrqSafeSpinlock>, + doorbell_shift: usize, } @@ -138,7 +148,86 @@ impl Regs { } impl NvmeController { + const ADMIN_QUEUE_SIZE: usize = 32; + const IO_QUEUE_SIZE: usize = 32; + + async fn create_queues(&'static self) -> Result<(), NvmeError> { + let admin_q = self.admin_q.get(); + let io_queue_count = self.io_queue_count.load(Ordering::Acquire); + + log::info!( + "Creating {} queue pairs for nvme{}", + io_queue_count, + self.controller_id.get() + ); + + // Request a CQ/SQ pair for I/O + admin_q + .request_no_data(SetFeatureRequest::NumberOfQueues( + io_queue_count as _, + io_queue_count as _, + )) + .await?; + + let mut queues = Vec::new(); + for i in 1..=io_queue_count { + let id = i as u32; + + let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(i) }; + let queue = QueuePair::new(id, i, Self::IO_QUEUE_SIZE, sq_doorbell, cq_doorbell) + .map_err(NvmeError::MemoryError)?; + + admin_q + .request_no_data(CreateIoCompletionQueue { + id, + vector: id, + size: Self::IO_QUEUE_SIZE, + data: queue.cq_physical_pointer(), + }) + .await?; + + admin_q + .request_no_data(CreateIoSubmissionQueue { + id, + cq_id: id, + size: Self::IO_QUEUE_SIZE, + data: queue.sq_physical_pointer(), + }) + .await?; + + queues.push(queue); + } + + self.ioqs.init(queues); + + Ok(()) + } + async fn late_init(&'static self) -> Result<(), NvmeError> { + let io_queue_count = cpu_count(); + self.io_queue_count.store(io_queue_count, Ordering::Release); + + { + // Register io_queue_count + 1 vectors + // TODO register vectors on different CPUs + let mut vt = self.vector_table.lock(); + + let range = vt + .register_range( + 0, + io_queue_count + 1, + message_interrupt_controller(), + InterruptAffinity::Any, + self, + ) + .unwrap(); + + // TODO handle different MSI range allocations + for (i, msi) in range.iter().enumerate() { + assert_eq!(i, msi.vector); + } + } + register_nvme_controller(self); let admin_q = self.admin_q.get(); @@ -148,35 +237,7 @@ impl NvmeController { // TODO do something with identify_controller - // Request a CQ/SQ pair for I/O - admin_q - .request_no_data(SetFeatureRequest::NumberOfQueues(1, 1)) - .await?; - - // Allocate the queue - let (sq_doorbell, cq_doorbell) = unsafe { self.doorbell_pair(1) }; - let io_q = - QueuePair::new(1, 0, 32, sq_doorbell, cq_doorbell).map_err(NvmeError::MemoryError)?; - - // Create the queue on the device side - admin_q - .request_no_data(CreateIoCompletionQueue { - id: 1, - size: 32, - vector: 0, - data: io_q.cq_physical_pointer(), - }) - .await?; - admin_q - .request_no_data(CreateIoSubmissionQueue { - id: 1, - cq_id: 1, - size: 32, - data: io_q.sq_physical_pointer(), - }) - .await?; - - self.ioqs.init(Vec::from_iter([io_q])); + self.create_queues().await?; // Identify namespaces self.enumerate_namespaces().await?; @@ -208,7 +269,6 @@ impl NvmeController { Ok(()) } - // TODO sane methods for IO pub async fn perform_io( &'static self, nsid: u32, @@ -216,9 +276,18 @@ impl NvmeController { buffer_address: PhysicalAddress, direction: IoDirection, ) -> Result<(), NvmeError> { - let ioq = &self.ioqs.get()[0]; + let _guard = IrqGuard::acquire(); + let cpu_index = cpu_index(); + let ioq = &self.ioqs.get()[cpu_index]; + + log::debug!( + "{:?} ioq #{}, nsid={}, lba={:#x}", + direction, + cpu_index, + nsid, + lba + ); - log::debug!("{:?} nsid={}, lba={:#x}", direction, nsid, lba); let cmd_id = match direction { IoDirection::Read => ioq.submit( IoRead { @@ -254,15 +323,16 @@ impl NvmeController { } impl MsiHandler for NvmeController { - fn handle_msi(&self, _vector: usize) -> bool { - // TODO check MSI-X pending bits - self.admin_q.get().process_completions(); - if let Some(qs) = self.ioqs.try_get() { - for q in qs { - q.process_completions(); - } + fn handle_msi(&self, vector: usize) -> bool { + if vector == 0 { + self.admin_q.get().process_completions() != 0 + } else if vector <= self.io_queue_count.load(Ordering::Acquire) + && let Some(ioqs) = self.ioqs.try_get() + { + ioqs[vector - 1].process_completions() != 0 + } else { + false } - true } } @@ -283,8 +353,7 @@ impl Device for NvmeController { core::hint::spin_loop(); } - let queue_slots = 32; - if queue_slots > regs.CAP.read(CAP::MQES) + 1 { + if Self::ADMIN_QUEUE_SIZE as u64 > regs.CAP.read(CAP::MQES) + 1 { todo!( "queue_slots too big, max = {}", regs.CAP.read(CAP::MQES) + 1 @@ -298,14 +367,16 @@ impl Device for NvmeController { let admin_q = QueuePair::new( 0, 0, - queue_slots as usize, + Self::ADMIN_QUEUE_SIZE, admin_sq_doorbell, admin_cq_doorbell, ) .unwrap(); - regs.AQA - .modify(AQA::ASQS.val(queue_slots as u32 - 1) + AQA::ACQS.val(queue_slots as u32 - 1)); + regs.AQA.modify( + AQA::ASQS.val(Self::ADMIN_QUEUE_SIZE as u32 - 1) + + AQA::ACQS.val(Self::ADMIN_QUEUE_SIZE as u32 - 1), + ); regs.ASQ.set(admin_q.sq_physical_pointer().into_raw()); regs.ACQ.set(admin_q.cq_physical_pointer().into_raw()); @@ -335,16 +406,6 @@ impl Device for NvmeController { self.admin_q.init(admin_q); - // Register the IRQs (TODO: use multiple) - { - let mut vt = self.vector_table.lock(); - - // Register vector 0 - vt[0] - .register(message_interrupt_controller(), self) - .unwrap(); - } - // Schedule late_init task runtime::spawn(self.late_init())?; @@ -356,11 +417,6 @@ impl Device for NvmeController { } } -// impl FromPciBus for NvmeController { -// fn from_pci_bus(info: &PciDeviceInfo) -> Result { -// } -// } - static NVME_CONTROLLERS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); @@ -373,9 +429,10 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { let mut msix = info.config_space.capability::().unwrap(); let mut vt = msix.vector_table()?; - for vector in vt.iter_mut() { - vector.set_masked(true); - } + // TODO is this really needed? PCI spec says this is masked on reset, though I'm not sure if + // firmware puts it back in masked state after loading the kernel + vt.mask_all(); + msix.set_function_mask(false); msix.set_enabled(true); let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); @@ -394,9 +451,12 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { regs: IrqSafeSpinlock::new(regs), admin_q: OneTimeInit::new(), ioqs: OneTimeInit::new(), - vector_table: IrqSafeSpinlock::new(vt), drive_table: IrqSafeSpinlock::new(BTreeMap::new()), controller_id: OneTimeInit::new(), + + vector_table: IrqSafeSpinlock::new(vt), + + io_queue_count: AtomicUsize::new(1), doorbell_shift, }))) } diff --git a/driver/block/nvme/src/queue.rs b/driver/block/nvme/src/queue.rs index 0688bec2..e36dbffd 100644 --- a/driver/block/nvme/src/queue.rs +++ b/driver/block/nvme/src/queue.rs @@ -356,10 +356,10 @@ impl QueuePair { command_id } - pub fn request_no_data<'r, C: Command>( - &'r self, + pub fn request_no_data( + &self, req: C, - ) -> impl Future> + 'r { + ) -> impl Future> + '_ { let command_id = self.submit(req, &[], true); self.wait_for_completion(command_id, ()) } diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index ddd324bd..de335793 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -1,6 +1,7 @@ //! PCI capability structures and queries -use device_api::interrupt::{MessageInterruptController, MsiHandler}; +use alloc::{vec, vec::Vec}; +use device_api::interrupt::{InterruptAffinity, MessageInterruptController, MsiHandler, MsiInfo}; use kernel_util::mem::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIoMut, @@ -24,6 +25,10 @@ pub struct MsiXEntry { pub control: u32, } +pub struct MsiXVectorTable<'a> { + vectors: DeviceMemoryIoMut<'a, [MsiXEntry]>, +} + /// MSI-X capability data structure pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> { space: &'s S, @@ -45,7 +50,7 @@ impl PciCapability for MsiXCapability { impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { // TODO use pending bits as well /// Maps and returns the vector table associated with the device's MSI-X capability - pub fn vector_table<'a>(&self) -> Result, Error> { + pub fn vector_table<'a>(&self) -> Result, Error> { let w0 = self.space.read_u16(self.offset + 2); let dw1 = self.space.read_u32(self.offset + 4); @@ -63,7 +68,10 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { log::debug!("MSI-X table address: {:#x}", base + table_offset); unsafe { - DeviceMemoryIoMut::map_slice(PhysicalAddress::from_raw(base + table_offset), table_size) + MsiXVectorTable::from_raw_parts( + PhysicalAddress::from_raw(base + table_offset), + table_size, + ) } } @@ -78,28 +86,66 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { } self.space.write_u32(self.offset, w0); } + + pub fn set_function_mask(&mut self, masked: bool) { + let mut w0 = self.space.read_u32(self.offset); + if masked { + w0 |= 1 << 30; + } else { + w0 &= !(1 << 30); + } + self.space.write_u32(self.offset, w0); + } +} + +impl MsiXVectorTable<'_> { + unsafe fn from_raw_parts(base: PhysicalAddress, len: usize) -> Result { + let vectors = DeviceMemoryIoMut::map_slice(base, len)?; + Ok(Self { vectors }) + } + + pub fn mask_all(&mut self) { + for vector in self.vectors.iter_mut() { + vector.set_masked(true); + } + } + + pub fn register_range( + &mut self, + start: usize, + end: usize, + ic: &C, + affinity: InterruptAffinity, + handler: &'static dyn MsiHandler, + ) -> Result, Error> { + assert!(end > start); + let mut range = vec![ + MsiInfo { + affinity, + ..Default::default() + }; + end - start + ]; + ic.register_msi_range(&mut range, handler)?; + + for (i, info) in range.iter().enumerate() { + let index = i + start; + self.vectors[index].address = info.address as _; + self.vectors[index].data = info.value; + self.vectors[index].set_masked(false); + } + + Ok(range) + } } impl MsiXEntry { /// If set, prevents the MSI-X interrupt from being delivered - pub fn set_masked(&mut self, masked: bool) { + fn set_masked(&mut self, masked: bool) { if masked { self.control |= 1; } else { self.control &= !1; } } - - /// Registers the MSI-X vector with the interrupt controller and enables it - pub fn register( - &mut self, - ic: &C, - handler: &'static dyn MsiHandler, - ) -> Result<(), Error> { - let info = ic.register_msi(handler)?; - self.address = info.address as _; - self.data = info.value as _; - self.set_masked(false); - Ok(()) - } } diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index f64e026e..03d85f6b 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -33,11 +33,19 @@ pub struct IrqOptions { pub trigger: IrqTrigger, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Default)] pub struct MsiInfo { pub address: usize, pub value: u32, pub vector: usize, + pub affinity: InterruptAffinity, +} + +#[derive(Clone, Copy, Debug, Default)] +pub enum InterruptAffinity { + #[default] + Any, + Specific(usize), } pub trait InterruptTable { @@ -45,7 +53,27 @@ pub trait InterruptTable { } pub trait MessageInterruptController { - fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result; + fn register_msi( + &self, + affinity: InterruptAffinity, + handler: &'static dyn MsiHandler, + ) -> Result { + let mut range = [MsiInfo { + affinity, + ..Default::default() + }]; + self.register_msi_range(&mut range, handler)?; + Ok(range[0]) + } + + #[allow(unused)] + fn register_msi_range( + &self, + range: &mut [MsiInfo], + handler: &'static dyn MsiHandler, + ) -> Result<(), Error> { + Err(Error::NotImplemented) + } fn handle_msi(&self, #[allow(unused)] vector: usize) {} } diff --git a/lib/device-api/src/lib.rs b/lib/device-api/src/lib.rs index 927b1f45..30357982 100644 --- a/lib/device-api/src/lib.rs +++ b/lib/device-api/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(trait_alias)] #![no_std] extern crate alloc; diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index 0a3c3b2f..6771f96b 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -13,6 +13,9 @@ extern "Rust" { pub fn __acquire_irq_guard() -> bool; pub fn __release_irq_guard(mask: bool); + pub fn __cpu_index() -> usize; + pub fn __cpu_count() -> usize; + pub fn __allocate_2m_page() -> Result; pub fn __allocate_page() -> Result; pub fn __allocate_contiguous_pages(count: usize) -> Result; diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index be09a9bc..35d3b856 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -27,6 +27,16 @@ pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController unsafe { api::__message_interrupt_controller() } } +#[inline] +pub fn cpu_index() -> usize { + unsafe { api::__cpu_index() } +} + +#[inline] +pub fn cpu_count() -> usize { + unsafe { api::__cpu_count() } +} + #[repr(C)] pub struct AlignedTo { pub align: [Align; 0], diff --git a/lib/kernel-util/src/mem/device.rs b/lib/kernel-util/src/mem/device.rs index 4e39b31e..44fd7408 100644 --- a/lib/kernel-util/src/mem/device.rs +++ b/lib/kernel-util/src/mem/device.rs @@ -117,6 +117,26 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { Ok(DeviceMemoryIo { inner, value }) } + /// Maps a physical address as device memory of type `[T]`. + /// + /// # Safety + /// + /// The caller must ensure the address actually points to a value of type `T`, as well as + /// proper access synchronization. The caller must also ensure the `len` is valid. + pub unsafe fn map_slice( + base: PhysicalAddress, + count: usize, + ) -> Result, Error> { + let layout = Layout::array::(count).unwrap(); + let inner = RawDeviceMemoryMapping::map(base, layout.size())?; + let value = core::slice::from_raw_parts(inner.address as *mut T, count); + + Ok(DeviceMemoryIo { + inner: Arc::new(inner), + value, + }) + } + /// Maps a physical address as device memory of type `T`. /// /// # Safety diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 3140f216..39d40fd9 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -120,6 +120,11 @@ impl PageBox { } impl PageBox> { + /// Consumes the [PageBox], returning a new one with [MaybeUninit] removed. + /// + /// # Safety + /// + /// See [MaybeUninit::assume_init_mut]. pub unsafe fn assume_init(self) -> PageBox { // SAFETY: Memory-safe, as: // 1. MaybeUninit is transparent @@ -135,6 +140,11 @@ impl PageBox> { } impl PageBox<[MaybeUninit]> { + /// Consumes the [PageBox], returning a new one with [MaybeUninit] removed. + /// + /// # Safety + /// + /// See [MaybeUninit::slice_assume_init_mut]. pub unsafe fn assume_init_slice(self) -> PageBox<[T]> { // SAFETY: Memory-safe, as: // 1. MaybeUninit is transparent @@ -147,10 +157,20 @@ impl PageBox<[MaybeUninit]> { PageBox { value, page_count } } + /// Returns a reference to the slice data with [MaybeUninit] removed. + /// + /// # Safety + /// + /// See [MaybeUninit::slice_assume_init_ref] pub unsafe fn assume_init_slice_ref(&self) -> &[T] { MaybeUninit::slice_assume_init_ref(self.deref()) } + /// Returns a mutable reference to the slice data with [MaybeUninit] removed. + /// + /// # Safety + /// + /// See [MaybeUninit::slice_assume_init_mut] pub unsafe fn assume_init_slice_mut(&mut self) -> &mut [T] { MaybeUninit::slice_assume_init_mut(self.deref_mut()) } diff --git a/lib/kernel-util/src/mem/table.rs b/lib/kernel-util/src/mem/table.rs index 15203094..2254d107 100644 --- a/lib/kernel-util/src/mem/table.rs +++ b/lib/kernel-util/src/mem/table.rs @@ -10,16 +10,16 @@ pub trait EntryLevel: Copy { #[const_trait] pub trait EntryLevelExt: Sized { - fn page_index(self) -> usize; - fn page_offset(self) -> usize; - fn page_count(self) -> usize; - fn page_align_up(self) -> Self; - fn page_align_down(self) -> Self; - fn is_page_aligned_for(self) -> bool; + fn page_index(&self) -> usize; + fn page_offset(&self) -> usize; + fn page_count(&self) -> usize; + fn page_align_up(&self) -> Self; + fn page_align_down(&self) -> Self; + fn is_page_aligned_for(&self) -> bool; } #[const_trait] -trait AddressLike: Sized { +trait AddressLike: Sized + Copy { fn into_usize(self) -> usize; fn from_usize(v: usize) -> Self; } @@ -48,32 +48,32 @@ impl const AddressLike for PhysicalAddress { impl const EntryLevelExt for T { #[inline(always)] - fn page_index(self) -> usize { + fn page_index(&self) -> usize { (self.into_usize() >> L::SHIFT) & 0x1FF } #[inline(always)] - fn page_offset(self) -> usize { + fn page_offset(&self) -> usize { self.into_usize() & (L::SIZE - 1) } #[inline(always)] - fn page_count(self) -> usize { + fn page_count(&self) -> usize { (self.into_usize() + L::SIZE - 1) / L::SIZE } #[inline(always)] - fn page_align_up(self) -> Self { + fn page_align_up(&self) -> Self { Self::from_usize((self.into_usize() + L::SIZE - 1) & !(L::SIZE - 1)) } #[inline(always)] - fn page_align_down(self) -> Self { + fn page_align_down(&self) -> Self { Self::from_usize(self.into_usize() & !(L::SIZE - 1)) } #[inline(always)] - fn is_page_aligned_for(self) -> bool { + fn is_page_aligned_for(&self) -> bool { self.page_offset::() == 0 } } diff --git a/lib/kernel-util/src/sync.rs b/lib/kernel-util/src/sync.rs index 146d8613..ceb8cce3 100644 --- a/lib/kernel-util/src/sync.rs +++ b/lib/kernel-util/src/sync.rs @@ -149,11 +149,18 @@ impl IrqSafeSpinlock { } impl IrqSafeSpinlock { - pub fn cloned(&self) -> T { + pub fn get_cloned(&self) -> T { self.lock().clone() } } +impl Clone for IrqSafeSpinlock { + fn clone(&self) -> Self { + let inner = self.lock(); + IrqSafeSpinlock::new(inner.clone()) + } +} + impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { type Target = T; diff --git a/lib/kernel-util/src/thread.rs b/lib/kernel-util/src/thread.rs index 384fb36d..6ef469b5 100644 --- a/lib/kernel-util/src/thread.rs +++ b/lib/kernel-util/src/thread.rs @@ -42,11 +42,13 @@ impl Termination for () { } impl Thread { - pub fn spawn, F: FnOnce() + Send + 'static>( + pub fn spawn, T: Termination, F: FnOnce() -> T + Send + 'static>( name: S, f: F, ) -> Result<(), Error> { - extern "C" fn closure_wrapper(closure_addr: usize) -> ! { + extern "C" fn closure_wrapper T + Send + 'static>( + closure_addr: usize, + ) -> ! { let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; let result = closure(); Thread::current().exit(result.into_exit_code()); @@ -58,7 +60,7 @@ impl Thread { let thread = unsafe { api::__create_kthread( name.into(), - closure_wrapper::, + closure_wrapper::, Box::into_raw(closure) as usize, ) }?; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 2b259669..b14fc17b 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -34,7 +34,7 @@ use kernel_util::mem::{ address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, }; -use crate::mem::phys::PhysicalMemoryRegion; +use crate::{mem::phys::PhysicalMemoryRegion, task::Cpu}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -245,6 +245,16 @@ fn __release_irq_guard(mask: bool) { } } +#[no_mangle] +fn __cpu_index() -> usize { + Cpu::local_id() as _ +} + +#[no_mangle] +fn __cpu_count() -> usize { + ArchitectureImpl::cpu_count() +} + #[no_mangle] fn __virtualize(addr: u64) -> usize { ArchitectureImpl::virtualize(addr) diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 1a5b3894..8e21a03f 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -2,11 +2,11 @@ use core::sync::atomic::Ordering; use abi::error::Error; -use alloc::vec::Vec; +use alloc::{vec, vec::Vec}; use device_api::{ interrupt::{ - IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, MsiHandler, - MsiInfo, + InterruptAffinity, IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, + MsiHandler, MsiInfo, }, Device, }; @@ -16,7 +16,7 @@ use kernel_util::{ device::DeviceMemoryIo, table::EntryLevelExt, }, - sync::IrqSafeSpinlock, + sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; use tock_registers::{ @@ -37,6 +37,7 @@ use crate::{ use super::{ APIC_IPI_VECTOR, APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR, + MAX_MSI_VECTORS, }; const TIMER_INTERVAL: u32 = 150000; @@ -145,7 +146,8 @@ register_structs! { /// Per-processor local APIC interface pub struct LocalApic { regs: DeviceMemoryIo<'static, Regs>, - msi_vectors: IrqSafeSpinlock>, + id: u32, + msi_vectors: Vec>>, } unsafe impl Send for LocalApic {} @@ -160,10 +162,11 @@ impl Device for LocalApic { impl MessageInterruptController for LocalApic { fn handle_msi(&self, vector: usize) { // TODO this is ugly + let row = &self.msi_vectors[vector]; let mut i = 0; loop { - let table = self.msi_vectors.lock(); + let table = row.lock(); let Some(&handler) = table.get(i) else { break; }; @@ -177,24 +180,42 @@ impl MessageInterruptController for LocalApic { } } - fn register_msi(&self, handler: &'static dyn MsiHandler) -> Result { - // TODO only 1 ISR vector allocated for MSIs - let vector = 0; - let mut table = self.msi_vectors.lock(); + fn register_msi_range( + &self, + range: &mut [MsiInfo], + handler: &'static dyn MsiHandler, + ) -> Result<(), Error> { + let _guard = IrqGuard::acquire(); - table.push(handler); + // TODO fill smallest vectors first + // TODO don't ignore affinity - // TODO magic numbers - let apic_vector = 32 + APIC_MSI_OFFSET + vector; + for (i, msi) in range.iter_mut().enumerate() { + let row = &self.msi_vectors[i]; + let mut row = row.lock(); - let value = apic_vector; - let address = Self::base(); + row.push(handler); - Ok(MsiInfo { - address: address.into_raw(), - value, - vector: vector as _, - }) + infoln!( + "Bind {}:{} -> apic{}:msi{}", + handler.display_name(), + i, + self.id, + i + ); + + let value = 32 + APIC_MSI_OFFSET + i as u32; + let address = IntoRaw::::into_raw(Self::base()) | ((self.id as usize) << 12); + + *msi = MsiInfo { + address, + value, + vector: i, + affinity: InterruptAffinity::Specific(self.id as _), + }; + } + + Ok(()) } } @@ -280,9 +301,12 @@ impl LocalApic { LocalVectorEntry::Mask::Masked + LocalVectorEntry::Vector.val(APIC_LINT1_VECTOR + 32), ); + let msi_vectors = vec![IrqSafeSpinlock::new(Vec::new()); MAX_MSI_VECTORS as _]; + Self { + id, regs, - msi_vectors: IrqSafeSpinlock::new(Vec::new()), + msi_vectors, } } diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index c1c4f8cb..f82c2a2e 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -78,7 +78,7 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { } unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) { - if vector != 0 { + if vector >= MAX_MSI_VECTORS as _ { todo!("Got a weird MSI with vector {}", vector); } From cd8d3f3363e19eb34a75b13c04ccb796592c2967 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 8 Dec 2023 17:51:21 +0200 Subject: [PATCH 119/211] doc: add a README --- README.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..6706bf0b --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +yggdrasil-kernel +================ + +Rust Unix-like operating system kernel. + +See also: + +* [ABI for kernel-user communication](https://git.alnyan.me/yggdrasil-abi) +* [Rust fork to use with the kernel](https://git.alnyan.me/yggdrasil/yggdrasil-rust) +* [Userspace programs](https://git.alnyan.me/yggdrasil/yggdrasil-userspace) +* [yboot — x86-64 UEFI bootloader](https://git.alnyan.me/yggdrasil/yboot) + +Main features +------------- + +* Architecture support: [aarch64](/src/arch/aarch64) and [x86_64](/src/arch/x86_64) +* Kernel/userspace preemptive multithreading +* Kernel-space multitasking with `async`/`await` runtime +* Symmetric Multiprocessing +* Unix-like virtual filesystem: + files, directories, block/char devices, symlinks, mounts +* In-memory read-write filesystem for tar-based initrd +* sysfs/devfs +* Binary formats: ELF + `#!/...` shebangs +* Rust-style interfaces for most of the stuff like memory management, devices etc. + +aarch64-specific: + +* PSCI for SMP start-up and power control +* PL011 serial port +* ARM generic timer as system/monotonic timer +* GICv2 IRQ controller + +x86_64-specific: + +* UEFI boot through [yboot](https://git.alnyan.me/yggdrasil/yboot) + (no plans for legacy boot) +* PCIe, with plans to extend to aarch64 as well +* I/O and Local APIC IRQ controllers +* PS/2 keyboard, +* i8253-based timer (got some problems with HPET on + real hw, had to revert, lol) +* COM ports +* ACPI, [work in progress](https://github.com/rust-osdev/acpi), mostly broken + on real hardware + * ACPI shutdown + * PCI IRQ pin routing + * Events like power button, etc. +* Fancy framebuffer console + +Userspace features: + +* Sanitized system calls better suited for Rust +* Userspace threads +* Synchronization primitives through futex-like interface +* Unix-like signals and exceptions + +General plans (in no particular order) +-------------------------------------- + +* Better unification of architecture code +* `async` for VFS (?) +* PCIe NVMe block device +* PCIe SATA block device +* PCIe XHCI USB devices +* Better algorithms for memory management + +Navigation +---------- + +* `src/arch` — architecture-specific code +* `src/device` — device driver implementations + * `bus` — bus devices like USB, PCIe etc. + * `display` — everything related to graphic displays + * `power` — power and reset controllers + * `serial` — serial transceiver drivers + * `devtree.rs` — stuff related to ARM DeviceTree + * `tty.rs` — Unix-style terminal driver implementation +* `src/fs` — in-kernel filesystems (sysfs/devfs) +* `src/mem` — memory management +* `src/proc` — process information management +* `src/syscall` — system call handling +* `src/task` — kernel and userspace tasks, processes and threads +* `src/util` — utilities used within the kernel +* `src/init.rs` — kernel init thread impl. \ No newline at end of file From 2efa5b8ff4ff595193643772c4068616e1257ae0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 11 Dec 2023 21:21:15 +0200 Subject: [PATCH 120/211] readme: Add an entry on NVMe --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6706bf0b..779207b5 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,13 @@ aarch64-specific: x86_64-specific: -* UEFI boot through [yboot](https://git.alnyan.me/yggdrasil/yboot) +* UEFI boot through [yboot](https://git.alnyan.me/yggdrasil/yboot) (no plans for legacy boot) * PCIe, with plans to extend to aarch64 as well + * NVMe drive support (read/write) * I/O and Local APIC IRQ controllers -* PS/2 keyboard, -* i8253-based timer (got some problems with HPET on +* PS/2 keyboard, +* i8253-based timer (got some problems with HPET on real hw, had to revert, lol) * COM ports * ACPI, [work in progress](https://github.com/rust-osdev/acpi), mostly broken @@ -82,4 +83,4 @@ Navigation * `src/syscall` — system call handling * `src/task` — kernel and userspace tasks, processes and threads * `src/util` — utilities used within the kernel -* `src/init.rs` — kernel init thread impl. \ No newline at end of file +* `src/init.rs` — kernel init thread impl. From 6848f6c56d3b524facd2dd9b14a3ed6c7eadb96d Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 14 Dec 2023 18:45:56 +0200 Subject: [PATCH 121/211] char/pipe: implement pipes --- lib/vfs/Cargo.toml | 1 + lib/vfs/src/file/mod.rs | 23 +++- lib/vfs/src/file/pipe.rs | 226 +++++++++++++++++++++++++++++++++++++++ src/proc/io.rs | 12 ++- src/syscall/mod.rs | 32 +++++- src/task/process.rs | 2 +- 6 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 lib/vfs/src/file/pipe.rs diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index bdb44b29..0a292833 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -13,6 +13,7 @@ kernel-util = { path = "../kernel-util" } ygg_driver_block = { path = "../../driver/block/core" } log = "0.4.20" +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } [dev-dependencies] hosted-tests = { path = "../hosted-tests" } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 00f0b5c8..6ff08c9e 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -16,11 +16,13 @@ use crate::{ use self::{ device::{BlockFile, CharFile}, directory::DirectoryFile, + pipe::PipeEnd, regular::RegularFile, }; mod device; mod directory; +mod pipe; mod regular; /// Per-file optional instance data created when a regular file is opened @@ -45,9 +47,19 @@ pub enum File { Regular(RegularFile), Block(BlockFile), Char(CharFile), + AnonymousPipe(PipeEnd), } impl File { + /// Constructs a pipe pair, returning its `(read, write)` ends + pub fn new_pipe_pair(capacity: usize) -> (Arc, Arc) { + let (read, write) = PipeEnd::new_pair(capacity); + ( + Arc::new(Self::AnonymousPipe(read)), + Arc::new(Self::AnonymousPipe(write)), + ) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -133,6 +145,7 @@ impl File { Self::Regular(file) => Some(&file.node), Self::Block(file) => Some(&file.node), Self::Char(file) => Some(&file.node), + Self::AnonymousPipe(_) => None, } } } @@ -143,6 +156,7 @@ impl Read for File { Self::Regular(file) => file.read(buf), Self::Block(file) => file.read(buf), Self::Char(file) => file.read(buf), + Self::AnonymousPipe(pipe) => pipe.read(buf), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -154,6 +168,7 @@ impl Write for File { Self::Regular(file) => file.write(buf), Self::Block(file) => file.write(buf), Self::Char(file) => file.write(buf), + Self::AnonymousPipe(pipe) => pipe.write(buf), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -164,7 +179,7 @@ impl Seek for File { match self { Self::Regular(file) => Ok(*file.position.lock()), Self::Block(file) => Ok(*file.position.lock()), - Self::Char(_) => Err(Error::InvalidOperation), + Self::Char(_) | Self::AnonymousPipe(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -173,7 +188,7 @@ impl Seek for File { match self { Self::Regular(file) => file.seek(from), Self::Block(file) => file.seek(from), - Self::Char(_) => Err(Error::InvalidOperation), + Self::Char(_) | Self::AnonymousPipe(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -200,6 +215,7 @@ impl fmt::Debug for File { .field("write", &file.write) .finish_non_exhaustive(), Self::Directory(_) => f.debug_struct("DirectoryFile").finish_non_exhaustive(), + Self::AnonymousPipe(_) => f.debug_struct("AnonymousPipe").finish_non_exhaustive(), } } } @@ -210,6 +226,7 @@ mod tests { use std::sync::{Arc, Mutex}; use kernel_util::sync::IrqSafeSpinlock; + use ygg_driver_block::BlockDevice; use yggdrasil_abi::{ error::Error, io::{DirectoryEntry, FileType, OpenOptions, SeekFrom}, @@ -217,7 +234,7 @@ mod tests { }; use crate::{ - device::{BlockDevice, CharDevice}, + device::CharDevice, file::DirectoryOpenPosition, impls::const_value_node, node::{AccessToken, CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl}, diff --git a/lib/vfs/src/file/pipe.rs b/lib/vfs/src/file/pipe.rs new file mode 100644 index 00000000..7e662344 --- /dev/null +++ b/lib/vfs/src/file/pipe.rs @@ -0,0 +1,226 @@ +use core::{ + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, + task::{Context, Poll}, +}; + +use alloc::{sync::Arc, vec, vec::Vec}; +use futures_util::{task::AtomicWaker, Future}; +use kernel_util::{block, sync::IrqSafeSpinlock}; +use yggdrasil_abi::error::Error; + +struct PipeInner { + data: Vec, + capacity: usize, + rd: usize, + wr: usize, +} + +pub struct Pipe { + inner: IrqSafeSpinlock, + shutdown: AtomicBool, + read_notify: AtomicWaker, + write_notify: AtomicWaker, +} + +pub enum PipeEnd { + Read(Arc), + Write(Arc), +} + +impl PipeInner { + pub fn new(capacity: usize) -> Self { + Self { + data: vec![0; capacity], + capacity, + rd: 0, + wr: 0, + } + } + + pub fn can_write(&self) -> bool { + (self.wr + 1) % self.capacity != self.rd + } + + pub fn can_read(&self) -> bool { + self.rd != self.wr + } + + pub unsafe fn write(&mut self, val: u8) { + self.data[self.wr] = val; + self.wr = (self.wr + 1) % self.capacity; + } + + pub unsafe fn read(&mut self) -> u8 { + let val = self.data[self.rd]; + self.rd = (self.rd + 1) % self.capacity; + val + } + + fn try_write(&mut self, val: u8) -> bool { + if self.can_write() { + unsafe { + self.write(val); + } + true + } else { + false + } + } + + fn try_read(&mut self) -> Option { + if self.can_read() { + Some(unsafe { self.read() }) + } else { + None + } + } +} + +impl Pipe { + pub fn new(capacity: usize) -> Self { + Self { + inner: IrqSafeSpinlock::new(PipeInner::new(capacity)), + shutdown: AtomicBool::new(false), + read_notify: AtomicWaker::new(), + write_notify: AtomicWaker::new(), + } + } + + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + self.read_notify.wake(); + self.write_notify.wake(); + } + + pub fn blocking_write(&self, val: u8) -> impl Future> + '_ { + struct F<'a> { + pipe: &'a Pipe, + val: u8, + } + + impl<'a> Future for F<'a> { + type Output = Result<(), Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut lock = self.pipe.inner.lock(); + + // Try fast path before acquiring write notify to avoid unnecessary contention + if self.pipe.shutdown.load(Ordering::Acquire) { + // TODO BrokenPipe + return Poll::Ready(Err(Error::ReadOnly)); + } else if lock.try_write(self.val) { + self.pipe.read_notify.wake(); + return Poll::Ready(Ok(())); + } + + self.pipe.write_notify.register(cx.waker()); + + if self.pipe.shutdown.load(Ordering::Acquire) { + Poll::Ready(Err(Error::ReadOnly)) + } else if lock.try_write(self.val) { + self.pipe.read_notify.wake(); + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + } + + F { pipe: self, val } + } + + pub fn blocking_read(&self) -> impl Future> + '_ { + struct F<'a> { + pipe: &'a Pipe, + } + + impl<'a> Future for F<'a> { + type Output = Option; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut lock = self.pipe.inner.lock(); + + if let Some(val) = lock.try_read() { + self.pipe.write_notify.wake(); + return Poll::Ready(Some(val)); + } else if self.pipe.shutdown.load(Ordering::Acquire) { + return Poll::Ready(None); + } + + self.pipe.read_notify.register(cx.waker()); + + if let Some(val) = lock.try_read() { + Poll::Ready(Some(val)) + } else if self.pipe.shutdown.load(Ordering::Acquire) { + Poll::Ready(None) + } else { + Poll::Pending + } + } + } + + F { pipe: self } + } +} + +impl PipeEnd { + pub fn new_pair(capacity: usize) -> (PipeEnd, PipeEnd) { + let pipe = Arc::new(Pipe::new(capacity)); + let read = PipeEnd::Read(pipe.clone()); + let write = PipeEnd::Write(pipe); + + (read, write) + } + + pub fn read(&self, buf: &mut [u8]) -> Result { + let PipeEnd::Read(read) = self else { + return Err(Error::InvalidOperation); + }; + + block! { + let mut pos = 0; + let mut rem = buf.len(); + + while rem != 0 { + if let Some(val) = read.blocking_read().await { + buf[pos] = val; + pos += 1; + rem -= 1; + } else { + break; + } + } + + Ok(pos) + }? + } + + pub fn write(&self, buf: &[u8]) -> Result { + let PipeEnd::Write(write) = self else { + return Err(Error::InvalidOperation); + }; + + block! { + let mut pos = 0; + let mut rem = buf.len(); + + while rem != 0 { + write.blocking_write(buf[pos]).await?; + pos += 1; + rem -= 1; + } + + Ok(pos) + }? + } +} + +impl Drop for PipeEnd { + fn drop(&mut self) { + match self { + Self::Read(read) => read.shutdown(), + Self::Write(write) => write.shutdown(), + } + } +} diff --git a/src/proc/io.rs b/src/proc/io.rs index 160e56a9..cc156865 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -62,12 +62,20 @@ impl ProcessIo { /// Removes and closes a [FileRef] from the struct pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { // Do nothing, file will be dropped and closed - self.files.remove(&fd).ok_or(Error::InvalidFile)?; - Ok(()) + if self.files.remove(&fd).is_some() { + Ok(()) + } else { + Err(Error::InvalidFile) + } } /// Removes all [FileRef]s from the struct which do not pass the `predicate` check pub fn retain bool>(&mut self, predicate: F) { self.files.retain(predicate); } + + /// Handles process exit by closing all of its files + pub fn handle_exit(&mut self) { + self.files.clear(); + } } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index a881a1b5..98f25d35 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -9,7 +9,7 @@ use abi::{ }; use alloc::sync::Arc; use kernel_util::{block, runtime, sync::IrqSafeSpinlockGuard}; -use vfs::{IoContext, NodeRef, Read, Seek, Write}; +use vfs::{File, IoContext, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ @@ -188,7 +188,17 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); run_with_io(process, |mut io| { - io.close_file(fd)?; + let res = io.close_file(fd); + + match res { + Err(Error::InvalidFile) => { + warnln!("Double close of fd {:?} in process {}", fd, process.id()); + } + _ => (), + } + + res?; + Ok(0) }) } @@ -262,6 +272,24 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::RemoveDirectory => { todo!() } + SyscallFunction::CreatePipe => { + let ends: &mut [MaybeUninit; 2] = arg_user_mut(args[0] as usize)?; + + run_with_io(process, |mut io| { + let (read, write) = File::new_pipe_pair(256); + + let read_fd = io.place_file(read)?; + let write_fd = io.place_file(write)?; + + infoln!("Read end: {:?}", read_fd); + infoln!("Write end: {:?}", write_fd); + + ends[0].write(read_fd); + ends[1].write(write_fd); + + Ok(0) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; diff --git a/src/task/process.rs b/src/task/process.rs index 261e02e8..446c4284 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -346,7 +346,7 @@ impl Process { inner.state = ProcessState::Terminated(code); // XXX - // self.io.lock().handle_exit(); + self.io.lock().handle_exit(); drop(inner); From b888a51805d3999199ef5cb1c7cb63937f1542ff Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 18 Dec 2023 17:04:01 +0200 Subject: [PATCH 122/211] block/core: better block device interface --- driver/block/core/Cargo.toml | 1 + driver/block/core/src/cache.rs | 56 ------ driver/block/core/src/device.rs | 292 +++++++++++++++++++++++++++++++ driver/block/core/src/lib.rs | 6 +- driver/block/core/src/request.rs | 19 ++ driver/block/nvme/src/drive.rs | 122 ++++++------- driver/block/nvme/src/lib.rs | 2 +- driver/block/nvme/src/queue.rs | 12 +- lib/hosted-tests/Cargo.toml | 1 + lib/hosted-tests/src/lib.rs | 18 ++ lib/kernel-util/src/lib.rs | 3 +- 11 files changed, 411 insertions(+), 121 deletions(-) delete mode 100644 driver/block/core/src/cache.rs create mode 100644 driver/block/core/src/device.rs create mode 100644 driver/block/core/src/request.rs diff --git a/driver/block/core/Cargo.toml b/driver/block/core/Cargo.toml index afded29d..bfcc1875 100644 --- a/driver/block/core/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -11,3 +11,4 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-util = { path = "../../../lib/kernel-util" } log = "0.4.20" +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/driver/block/core/src/cache.rs b/driver/block/core/src/cache.rs deleted file mode 100644 index 69dba62a..00000000 --- a/driver/block/core/src/cache.rs +++ /dev/null @@ -1,56 +0,0 @@ -use core::{fmt, mem::MaybeUninit}; - -use alloc::collections::{btree_map::Entry, BTreeMap}; -use kernel_util::mem::PageBox; -use yggdrasil_abi::error::Error; - -// TODO LRU -pub struct BlockCache { - table: BTreeMap, - block_size: usize, -} - -pub type CachedBlock = PageBox<[u8]>; - -trait TryGetOrInsertWith { - fn try_get_or_insert_with Result>( - &mut self, - key: K, - f: F, - ) -> Result<&mut V, E>; -} - -impl BlockCache { - pub fn new(block_size: usize) -> Self { - log::debug!("Block cache created with bs={}", block_size); - Self { - table: BTreeMap::new(), - block_size, - } - } - - pub fn get_or_fetch_with< - E: From, - F: FnOnce(&mut PageBox<[MaybeUninit]>) -> Result<(), E>, - >( - &mut self, - index: K, - f: F, - ) -> Result<&mut CachedBlock, E> - where - K: Copy + fmt::Display, - { - if let Entry::Vacant(entry) = self.table.entry(index) { - let mut block = PageBox::new_uninit_slice(self.block_size)?; - log::debug!("Missed block with index {}, fetching", index); - f(&mut block)?; - entry.insert(unsafe { block.assume_init_slice() }); - } - - Ok(self.table.get_mut(&index).unwrap()) - } - - pub fn evict(&mut self, index: K) { - self.table.remove(&index); - } -} diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs new file mode 100644 index 00000000..3812b19e --- /dev/null +++ b/driver/block/core/src/device.rs @@ -0,0 +1,292 @@ +use core::{ + ops::Range, + pin::Pin, + task::{Context, Poll}, +}; + +use alloc::boxed::Box; +use futures_util::Future; +use kernel_util::{block, mem::PageBox, runtime::QueueWaker}; +use yggdrasil_abi::{error::Error, io::DeviceRequest}; + +use crate::{ + request::{IoOperation, IoRequest, IoSubmissionId}, + BlockDevice, +}; + +pub trait NgBlockDevice: Sync { + fn bus_id(&self) -> u32; // HBA, controller ID, etc. + fn unit_id(&self) -> u32; // Drive, slot, connector ID, etc. + + fn block_size(&self) -> u64; + fn block_count(&self) -> u64; + fn max_blocks_per_request(&self) -> u64; + + fn submit_request( + &self, + request: IoRequest, + ) -> impl Future> + Send; + + fn poll_completion(&self, id: IoSubmissionId) -> Poll>; + fn completion_notify(&self, id: IoSubmissionId) -> &QueueWaker; + + fn wait_for_completion( + &self, + id: IoSubmissionId, + ) -> impl Future> + Send + '_ + where + Self: Sized, + { + struct F<'f, D: NgBlockDevice + 'f> { + device: &'f D, + notify: &'f QueueWaker, + id: IoSubmissionId, + } + + impl<'f, D: NgBlockDevice + 'f> Future for F<'f, D> { + type Output = Result<(), Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.notify.register(cx.waker()); + match self.device.poll_completion(self.id) { + Poll::Ready(result) => { + self.notify.remove(cx.waker()); + Poll::Ready(result) + } + Poll::Pending => Poll::Pending, + } + } + } + + F { + device: self, + notify: self.completion_notify(id), + id, + } + } +} + +pub struct NgBlockDeviceWrapper<'a, D: NgBlockDevice + 'a> { + device: &'a D, + + block_size: u64, + block_count: u64, + #[allow(unused)] + max_blocks_per_request: u64, +} + +#[derive(Debug, PartialEq)] +struct BlockChunk { + lba_start: u64, + lba_count: usize, + buffer_offset: usize, + lba_offset: usize, + byte_count: usize, +} + +struct BlockChunkIter { + remaining: usize, + buffer_offset: usize, + position: u64, + + block_size: u64, + max_blocks_per_request: u64, +} + +impl BlockChunk { + pub fn block_range(&self) -> Range { + self.lba_offset..self.lba_offset + self.byte_count + } + + pub fn buffer_range(&self) -> Range { + self.buffer_offset..self.buffer_offset + self.byte_count + } +} + +impl BlockChunkIter { + pub fn new(pos: u64, count: usize, lba_size: u64, max_lba_per_request: u64) -> Self { + Self { + remaining: count, + buffer_offset: 0, + position: pos, + + block_size: lba_size, + max_blocks_per_request: max_lba_per_request, + } + } +} + +impl Iterator for BlockChunkIter { + type Item = BlockChunk; + + fn next(&mut self) -> Option { + if self.remaining == 0 { + return None; + } + + let lba_start = self.position / self.block_size; + let lba_end = + (self.position + self.remaining as u64 + self.block_size - 1) / self.block_size; + + let lba_count = core::cmp::min(lba_end - lba_start, self.max_blocks_per_request); + + let lba_offset = (self.position % self.block_size) as usize; + let byte_count = core::cmp::min( + (lba_count * self.block_size) as usize - lba_offset, + self.remaining, + ); + + let buffer_offset = self.buffer_offset; + + self.position += byte_count as u64; + self.buffer_offset += byte_count; + self.remaining -= byte_count; + + Some(BlockChunk { + lba_start, + lba_count: lba_count as usize, + buffer_offset, + lba_offset, + byte_count, + }) + } +} + +impl<'a, D: NgBlockDevice + 'a> NgBlockDeviceWrapper<'a, D> { + pub fn new(device: &'a D) -> &'a dyn BlockDevice { + let block_size = device.block_size(); + let block_count = device.block_count(); + let max_blocks_per_request = device.max_blocks_per_request(); + + Box::leak(Box::new(Self { + device, + block_size, + block_count, + max_blocks_per_request, + })) + } + + async fn read_range_inner(&self, lba: u64, count: usize) -> Result, Error> { + let mut data = PageBox::new_uninit_slice(self.block_size as usize * count)?; + + let id = self + .device + .submit_request(IoRequest { + operation: IoOperation::Read { lba, count }, + data: &mut data, + }) + .await?; + + self.device.wait_for_completion(id).await?; + + Ok(unsafe { data.assume_init_slice() }) + } +} + +impl<'a, D: NgBlockDevice + 'a> BlockDevice for NgBlockDeviceWrapper<'a, D> { + fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { + // TODO block cache + block! { + let mut bytes_read = 0; + + for chunk in + BlockChunkIter::new(pos, buf.len(), self.block_size, self.max_blocks_per_request) + { + log::debug!( + "Read chunk: lba_start={}, lba_count={}", + chunk.lba_start, + chunk.lba_count + ); + + let block = self.read_range_inner(chunk.lba_start, chunk.lba_count).await?; + + buf[chunk.buffer_range()].copy_from_slice(&block[chunk.block_range()]); + + bytes_read += chunk.byte_count; + } + + Ok(bytes_read) + }? + } + + fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { + todo!() + } + + fn size(&self) -> Result { + Ok(self.block_size * self.block_count) + } + + fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { + todo!() + } +} + +#[cfg(test)] +mod tests { + use crate::device::BlockChunk; + + use super::BlockChunkIter; + + #[test] + fn block_chunk_iter() { + let mut it = BlockChunkIter { + remaining: 512 * 9 + 1, + position: 123, + block_size: 512, + buffer_offset: 0, + max_blocks_per_request: 2, + }; + + assert_eq!( + it.next().unwrap(), + BlockChunk { + lba_start: 0, + lba_count: 2, + buffer_offset: 0, + lba_offset: 123, + byte_count: 901 + } + ); + assert_eq!( + it.next().unwrap(), + BlockChunk { + lba_start: 2, + lba_count: 2, + buffer_offset: 1024 - 123, + lba_offset: 0, + byte_count: 1024 + } + ); + assert_eq!( + it.next().unwrap(), + BlockChunk { + lba_start: 4, + lba_count: 2, + buffer_offset: 2 * 1024 - 123, + lba_offset: 0, + byte_count: 1024 + } + ); + assert_eq!( + it.next().unwrap(), + BlockChunk { + lba_start: 6, + lba_count: 2, + buffer_offset: 3 * 1024 - 123, + lba_offset: 0, + byte_count: 1024 + } + ); + assert_eq!( + it.next().unwrap(), + BlockChunk { + lba_start: 8, + lba_count: 2, + buffer_offset: 4 * 1024 - 123, + lba_offset: 0, + byte_count: 512 + 123 + 1 + } + ); + } +} diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index d89f6fb4..e32c574e 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -4,7 +4,11 @@ extern crate alloc; use yggdrasil_abi::{error::Error, io::DeviceRequest}; -pub mod cache; +pub mod device; +pub mod request; + +pub use device::{NgBlockDevice, NgBlockDeviceWrapper}; +pub use request::{IoOperation, IoRequest, IoSubmissionId}; /// Block device interface #[allow(unused)] diff --git a/driver/block/core/src/request.rs b/driver/block/core/src/request.rs new file mode 100644 index 00000000..2fbb634e --- /dev/null +++ b/driver/block/core/src/request.rs @@ -0,0 +1,19 @@ +use core::mem::MaybeUninit; + +use kernel_util::mem::PageBox; + +pub enum IoOperation { + Read { lba: u64, count: usize }, + Write { lba: u64, count: usize }, +} + +pub struct IoRequest<'a> { + pub operation: IoOperation, + pub data: &'a mut PageBox<[MaybeUninit]>, +} + +#[derive(Clone, Copy, Debug)] +pub struct IoSubmissionId { + pub queue_id: usize, + pub command_id: usize, +} diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index cf788e62..6931d501 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -1,23 +1,20 @@ -use core::mem::MaybeUninit; +use core::task::Poll; use alloc::{boxed::Box, format}; use kernel_fs::devfs; -use kernel_util::{ - block, - mem::{address::AsPhysicalAddress, PageBox}, - sync::IrqSafeSpinlock, +use kernel_util::{cpu_index, mem::address::AsPhysicalAddress, runtime::QueueWaker}; +use ygg_driver_block::{ + IoOperation, IoRequest, IoSubmissionId, NgBlockDevice, NgBlockDeviceWrapper, }; -use ygg_driver_block::{cache::BlockCache, BlockDevice}; -use yggdrasil_abi::{error::Error, io::DeviceRequest}; +use yggdrasil_abi::error::Error; -use crate::{command::IdentifyNamespaceRequest, IoDirection}; +use crate::command::{IdentifyNamespaceRequest, IoRead}; use super::{error::NvmeError, NvmeController}; #[allow(unused)] pub struct NvmeDrive { controller: &'static NvmeController, - cache: IrqSafeSpinlock>, nsid: u32, total_lba_count: u64, lba_size: u64, @@ -45,78 +42,81 @@ impl NvmeDrive { let dev = Box::leak(Box::new(NvmeDrive { controller, - cache: IrqSafeSpinlock::new(BlockCache::new(lba_size as _)), nsid, total_lba_count, lba_size, })); let node_name = format!("nvme{}n{}", controller.controller_id.get(), nsid); - devfs::add_named_block_device(dev, node_name).ok(); + let blk = NgBlockDeviceWrapper::new(dev); + devfs::add_named_block_device(blk, node_name).ok(); // TODO probe partitions Ok(dev) } - - async fn read_block( - &self, - lba: u64, - block: &mut PageBox<[MaybeUninit]>, - ) -> Result<(), NvmeError> { - self.controller - .perform_io( - self.nsid, - lba, - unsafe { block.as_physical_address() }, - IoDirection::Read, - ) - .await - } } -impl BlockDevice for NvmeDrive { - fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { - let mut cache = self.cache.lock(); - let mut rem = buf.len(); - let mut off = 0; +impl NgBlockDevice for NvmeDrive { + fn bus_id(&self) -> u32 { + (*self.controller.controller_id.get()) as _ + } - while rem != 0 { - let lba = pos / self.lba_size; + fn unit_id(&self) -> u32 { + self.nsid + } - if lba == self.total_lba_count { - break; + fn block_size(&self) -> u64 { + self.lba_size + } + + fn block_count(&self) -> u64 { + self.total_lba_count + } + + fn max_blocks_per_request(&self) -> u64 { + // TODO get from identify + 8 + } + + async fn submit_request(&self, request: IoRequest<'_>) -> Result { + let queue_id = cpu_index(); + let ioq = &self.controller.ioqs.get()[queue_id]; + + let command_id = match request.operation { + IoOperation::Read { lba, count } => { + log::debug!( + "Submit read of {} lbas from ns {} to queue {}", + count, + self.nsid, + queue_id + ); + let range = unsafe { request.data.as_physical_address() }; + ioq.submit( + IoRead { + lba, + count: count as _, + nsid: self.nsid, + }, + &[range], + true, + ) } + IoOperation::Write { .. } => todo!(), + }; - let block_offset = (pos % self.lba_size) as usize; - let count = core::cmp::min(self.lba_size as usize - block_offset, rem); - - let block = cache.get_or_fetch_with(lba, |block| { - block! { - self.read_block(lba, block).await - }? - .map_err(|_| Error::InvalidOperation) - })?; - - buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); - - rem -= count; - off += count; - pos += count as u64; - } - - Ok(off) + Ok(IoSubmissionId { + queue_id: queue_id as _, + command_id: command_id as _, + }) } - fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { - todo!() + fn poll_completion(&self, id: IoSubmissionId) -> Poll> { + let ioq = &self.controller.ioqs.get()[id.queue_id]; + ioq.poll_completion(id.command_id as _) } - fn size(&self) -> Result { - Ok(self.lba_size * self.total_lba_count) - } - - fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { - todo!() + fn completion_notify(&self, id: IoSubmissionId) -> &QueueWaker { + &self.controller.ioqs.get()[id.queue_id].completion_notify } } diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index d70bb98e..3c133c9b 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(strict_provenance, const_trait_impl, let_chains)] +#![feature(strict_provenance, const_trait_impl, let_chains, if_let_guard)] #![allow(missing_docs)] #![no_std] diff --git a/driver/block/nvme/src/queue.rs b/driver/block/nvme/src/queue.rs index e36dbffd..050df88a 100644 --- a/driver/block/nvme/src/queue.rs +++ b/driver/block/nvme/src/queue.rs @@ -97,7 +97,7 @@ pub struct QueuePair { sq_base: PhysicalAddress, cq_base: PhysicalAddress, - completion_notify: QueueWaker, + pub completion_notify: QueueWaker, inner: IrqSafeSpinlock, } @@ -285,6 +285,16 @@ impl QueuePair { self.cq_base } + pub fn poll_completion(&self, command_id: u32) -> Poll> { + let mut inner = self.inner.lock(); + + match inner.completed.remove(&command_id) { + Some(result) if let Some(error) = result.error() => todo!(), + Some(_) => Poll::Ready(Ok(())), + None => Poll::Pending, + } + } + pub fn wait_for_completion<'r, T: Unpin + 'r>( &'r self, command_id: u32, diff --git a/lib/hosted-tests/Cargo.toml b/lib/hosted-tests/Cargo.toml index 2cdf5853..9fbf978d 100644 --- a/lib/hosted-tests/Cargo.toml +++ b/lib/hosted-tests/Cargo.toml @@ -7,3 +7,4 @@ authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/lib/hosted-tests/src/lib.rs b/lib/hosted-tests/src/lib.rs index aa7fc159..f85277bb 100644 --- a/lib/hosted-tests/src/lib.rs +++ b/lib/hosted-tests/src/lib.rs @@ -1,5 +1,10 @@ #![no_std] +extern crate alloc; + +use alloc::sync::Arc; +use yggdrasil_abi::error::Error; + #[no_mangle] fn __acquire_irq_guard() -> bool { false @@ -7,3 +12,16 @@ fn __acquire_irq_guard() -> bool { #[no_mangle] fn __release_irq_guard(_: bool) {} + +#[no_mangle] +fn __current_thread() -> Arc<()> { + Arc::new(()) +} + +#[no_mangle] +fn __suspend_current(_: &()) -> Result<(), Error> { + todo!(); +} + +#[no_mangle] +fn __enqueue(_: &Arc<()>) {} diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 35d3b856..73f29421 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -7,7 +7,8 @@ slice_ptr_get, strict_provenance, never_type, - let_chains + let_chains, + allocator_api )] use device_api::interrupt::MessageInterruptController; From 6efc2d1ed530814c71916072880fbaaa2da17fb4 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 18 Dec 2023 18:35:55 +0200 Subject: [PATCH 123/211] block/core: add GPT partition probing --- driver/block/core/Cargo.toml | 3 + driver/block/core/src/device.rs | 6 +- driver/block/core/src/lib.rs | 53 +++++++++++ driver/block/core/src/partition.rs | 137 +++++++++++++++++++++++++++++ driver/block/nvme/src/drive.rs | 10 ++- driver/fs/kernel-fs/src/devfs.rs | 13 +++ 6 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 driver/block/core/src/partition.rs diff --git a/driver/block/core/Cargo.toml b/driver/block/core/Cargo.toml index bfcc1875..cdcc3170 100644 --- a/driver/block/core/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -12,3 +12,6 @@ kernel-util = { path = "../../../lib/kernel-util" } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } +bytemuck = { version = "1.14.0", features = ["derive"] } +static_assertions = "1.1.0" +uuid = { version = "1.6.1", default-features = false, features = ["bytemuck"] } diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index 3812b19e..71e3b86e 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -69,8 +69,8 @@ pub trait NgBlockDevice: Sync { pub struct NgBlockDeviceWrapper<'a, D: NgBlockDevice + 'a> { device: &'a D, - block_size: u64, - block_count: u64, + pub(crate) block_size: u64, + pub(crate) block_count: u64, #[allow(unused)] max_blocks_per_request: u64, } @@ -153,7 +153,7 @@ impl Iterator for BlockChunkIter { } impl<'a, D: NgBlockDevice + 'a> NgBlockDeviceWrapper<'a, D> { - pub fn new(device: &'a D) -> &'a dyn BlockDevice { + pub fn new(device: &'a D) -> &'a Self { let block_size = device.block_size(); let block_count = device.block_count(); let max_blocks_per_request = device.max_blocks_per_request(); diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index e32c574e..cca39db7 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -2,14 +2,58 @@ extern crate alloc; +use alloc::{boxed::Box, vec::Vec}; +use kernel_util::runtime; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; +mod partition; pub mod request; pub use device::{NgBlockDevice, NgBlockDeviceWrapper}; pub use request::{IoOperation, IoRequest, IoSubmissionId}; +use crate::partition::Partition; + +pub fn probe_partitions< + D: NgBlockDevice + 'static, + F: Fn(usize, &'static dyn BlockDevice) -> Result<(), Error> + Send + 'static, +>( + dev: &'static NgBlockDeviceWrapper, + callback: F, +) -> Result<(), Error> { + async fn probe_table( + dev: &'static NgBlockDeviceWrapper<'static, D>, + ) -> Result>>, Error> { + if let Some(partitions) = partition::probe_gpt(dev)? { + return Ok(Some(partitions)); + } + + Ok(None) + } + + runtime::spawn(async move { + match probe_table(dev).await { + Ok(Some(partitions)) => { + // Create block devices for the partitions + for (i, partition) in partitions.into_iter().enumerate() { + let partition_blkdev = Box::leak(Box::new(partition)); + + if let Err(error) = callback(i, partition_blkdev) { + log::warn!("Could not add partition {}: {:?}", i, error); + } + } + } + Ok(None) => { + log::warn!("Unknown or missing partition table"); + } + Err(error) => { + log::warn!("Could not probe partition table: {:?}", error); + } + } + }) +} + /// Block device interface #[allow(unused)] pub trait BlockDevice: Sync { @@ -40,4 +84,13 @@ pub trait BlockDevice: Sync { fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { Err(Error::NotImplemented) } + + fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> { + let count = self.read(pos, buf)?; + if count == buf.len() { + Ok(()) + } else { + Err(Error::MissingData) + } + } } diff --git a/driver/block/core/src/partition.rs b/driver/block/core/src/partition.rs new file mode 100644 index 00000000..ee59aeb3 --- /dev/null +++ b/driver/block/core/src/partition.rs @@ -0,0 +1,137 @@ +use core::mem::{size_of, MaybeUninit}; + +use alloc::{vec, vec::Vec}; +use bytemuck::{Pod, Zeroable}; +use kernel_util::mem::PageBox; +use static_assertions::const_assert_eq; +use uuid::Uuid; +use yggdrasil_abi::{error::Error, io::DeviceRequest}; + +use crate::{BlockDevice, NgBlockDevice, NgBlockDeviceWrapper}; + +pub struct Partition<'a, D: NgBlockDevice + 'a> { + pub device: &'a NgBlockDeviceWrapper<'a, D>, + pub lba_start: u64, + pub lba_end: u64, +} + +#[derive(Clone, Copy)] +#[repr(C)] +struct GptHeader { + signature: [u8; 8], + revision: u32, + header_size: u32, + crc32: u32, + _0: u32, + header_lba: u64, + alternate_header_lba: u64, + first_usable_lba: u64, + last_usable_lba: u64, + guid: [u8; 16], + partition_table_lba: u64, + partition_table_len: u32, + partition_table_entry_size: u32, + partition_table_crc32: u32, + _1: [u8; 420], +} + +#[derive(Clone, Copy, Zeroable, Pod)] +#[repr(C)] +struct GptEntry { + type_guid: Uuid, + part_guid: Uuid, + lba_start: u64, + lba_end: u64, + attrs: u64, +} + +const_assert_eq!(size_of::(), 512); + +impl<'a, D: NgBlockDevice + 'a> Partition<'a, D> { + fn end_byte(&self) -> u64 { + self.lba_end * self.device.block_size + } + + fn start_byte(&self) -> u64 { + self.lba_start * self.device.block_size + } +} + +impl<'a, D: NgBlockDevice + 'a> BlockDevice for Partition<'a, D> { + fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { + if pos >= self.end_byte() { + return Ok(0); + } + + let start = self.start_byte() + pos; + let end = core::cmp::min(start + buf.len() as u64, self.end_byte()); + let count = (end - start) as usize; + + self.device.read(start, &mut buf[..count]) + } + + fn write(&'static self, pos: u64, buf: &[u8]) -> Result { + if pos >= self.end_byte() { + return Ok(0); + } + + let start = self.start_byte() + pos; + let end = core::cmp::min(start + buf.len() as u64, self.end_byte()); + let count = (end - start) as usize; + + self.device.write(start, &buf[..count]) + } + + fn size(&self) -> Result { + Ok((self.lba_end - self.lba_start) * self.device.block_size) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + self.device.device_request(req) + } +} + +unsafe fn read_struct_lba(dev: &'static dyn BlockDevice, lba: u64) -> Result { + assert_eq!(size_of::(), 512); + let mut data = MaybeUninit::::uninit(); + let buffer = core::slice::from_raw_parts_mut(data.as_mut_ptr() as *mut u8, 512); + dev.read_exact(lba * 512, buffer)?; + Ok(data.assume_init()) +} + +pub(crate) fn probe_gpt( + dev: &'static NgBlockDeviceWrapper<'static, D>, +) -> Result>>, Error> { + let header = unsafe { read_struct_lba::(dev, 1) }?; + + if &header.signature != b"EFI PART" { + // Not a GPT partition table + return Ok(None); + } + + let pt_entsize = header.partition_table_entry_size as usize; + let pt_len = header.partition_table_len as usize; + let mut pt_data = PageBox::new_slice(0, pt_len * pt_entsize)?; + + assert!(size_of::() <= pt_entsize); + + dev.read_exact(header.partition_table_lba * 512, &mut pt_data)?; + + let mut partitions = vec![]; + for i in 0..pt_len { + let pt_entry_data = &pt_data[i * pt_entsize..i * pt_entsize + size_of::()]; + let pt_entry: &GptEntry = bytemuck::from_bytes(pt_entry_data); + + if pt_entry.type_guid.is_nil() { + continue; + } + + partitions.push(Partition { + device: dev, + lba_start: pt_entry.lba_start, + lba_end: pt_entry.lba_end, + }); + } + + Ok(Some(partitions)) +} diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index 6931d501..d0f1cfa3 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -4,7 +4,7 @@ use alloc::{boxed::Box, format}; use kernel_fs::devfs; use kernel_util::{cpu_index, mem::address::AsPhysicalAddress, runtime::QueueWaker}; use ygg_driver_block::{ - IoOperation, IoRequest, IoSubmissionId, NgBlockDevice, NgBlockDeviceWrapper, + probe_partitions, IoOperation, IoRequest, IoSubmissionId, NgBlockDevice, NgBlockDeviceWrapper, }; use yggdrasil_abi::error::Error; @@ -49,9 +49,11 @@ impl NvmeDrive { let node_name = format!("nvme{}n{}", controller.controller_id.get(), nsid); let blk = NgBlockDeviceWrapper::new(dev); - devfs::add_named_block_device(blk, node_name).ok(); - - // TODO probe partitions + devfs::add_named_block_device(blk, node_name.clone()).ok(); + probe_partitions(blk, move |index, partition| { + devfs::add_block_device_partition(node_name.clone(), index, partition) + }) + .ok(); Ok(dev) } diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 5d9bb7f7..6b348da4 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -55,6 +55,19 @@ pub fn add_named_block_device>( DEVFS_ROOT.get().add_child(name, node) } +pub fn add_block_device_partition>( + base_name: S, + index: usize, + partition: &'static dyn BlockDevice, +) -> Result<(), Error> { + let name = format!("{}p{}", base_name.into(), index + 1); + log::info!("Add partition: {}", name); + + let node = Node::block(partition, NodeFlags::IN_MEMORY_PROPS); + + DEVFS_ROOT.get().add_child(name, node) +} + /// Adds a character device to the devfs pub fn add_char_device(dev: &'static dyn CharDevice, kind: CharDeviceType) -> Result<(), Error> { static TTY_COUNT: AtomicUsize = AtomicUsize::new(0); From 4d9ecc24fff82aacd6c725bb8b04d442ffdbfae7 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 12 Dec 2023 18:34:21 +0200 Subject: [PATCH 124/211] block/ahci: initial support for AHCI SATA drives --- Cargo.toml | 1 + driver/block/ahci/Cargo.toml | 23 ++ driver/block/ahci/src/command.rs | 382 +++++++++++++++++++++++ driver/block/ahci/src/error.rs | 7 + driver/block/ahci/src/lib.rs | 492 ++++++++++++++++++++++++++++++ driver/block/ahci/src/regs.rs | 190 ++++++++++++ driver/bus/pci/src/lib.rs | 11 +- lib/kernel-util/src/mem/device.rs | 16 + lib/kernel-util/src/mem/mod.rs | 10 + src/arch/x86_64/mod.rs | 7 + 10 files changed, 1136 insertions(+), 3 deletions(-) create mode 100644 driver/block/ahci/Cargo.toml create mode 100644 driver/block/ahci/src/command.rs create mode 100644 driver/block/ahci/src/error.rs create mode 100644 driver/block/ahci/src/lib.rs create mode 100644 driver/block/ahci/src/regs.rs diff --git a/Cargo.toml b/Cargo.toml index 86719329..b51c072b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ device-api-macros = { path = "lib/device-api/macros" } # Drivers ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_nvme = { path = "driver/block/nvme" } +ygg_driver_ahci = { path = "driver/block/ahci" } ygg_driver_block = { path = "driver/block/core" } kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } diff --git a/driver/block/ahci/Cargo.toml b/driver/block/ahci/Cargo.toml new file mode 100644 index 00000000..34366bdb --- /dev/null +++ b/driver/block/ahci/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "ygg_driver_ahci" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } +vfs = { path = "../../../lib/vfs" } + +ygg_driver_pci = { path = "../../bus/pci" } +ygg_driver_block = { path = "../../block/core" } +kernel-fs = { path = "../../fs/kernel-fs" } + +log = "0.4.20" +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } +static_assertions = "1.1.0" +tock-registers = "0.8.1" +bytemuck = { version = "1.14.0", features = ["derive"] } +memoffset = "0.9.0" diff --git a/driver/block/ahci/src/command.rs b/driver/block/ahci/src/command.rs new file mode 100644 index 00000000..35eda27a --- /dev/null +++ b/driver/block/ahci/src/command.rs @@ -0,0 +1,382 @@ +use core::mem::{size_of, MaybeUninit}; + +use alloc::string::String; +use bytemuck::{Pod, Zeroable}; +use kernel_util::mem::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + PageBox, +}; +use static_assertions::const_assert_eq; +use tock_registers::register_structs; + +use crate::{error::AhciError, MAX_PRD_SIZE}; + +pub const COMMAND_LIST_LENGTH: usize = 32; + +const AHCI_FIS_REG_H2D_COMMAND: u8 = 1 << 7; +const AHCI_FIS_REG_H2D: u8 = 0x27; + +pub enum Assert {} +pub trait IsTrue {} + +impl IsTrue for Assert {} + +#[repr(C)] +pub struct AtaString +where + Assert<{ N % 2 == 0 }>: IsTrue, +{ + data: [u8; N], +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub struct PhysicalRegionDescriptor { + buffer_address: u64, + _0: u32, + dbc: u32, +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub struct CommandListEntry { + attr: u16, + prdtl: u16, + prdbc: u32, + ctba: u64, + _0: [u32; 4], +} + +#[derive(Clone, Copy, Zeroable)] +#[repr(C)] +pub union SentFis { + reg_h2d: RegisterHostToDeviceFis, + raw: RawFis, +} + +#[derive(Clone, Copy, Zeroable)] +#[repr(C)] +pub struct RegisterHostToDeviceFis { + pub ty: u8, + pub cmd_port: u8, + pub cmd: u8, + pub feature_low: u8, + + pub lba0: u8, + pub lba1: u8, + pub lba2: u8, + pub device: u8, + + pub lba3: u8, + pub lba4: u8, + pub lba5: u8, + pub feature_high: u8, + + pub count: u16, + pub icc: u8, + pub control: u8, + + _0: u32, +} + +#[derive(Clone, Copy, Zeroable, Pod)] +#[repr(C)] +pub struct RawFis { + pub bytes: [u8; 64], +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct CommandTable { + fis: SentFis, // 0x00..0x40 + _0: [u8; 16], // 0x40..0x50 + _1: [u8; 48], // 0x50..0x80 + prdt: [PhysicalRegionDescriptor; 248], // 0x80..0x1000 +} + +#[derive(Clone, Copy, Zeroable, Pod)] +#[repr(C)] +pub struct ReceivedFis { + _dsfis: [u8; 0x1C], // 0x00..0x1C + _0: [u8; 0x04], // 0x1C..0x20 + _psfis: [u8; 0x14], // 0x20..0x34 + _1: [u8; 0x0C], // 0x34..0x40 + _rfis: [u8; 0x14], // 0x40..0x54 + _2: [u8; 0x04], // 0x54..0x58 + _sdbfis: [u8; 0x08], // 0x58..0x60 + _ufis: [u8; 0x40], // 0x60..0xA0 + _3: [u8; 0x60], // 0xA0..0x100 +} + +register_structs! { + // Offsets in the ATA8-ACS spec are in words, so each value take from there is + // multiplied by two + pub AtaIdentifyResponse { + (0 => _0), + (20 => pub serial_number: AtaString<20>), + (40 => _1), + (54 => pub model_number: AtaString<40>), + (94 => _2), + (98 => pub capabilities: [u16; 2]), + (102 => _3), + (120 => pub logical_sector_count_28: [u8; 4]), + (124 => _4), + (138 => pub additional_features: u16), + (140 => _5), + (164 => pub command_sets: [u16; 6]), + (176 => _6), + (200 => pub logical_sector_count_qw: u64), + (208 => _7), + (212 => pub phys_logical_sector_size: u16), + (214 => _8), + (234 => pub logical_sector_size: [u16; 2]), + (238 => _9), + (460 => pub ext_logical_sector_count_qw: [u8; 8]), + (468 => _10), + (512 => @END), + } +} + +const_assert_eq!(size_of::(), 0x40); +const_assert_eq!(size_of::(), 0x1000); +const_assert_eq!(size_of::(), 32); +const_assert_eq!(size_of::(), 0x100); + +pub struct AtaIdentify { + buffer: PageBox>, + regions: [(PhysicalAddress, usize); 1], +} + +pub struct AtaReadDmaEx { + lba: u64, + sector_count: usize, + regions: [(PhysicalAddress, usize); 1], +} + +pub trait AtaCommand { + type Response; + + const COMMAND_ID: u8; + + fn lba(&self) -> u64; + fn sector_count(&self) -> usize; + fn regions(&self) -> &[(PhysicalAddress, usize)]; + unsafe fn into_response(self) -> Self::Response; +} + +impl AtaIdentify { + pub fn create() -> Result { + PageBox::new_uninit() + .map(Self::with_data) + .map_err(AhciError::MemoryError) + } + + pub fn with_data(buffer: PageBox>) -> Self { + Self { + regions: [( + unsafe { buffer.as_physical_address() }, + size_of::(), + )], + buffer, + } + } +} + +impl AtaReadDmaEx { + pub fn new(lba: u64, buffer: &PageBox<[MaybeUninit]>) -> Self { + assert_eq!(buffer.len() % 512, 0); + assert_ne!(buffer.len(), 0); + + let sector_count = buffer.len() / 512; + Self { + lba, + sector_count, + regions: [(unsafe { buffer.as_physical_address() }, buffer.len())], + } + } +} + +impl AtaCommand for AtaIdentify { + type Response = PageBox; + + const COMMAND_ID: u8 = 0xEC; + + fn lba(&self) -> u64 { + 0 + } + + fn sector_count(&self) -> usize { + 0 + } + + fn regions(&self) -> &[(PhysicalAddress, usize)] { + &self.regions + } + + unsafe fn into_response(self) -> Self::Response { + self.buffer.assume_init() + } +} + +impl AtaCommand for AtaReadDmaEx { + type Response = (); + + const COMMAND_ID: u8 = 0x25; + + fn lba(&self) -> u64 { + self.lba + } + + fn sector_count(&self) -> usize { + self.sector_count + } + + fn regions(&self) -> &[(PhysicalAddress, usize)] { + &self.regions + } + + unsafe fn into_response(self) -> Self::Response { + () + } +} + +impl CommandTable { + pub fn setup_command(&mut self, command: &C) -> Result<(), AhciError> { + let lba = command.lba(); + assert_eq!(lba & !0xFFFFFFFFFF, 0); + let count = command.sector_count().try_into().unwrap(); + + if C::COMMAND_ID == AtaIdentify::COMMAND_ID { + self.fis = SentFis { + reg_h2d: RegisterHostToDeviceFis { + ty: AHCI_FIS_REG_H2D, + cmd_port: AHCI_FIS_REG_H2D_COMMAND, + cmd: C::COMMAND_ID, + + ..RegisterHostToDeviceFis::zeroed() + }, + }; + } else { + self.fis = SentFis { + reg_h2d: RegisterHostToDeviceFis { + ty: AHCI_FIS_REG_H2D, + cmd_port: AHCI_FIS_REG_H2D_COMMAND, + cmd: C::COMMAND_ID, + device: 1 << 6, // LBA mode + lba0: lba as u8, + lba1: (lba >> 8) as u8, + lba2: (lba >> 16) as u8, + lba3: (lba >> 24) as u8, + lba4: (lba >> 32) as u8, + lba5: (lba >> 40) as u8, + count, + + ..RegisterHostToDeviceFis::zeroed() + }, + }; + } + + let regions = command.regions(); + for (i, &(base, size)) in regions.iter().enumerate() { + let last = i == regions.len() - 1; + self.prdt[i] = PhysicalRegionDescriptor::new(base, size, last)?; + } + + Ok(()) + } +} + +impl CommandListEntry { + pub fn new(command_table_entry: PhysicalAddress, prd_count: usize) -> Result { + if prd_count > 0xFFFF { + todo!() + } + Ok(Self { + // attr = FIS size in dwords + attr: (size_of::() / size_of::()) as _, + prdtl: prd_count as _, + prdbc: 0, + ctba: command_table_entry.into_raw(), + _0: [0; 4], + }) + } +} + +unsafe impl Zeroable for CommandTable { + fn zeroed() -> Self { + Self { + fis: SentFis::zeroed(), + _0: [0; 16], + _1: [0; 48], + prdt: [PhysicalRegionDescriptor::zeroed(); 248], + } + } +} + +impl PhysicalRegionDescriptor { + pub fn new( + address: PhysicalAddress, + byte_count: usize, + is_last: bool, + ) -> Result { + if byte_count >= MAX_PRD_SIZE { + return Err(AhciError::RegionTooLarge); + } + + let dbc_mask = (is_last as u32) << 31; + Ok(Self { + buffer_address: address.into_raw(), + _0: 0, + dbc: ((byte_count as u32 - 1) << 1) | 1 | dbc_mask, + }) + } +} + +impl AtaIdentifyResponse { + pub fn logical_sector_count(&self) -> u64 { + // If logical_sector_count_28 == 0x0FFFFFFF, and logical_sector_count_qw >= 0x0FFFFFFF, + // then ACCEESSIBLE CAPACITY (?) field contains the total number of user addressable + // LBAs (see 4.1) + // bit 3 in additional_features -> logical_sector_count_qw: + // 0 -> max value = 0xFFFFFFFFFFFF (48) + // 1 -> max value = 0xFFFFFFFF (32) + // If bit 3 in additional_features is set, ext_logical_sector_count_qw contains + // the maximum addressable LBA count. Max value = 0xFFFFFFFFFFFF (48) + + if self.command_sets[1] & (1 << 10) != 0 { + // 48-bit supported + if self.additional_features & (1 << 3) != 0 { + // Use ext_logical_sector_count_qw + todo!() + } else { + // Use logical_sector_count_qw + self.logical_sector_count_qw + } + } else { + todo!() + } + } +} + +impl AtaString +where + Assert<{ N % 2 == 0 }>: IsTrue, +{ + pub fn to_string(&self) -> String { + let mut buf = [0; N]; + + for i in (0..N).step_by(2) { + buf[i] = self.data[i + 1]; + buf[i + 1] = self.data[i]; + } + let mut len = 0; + for i in (0..N).rev() { + if buf[i] != b' ' { + len = i + 1; + break; + } + } + + String::from(core::str::from_utf8(&buf[..len]).unwrap()) + } +} diff --git a/driver/block/ahci/src/error.rs b/driver/block/ahci/src/error.rs new file mode 100644 index 00000000..a6ea25eb --- /dev/null +++ b/driver/block/ahci/src/error.rs @@ -0,0 +1,7 @@ +use yggdrasil_abi::error::Error; + +#[derive(Debug)] +pub enum AhciError { + MemoryError(Error), + RegionTooLarge, +} diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs new file mode 100644 index 00000000..6f9136fe --- /dev/null +++ b/driver/block/ahci/src/lib.rs @@ -0,0 +1,492 @@ +#![feature(generic_const_exprs, inline_const)] +#![no_std] + +extern crate alloc; + +use core::{ + mem::{size_of, MaybeUninit}, + pin::Pin, + sync::atomic::{AtomicU32, Ordering}, + task::{Context, Poll}, + time::Duration, +}; + +use alloc::{boxed::Box, collections::BTreeMap, format, string::String, vec, vec::Vec}; +use bytemuck::Zeroable; +use device_api::Device; +use error::AhciError; +use futures_util::{task::AtomicWaker, Future}; +use kernel_fs::devfs; +use kernel_util::{ + block, + mem::{ + address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, + device::DeviceMemoryIo, + PageBox, + }, + runtime::{self, QueueWaker}, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; +use regs::{PortRegs, Regs}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use ygg_driver_block::{cache::BlockCache, BlockDevice}; +use ygg_driver_pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo}; +use yggdrasil_abi::{error::Error, io::DeviceRequest}; + +use crate::{ + command::{ + AtaCommand, AtaIdentify, AtaIdentifyResponse, AtaReadDmaEx, CommandListEntry, CommandTable, + ReceivedFis, COMMAND_LIST_LENGTH, + }, + regs::{CommandState, Version, CAP, CMD, GHC, IS, SSTS, TFD}, +}; + +mod command; +mod error; +mod regs; + +const MAX_PORTS: usize = 30; // 32 actually, but I don't want the struct to take another page, lol +const MAX_PRD_SIZE: usize = 8192; + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum PortType { + Sata, +} + +pub struct AhciController { + regs: IrqSafeSpinlock>, + ports: OneTimeInit>, + + version: Version, + max_port_count: usize, + ahci_only: bool, + has_64_bit: bool, +} + +struct PortInner { + regs: DeviceMemoryIo<'static, PortRegs>, + + _received_fis: PageBox, + command_list: PageBox<[CommandListEntry]>, +} + +struct PortInfo { + model: String, + serial: String, + lba_count: u64, +} + +struct AhciPort { + inner: IrqSafeSpinlock, + ahci: &'static AhciController, + ty: PortType, + index: usize, + info: OneTimeInit, + cache: IrqSafeSpinlock>, + + command_allocation: IrqSafeSpinlock, + // One command index can only be waited for by one task, so this approach is usable + command_completion: [AtomicWaker; COMMAND_LIST_LENGTH], + command_available: QueueWaker, +} + +impl PortInner { + fn submit_command( + &mut self, + index: usize, + command: &C, + ) -> Result<(), AhciError> { + let list_entry = &mut self.command_list[index]; + let mut table_entry = + PageBox::new(CommandTable::zeroed()).map_err(AhciError::MemoryError)?; + + table_entry.setup_command(command)?; + *list_entry = CommandListEntry::new( + unsafe { table_entry.as_physical_address() }, + command.regions().len(), + )?; + + // Sync before send + unsafe { + core::arch::asm!("wbinvd"); + } + + while self.regs.TFD.matches_any(TFD::BSY::SET + TFD::DRQ::SET) { + core::hint::spin_loop(); + } + + let ci = self.regs.CI.get(); + assert_eq!(ci & (1 << index), 0); + self.regs.CI.set(ci | (1 << index)); + + Ok(()) + } +} + +impl AhciPort { + pub async fn create( + regs: DeviceMemoryIo<'static, PortRegs>, + ahci: &'static AhciController, + index: usize, + ) -> Result<&'static Self, AhciError> { + log::debug!("Initialize port {}", index); + regs.stop()?; + + if !ahci.has_64_bit { + todo!("Handle controllers incapable of 64 bit"); + } + + let received_fis = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; + let command_list = PageBox::new_slice(CommandListEntry::zeroed(), COMMAND_LIST_LENGTH) + .map_err(AhciError::MemoryError)?; + + regs.set_received_fis_address_64(unsafe { received_fis.as_physical_address() }); + regs.set_command_list_address_64(unsafe { command_list.as_physical_address() }); + + regs.start()?; + + let inner = PortInner { + regs, + command_list, + received_fis, + }; + let command_completion = [const { AtomicWaker::new() }; 32]; + let command_available = QueueWaker::new(); + let command_allocation = IrqSafeSpinlock::new(0); + + let port = Box::leak(Box::new(Self { + inner: IrqSafeSpinlock::new(inner), + ty: PortType::Sata, + info: OneTimeInit::new(), + cache: IrqSafeSpinlock::new(BlockCache::new(512)), + ahci, + index, + + command_completion, + command_allocation, + command_available, + })); + + runtime::spawn(port.poll_task()).unwrap(); + + let identify = port.perform_command(AtaIdentify::create()?).await?; + let model = identify.model_number.to_string(); + let serial = identify.serial_number.to_string(); + let lba_count = identify.logical_sector_count(); + + // TODO can sector size be different from 512 in ATA? + // should logical sector size be accounted for? + // TODO test for ReadDmaEx capability (?) + + port.info.init(PortInfo { + model, + serial, + lba_count, + }); + + Ok(port) + } + + async fn perform_command(&self, command: C) -> Result { + let slot = self.allocate_command().await?; + log::trace!( + "Submit command on port {}, cmd index = {}", + self.index, + slot + ); + self.inner.lock().submit_command(slot, &command)?; + self.wait_for_completion(slot).await?; + self.free_command(slot); + Ok(unsafe { command.into_response() }) + } + + async fn read_block( + &self, + lba: u64, + block: &mut PageBox<[MaybeUninit]>, + ) -> Result<(), AhciError> { + log::debug!("read_block port={}, lba={}", self.index, lba); + self.perform_command(AtaReadDmaEx::new(lba, block)).await + } + + fn allocate_command<'r>(&'r self) -> impl Future> + 'r { + struct F<'f> { + waker: &'f QueueWaker, + state: &'f IrqSafeSpinlock, + } + + impl<'f> Future for F<'f> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.waker.register(cx.waker()); + + let mut state = self.state.lock(); + + if *state != u32::MAX { + self.waker.remove(cx.waker()); + for i in 0..32 { + if *state & (1 << i) == 0 { + *state |= 1 << i; + return Poll::Ready(Ok(i)); + } + } + + panic!("Unreachable"); + } else { + Poll::Pending + } + } + } + + log::debug!("allocate_command"); + + let waker = &self.command_available; + let state = &self.command_allocation; + + F { waker, state } + } + + fn wait_for_completion<'r>( + &'r self, + index: usize, + ) -> impl Future> + 'r { + struct F<'f> { + waker: &'f AtomicWaker, + inner: &'f IrqSafeSpinlock, + index: usize, + } + + impl<'f> Future for F<'f> { + type Output = Result<(), AhciError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let inner = self.inner.lock(); + + if let CommandState::Ready(status) = inner.regs.command_status(self.index) { + return Poll::Ready(status.into()); + } + + self.waker.register(cx.waker()); + + match inner.regs.command_status(self.index) { + CommandState::Pending => Poll::Pending, + CommandState::Ready(status) => Poll::Ready(status.into()), + } + } + } + + log::debug!("wait_for_completion {}", index); + + let waker = &self.command_completion[index]; + let inner = &self.inner; + + F { + waker, + inner, + index, + } + } + + fn free_command(&self, index: usize) { + { + let mut alloc = self.command_allocation.lock(); + assert_ne!(*alloc & (1 << index), 0); + *alloc &= !(1 << index); + } + self.command_available.wake_one(); + } + + // TODO use MSI + async fn poll_task(&self) { + loop { + for i in 0..32 { + self.command_completion[i].wake(); + } + + runtime::sleep(Duration::from_millis(50)).await; + } + } +} + +impl AhciController { + async fn late_init(&'static self) -> Result<(), AhciError> { + let regs = self.regs.lock(); + + regs.GHC.modify(GHC::HR::SET); + + while regs.GHC.matches_all(GHC::HR::SET) { + core::hint::spin_loop(); + } + + if !self.ahci_only { + regs.GHC.modify(GHC::AE::SET); + } + + // TODO configure interrupts + + let pi = regs.PI.get(); + + let mut ports = vec![]; + + for i in 0..self.max_port_count { + if pi & (1 << i) == 0 { + continue; + } + + let port = ®s.PORTS[i]; + + if !port.SSTS.matches_all(SSTS::DET::Online + SSTS::IPM::Active) { + continue; + } + + port.start()?; + + // TODO wait here + + let sig = port.SIG.get(); + if sig != PortRegs::SIG_SATA { + log::warn!("Skipping unknown port {} with signature {:#x}", i, sig); + continue; + } + + let port = unsafe { regs.extract(|regs| ®s.PORTS[i]) }; + + let port = match AhciPort::create(port, self, i).await { + Ok(port) => port, + Err(error) => { + log::warn!("Port {} init error: {:?}", i, error); + continue; + } + }; + + ports.push(port); + } + + // Dump info about the drives + for (i, &port) in ports.iter().enumerate() { + let info = port.info.get(); + log::info!( + "Port {}: model={:?}, serial={:?}, lba_count={}", + i, + info.model, + info.serial, + info.lba_count + ); + } + + { + let mut lock = SATA_DRIVES.lock(); + for &port in ports.iter() { + let n = lock.len(); + if n >= 25 { + todo!("Too many drives, ran out of letters"); + } + let n = n as u8; + lock.push(port); + + let name = format!("sd{}", (n + b'a') as char); + devfs::add_named_block_device(port, name).unwrap(); + } + } + + self.ports.init(ports); + + log::debug!("All ports initialized"); + + Ok(()) + } +} + +impl BlockDevice for AhciPort { + fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { + let info = self.info.try_get().ok_or(Error::PermissionDenied)?; + + let mut cache = self.cache.lock(); + let mut rem = buf.len(); + let mut off = 0; + + while rem != 0 { + let lba = pos / 512; + + if lba >= info.lba_count { + break; + } + + let block_offset = (pos % 512) as usize; + let count = core::cmp::min(512 - block_offset, rem); + + let block = cache.get_or_fetch_with(lba, |block| { + block! { + self.read_block(lba, block).await + }? + .map_err(|_| Error::InvalidOperation) + })?; + + buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); + + rem -= count; + off += count; + pos += count as u64; + } + + Ok(off) + } + + fn write(&'static self, pos: u64, buf: &[u8]) -> Result { + todo!() + } + + fn size(&self) -> Result { + let info = self.info.try_get().ok_or(Error::PermissionDenied)?; + Ok(info.lba_count * 512) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + todo!() + } +} + +impl Device for AhciController { + unsafe fn init(&'static self) -> Result<(), Error> { + runtime::spawn(self.late_init())?; + Ok(()) + } + + fn display_name(&self) -> &'static str { + "AHCI SATA Controller" + } +} + +static SATA_DRIVES: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); + +pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { + let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?; + let PciBaseAddress::Memory(bar5) = bar5 else { + return Err(Error::InvalidOperation); + }; + + // Map the registers + let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar5)) }?; + let version = Version::try_from(regs.VS.get())?; + let ahci_only = regs.CAP.matches_all(CAP::SAM::SET); + let max_port_count = regs.CAP.read(CAP::NP) as usize; + let has_64_bit = regs.CAP.matches_all(CAP::S64A::SET); + + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); + cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); + cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; + info.config_space.set_command(cmd.bits()); + + log::info!("AHCI SATA Controller {:?}", version); + + Ok(Box::leak(Box::new(AhciController { + regs: IrqSafeSpinlock::new(regs), + ports: OneTimeInit::new(), + version, + max_port_count, + ahci_only, + has_64_bit, + }))) +} diff --git a/driver/block/ahci/src/regs.rs b/driver/block/ahci/src/regs.rs new file mode 100644 index 00000000..09e4ebfd --- /dev/null +++ b/driver/block/ahci/src/regs.rs @@ -0,0 +1,190 @@ +use kernel_util::mem::address::{IntoRaw, PhysicalAddress}; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite}, +}; +use yggdrasil_abi::error::Error; + +use crate::error::AhciError; + +register_bitfields! { + u32, + pub CAP [ + NP OFFSET(0) NUMBITS(5) [], + NCS OFFSET(8) NUMBITS(5) [], + SAM OFFSET(18) NUMBITS(1) [], + S64A OFFSET(31) NUMBITS(1) [], + ], + pub GHC [ + HR OFFSET(0) NUMBITS(1) [], + IE OFFSET(1) NUMBITS(1) [], + AE OFFSET(31) NUMBITS(1) [], + ], + + pub IS [ + TFES OFFSET(30) NUMBITS(1) [], + DHRS OFFSET(0) NUMBITS(1) [], + ], + pub IE [ + DHRS OFFSET(0) NUMBITS(1) [], + ], + pub CMD [ + CR OFFSET(15) NUMBITS(1) [], + FR OFFSET(14) NUMBITS(1) [], + CCS OFFSET(8) NUMBITS(5) [], + FRE OFFSET(4) NUMBITS(1) [], + POD OFFSET(2) NUMBITS(1) [], + ST OFFSET(0) NUMBITS(1) [], + ], + pub SSTS [ + IPM OFFSET(8) NUMBITS(4) [ + NotPresent = 0, + Active = 1, + ], + DET OFFSET(0) NUMBITS(4) [ + NotPresent = 0, + Online = 3, + ], + ], + pub TFD [ + BSY OFFSET(7) NUMBITS(1) [], + DRQ OFFSET(3) NUMBITS(1) [], + ERR OFFSET(0) NUMBITS(1) [], + ] +} + +register_structs! { + #[allow(non_snake_case)] + pub Regs { + (0x0000 => pub CAP: ReadOnly), + (0x0004 => pub GHC: ReadWrite), + (0x0008 => pub IS: ReadOnly), + (0x000C => pub PI: ReadOnly), + (0x0010 => pub VS: ReadOnly), + (0x0014 => _0), + (0x0100 => pub PORTS: [PortRegs; 30]), + (0x1000 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + pub PortRegs { + (0x00 => pub CLB: ReadWrite), + (0x04 => pub CLBU: ReadWrite), + (0x08 => pub FB: ReadWrite), + (0x0C => pub FBU: ReadWrite), + (0x10 => pub IS: ReadOnly), + (0x14 => pub IE: ReadWrite), + (0x18 => pub CMD: ReadWrite), + (0x1C => _0), + (0x20 => pub TFD: ReadWrite), + (0x24 => pub SIG: ReadOnly), + (0x28 => pub SSTS: ReadOnly), + (0x2C => pub SCTL: ReadOnly), + (0x30 => pub SERR: ReadOnly), + (0x34 => pub SACT: ReadOnly), + (0x38 => pub CI: ReadWrite), + (0x3C => pub SNTF: ReadOnly), + (0x40 => _1), + (0x80 => @END), + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum Version { + V0_95, + V1_0, + V1_1, + V1_2, + V1_3, + V1_3_1, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum CommandStatus { + Success, + TaskFileError, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum CommandState { + Pending, + Ready(CommandStatus), +} + +impl PortRegs { + pub const SIG_SATA: u32 = 0x101; + + // NOTE: usually doesn't take long, so not async, I guess + pub fn stop(&self) -> Result<(), AhciError> { + self.CMD.modify(CMD::ST::CLEAR + CMD::FRE::CLEAR); + + // TODO timeout here + while self.CMD.matches_any(CMD::FR::SET + CMD::CR::SET) { + core::hint::spin_loop(); + } + + Ok(()) + } + + pub fn start(&self) -> Result<(), AhciError> { + while self.CMD.matches_all(CMD::CR::SET) { + core::hint::spin_loop(); + } + + self.CMD.modify(CMD::ST::SET + CMD::FRE::SET); + + Ok(()) + } + + pub fn set_received_fis_address_64(&self, address: PhysicalAddress) { + let address: u64 = address.into_raw(); + self.FB.set(address as u32); + self.FBU.set((address >> 32) as u32); + } + + pub fn set_command_list_address_64(&self, address: PhysicalAddress) { + let address: u64 = address.into_raw(); + self.CLB.set(address as u32); + self.CLBU.set((address >> 32) as u32); + } + + pub fn command_status(&self, index: usize) -> CommandState { + if self.CI.get() & (1 << index) == 0 { + CommandState::Ready(if self.IS.matches_all(IS::TFES::SET) { + CommandStatus::TaskFileError + } else { + CommandStatus::Success + }) + } else { + CommandState::Pending + } + } +} + +impl TryFrom for Version { + type Error = Error; + + fn try_from(value: u32) -> Result { + match value { + 0x00000905 => Ok(Self::V0_95), + 0x00010000 => Ok(Self::V1_0), + 0x00010100 => Ok(Self::V1_1), + 0x00010200 => Ok(Self::V1_2), + 0x00010300 => Ok(Self::V1_3), + 0x00010301 => Ok(Self::V1_3_1), + _ => Err(Error::InvalidArgument), + } + } +} + +impl Into> for CommandStatus { + fn into(self) -> Result<(), AhciError> { + match self { + Self::Success => Ok(()), + Self::TaskFileError => Err(todo!()), + } + } +} diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index b2ae181e..ea782c78 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -116,7 +116,7 @@ pub struct PciDriver { /// Used to store PCI bus devices which were enumerated by the kernel pub struct PciBusDevice { info: PciDeviceInfo, - driver: Option>, + driver: Option<&'static dyn Device>, } /// Represents a single PCIe bus segment @@ -205,7 +205,9 @@ impl PciBusManager { /// Walks the bus device list and calls init/init_irq functions on any devices with associated /// drivers pub fn setup_bus_devices() -> Result<(), Error> { + log::info!("Setting up bus devices"); Self::walk_bus_devices(|device| { + log::info!("Set up {}", device.info.address); setup_bus_device(device)?; Ok(true) }) @@ -322,11 +324,14 @@ fn setup_bus_device(device: &mut PciBusDevice) -> Result<(), Error> { { // TODO add the device to the bus log::debug!(" -> {:?}", driver.name); - let device = (driver.probe)(&device.info)?; + let instance = (driver.probe)(&device.info)?; unsafe { - device.init()?; + instance.init()?; } + + device.driver.replace(instance); + break; } else { log::debug!(" -> No driver"); } diff --git a/lib/kernel-util/src/mem/device.rs b/lib/kernel-util/src/mem/device.rs index 44fd7408..822d33ba 100644 --- a/lib/kernel-util/src/mem/device.rs +++ b/lib/kernel-util/src/mem/device.rs @@ -154,6 +154,20 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { } } +impl<'a, T: ?Sized> DeviceMemoryIo<'a, T> { + pub unsafe fn extract<'b, U: ?Sized + 'b, F: FnOnce(&'a T) -> *const U>( + &'a self, + f: F, + ) -> DeviceMemoryIo<'b, U> { + let value = f(self.value); + + DeviceMemoryIo { + inner: self.inner.clone(), + value: &*value, + } + } +} + impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> { type Target = T; @@ -168,6 +182,8 @@ impl AsPhysicalAddress for DeviceMemoryIo<'_, T> { } } +unsafe impl Send for DeviceMemoryIo<'_, T> {} + impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> { /// Maps a physical address as device memory to a slice `[T; len]` /// diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 39d40fd9..7392a516 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -74,6 +74,16 @@ impl PageBox { Ok(result) } + pub fn new_slice_with T>(f: F, count: usize) -> Result, Error> { + let mut value = Self::new_uninit_slice(count)?; + + for i in 0..count { + value[i].write(f(i)); + } + + Ok(unsafe { value.assume_init_slice() }) + } + pub fn new_uninit() -> Result>, Error> { let (base, page_count) = PageBox::>::alloc()?; let value = base.virtualize_raw() as *mut MaybeUninit; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 6e315029..6e4613a5 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -391,6 +391,13 @@ impl X86_64 { Some(0x02), ygg_driver_nvme::probe, ); + ygg_driver_pci::register_class_driver( + "AHCI SATA Controller", + 0x01, + Some(0x06), + Some(0x01), + ygg_driver_ahci::probe, + ); match self.boot_data.get() { &BootData::YBoot(data) => { From dc864cc7a6773d182071aee0439daa30c11c3d8e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 12 Dec 2023 18:55:46 +0200 Subject: [PATCH 125/211] block/ahci: split AhciPort into its own file --- driver/block/ahci/src/error.rs | 1 + driver/block/ahci/src/lib.rs | 347 ++------------------------------- driver/block/ahci/src/port.rs | 337 ++++++++++++++++++++++++++++++++ driver/block/ahci/src/regs.rs | 2 +- driver/bus/pci/src/lib.rs | 2 +- 5 files changed, 357 insertions(+), 332 deletions(-) create mode 100644 driver/block/ahci/src/port.rs diff --git a/driver/block/ahci/src/error.rs b/driver/block/ahci/src/error.rs index a6ea25eb..d9325ca6 100644 --- a/driver/block/ahci/src/error.rs +++ b/driver/block/ahci/src/error.rs @@ -4,4 +4,5 @@ use yggdrasil_abi::error::Error; pub enum AhciError { MemoryError(Error), RegionTooLarge, + TaskFileError, } diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 6f9136fe..6aff475b 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -1,58 +1,39 @@ #![feature(generic_const_exprs, inline_const)] +#![allow(incomplete_features)] #![no_std] extern crate alloc; -use core::{ - mem::{size_of, MaybeUninit}, - pin::Pin, - sync::atomic::{AtomicU32, Ordering}, - task::{Context, Poll}, - time::Duration, -}; - -use alloc::{boxed::Box, collections::BTreeMap, format, string::String, vec, vec::Vec}; -use bytemuck::Zeroable; +use alloc::{boxed::Box, format, vec, vec::Vec}; use device_api::Device; use error::AhciError; -use futures_util::{task::AtomicWaker, Future}; use kernel_fs::devfs; use kernel_util::{ - block, mem::{ - address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, + address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIo, - PageBox, }, - runtime::{self, QueueWaker}, + runtime, sync::IrqSafeSpinlock, util::OneTimeInit, }; +use port::AhciPort; use regs::{PortRegs, Regs}; -use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use ygg_driver_block::{cache::BlockCache, BlockDevice}; +use tock_registers::interfaces::{ReadWriteable, Readable}; use ygg_driver_pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo}; -use yggdrasil_abi::{error::Error, io::DeviceRequest}; +use yggdrasil_abi::error::Error; -use crate::{ - command::{ - AtaCommand, AtaIdentify, AtaIdentifyResponse, AtaReadDmaEx, CommandListEntry, CommandTable, - ReceivedFis, COMMAND_LIST_LENGTH, - }, - regs::{CommandState, Version, CAP, CMD, GHC, IS, SSTS, TFD}, -}; +use crate::regs::{Version, CAP, GHC, SSTS}; mod command; mod error; +mod port; mod regs; -const MAX_PORTS: usize = 30; // 32 actually, but I don't want the struct to take another page, lol const MAX_PRD_SIZE: usize = 8192; - -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum PortType { - Sata, -} +const MAX_COMMANDS: usize = u32::BITS as usize; +const SECTOR_SIZE: usize = 512; +const MAX_DRIVES: usize = (b'z' - b'a') as usize; pub struct AhciController { regs: IrqSafeSpinlock>, @@ -64,254 +45,10 @@ pub struct AhciController { has_64_bit: bool, } -struct PortInner { - regs: DeviceMemoryIo<'static, PortRegs>, - - _received_fis: PageBox, - command_list: PageBox<[CommandListEntry]>, -} - -struct PortInfo { - model: String, - serial: String, - lba_count: u64, -} - -struct AhciPort { - inner: IrqSafeSpinlock, - ahci: &'static AhciController, - ty: PortType, - index: usize, - info: OneTimeInit, - cache: IrqSafeSpinlock>, - - command_allocation: IrqSafeSpinlock, - // One command index can only be waited for by one task, so this approach is usable - command_completion: [AtomicWaker; COMMAND_LIST_LENGTH], - command_available: QueueWaker, -} - -impl PortInner { - fn submit_command( - &mut self, - index: usize, - command: &C, - ) -> Result<(), AhciError> { - let list_entry = &mut self.command_list[index]; - let mut table_entry = - PageBox::new(CommandTable::zeroed()).map_err(AhciError::MemoryError)?; - - table_entry.setup_command(command)?; - *list_entry = CommandListEntry::new( - unsafe { table_entry.as_physical_address() }, - command.regions().len(), - )?; - - // Sync before send - unsafe { - core::arch::asm!("wbinvd"); - } - - while self.regs.TFD.matches_any(TFD::BSY::SET + TFD::DRQ::SET) { - core::hint::spin_loop(); - } - - let ci = self.regs.CI.get(); - assert_eq!(ci & (1 << index), 0); - self.regs.CI.set(ci | (1 << index)); - - Ok(()) - } -} - -impl AhciPort { - pub async fn create( - regs: DeviceMemoryIo<'static, PortRegs>, - ahci: &'static AhciController, - index: usize, - ) -> Result<&'static Self, AhciError> { - log::debug!("Initialize port {}", index); - regs.stop()?; - - if !ahci.has_64_bit { - todo!("Handle controllers incapable of 64 bit"); - } - - let received_fis = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; - let command_list = PageBox::new_slice(CommandListEntry::zeroed(), COMMAND_LIST_LENGTH) - .map_err(AhciError::MemoryError)?; - - regs.set_received_fis_address_64(unsafe { received_fis.as_physical_address() }); - regs.set_command_list_address_64(unsafe { command_list.as_physical_address() }); - - regs.start()?; - - let inner = PortInner { - regs, - command_list, - received_fis, - }; - let command_completion = [const { AtomicWaker::new() }; 32]; - let command_available = QueueWaker::new(); - let command_allocation = IrqSafeSpinlock::new(0); - - let port = Box::leak(Box::new(Self { - inner: IrqSafeSpinlock::new(inner), - ty: PortType::Sata, - info: OneTimeInit::new(), - cache: IrqSafeSpinlock::new(BlockCache::new(512)), - ahci, - index, - - command_completion, - command_allocation, - command_available, - })); - - runtime::spawn(port.poll_task()).unwrap(); - - let identify = port.perform_command(AtaIdentify::create()?).await?; - let model = identify.model_number.to_string(); - let serial = identify.serial_number.to_string(); - let lba_count = identify.logical_sector_count(); - - // TODO can sector size be different from 512 in ATA? - // should logical sector size be accounted for? - // TODO test for ReadDmaEx capability (?) - - port.info.init(PortInfo { - model, - serial, - lba_count, - }); - - Ok(port) - } - - async fn perform_command(&self, command: C) -> Result { - let slot = self.allocate_command().await?; - log::trace!( - "Submit command on port {}, cmd index = {}", - self.index, - slot - ); - self.inner.lock().submit_command(slot, &command)?; - self.wait_for_completion(slot).await?; - self.free_command(slot); - Ok(unsafe { command.into_response() }) - } - - async fn read_block( - &self, - lba: u64, - block: &mut PageBox<[MaybeUninit]>, - ) -> Result<(), AhciError> { - log::debug!("read_block port={}, lba={}", self.index, lba); - self.perform_command(AtaReadDmaEx::new(lba, block)).await - } - - fn allocate_command<'r>(&'r self) -> impl Future> + 'r { - struct F<'f> { - waker: &'f QueueWaker, - state: &'f IrqSafeSpinlock, - } - - impl<'f> Future for F<'f> { - type Output = Result; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.waker.register(cx.waker()); - - let mut state = self.state.lock(); - - if *state != u32::MAX { - self.waker.remove(cx.waker()); - for i in 0..32 { - if *state & (1 << i) == 0 { - *state |= 1 << i; - return Poll::Ready(Ok(i)); - } - } - - panic!("Unreachable"); - } else { - Poll::Pending - } - } - } - - log::debug!("allocate_command"); - - let waker = &self.command_available; - let state = &self.command_allocation; - - F { waker, state } - } - - fn wait_for_completion<'r>( - &'r self, - index: usize, - ) -> impl Future> + 'r { - struct F<'f> { - waker: &'f AtomicWaker, - inner: &'f IrqSafeSpinlock, - index: usize, - } - - impl<'f> Future for F<'f> { - type Output = Result<(), AhciError>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let inner = self.inner.lock(); - - if let CommandState::Ready(status) = inner.regs.command_status(self.index) { - return Poll::Ready(status.into()); - } - - self.waker.register(cx.waker()); - - match inner.regs.command_status(self.index) { - CommandState::Pending => Poll::Pending, - CommandState::Ready(status) => Poll::Ready(status.into()), - } - } - } - - log::debug!("wait_for_completion {}", index); - - let waker = &self.command_completion[index]; - let inner = &self.inner; - - F { - waker, - inner, - index, - } - } - - fn free_command(&self, index: usize) { - { - let mut alloc = self.command_allocation.lock(); - assert_ne!(*alloc & (1 << index), 0); - *alloc &= !(1 << index); - } - self.command_available.wake_one(); - } - - // TODO use MSI - async fn poll_task(&self) { - loop { - for i in 0..32 { - self.command_completion[i].wake(); - } - - runtime::sleep(Duration::from_millis(50)).await; - } - } -} - impl AhciController { async fn late_init(&'static self) -> Result<(), AhciError> { + log::info!("Initializing AHCI SATA Controller {:?}", self.version); + let regs = self.regs.lock(); regs.GHC.modify(GHC::HR::SET); @@ -366,7 +103,7 @@ impl AhciController { // Dump info about the drives for (i, &port) in ports.iter().enumerate() { - let info = port.info.get(); + let info = port.info().unwrap(); log::info!( "Port {}: model={:?}, serial={:?}, lba_count={}", i, @@ -380,7 +117,7 @@ impl AhciController { let mut lock = SATA_DRIVES.lock(); for &port in ports.iter() { let n = lock.len(); - if n >= 25 { + if n >= MAX_DRIVES { todo!("Too many drives, ran out of letters"); } let n = n as u8; @@ -399,57 +136,9 @@ impl AhciController { } } -impl BlockDevice for AhciPort { - fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { - let info = self.info.try_get().ok_or(Error::PermissionDenied)?; - - let mut cache = self.cache.lock(); - let mut rem = buf.len(); - let mut off = 0; - - while rem != 0 { - let lba = pos / 512; - - if lba >= info.lba_count { - break; - } - - let block_offset = (pos % 512) as usize; - let count = core::cmp::min(512 - block_offset, rem); - - let block = cache.get_or_fetch_with(lba, |block| { - block! { - self.read_block(lba, block).await - }? - .map_err(|_| Error::InvalidOperation) - })?; - - buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); - - rem -= count; - off += count; - pos += count as u64; - } - - Ok(off) - } - - fn write(&'static self, pos: u64, buf: &[u8]) -> Result { - todo!() - } - - fn size(&self) -> Result { - let info = self.info.try_get().ok_or(Error::PermissionDenied)?; - Ok(info.lba_count * 512) - } - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - todo!() - } -} - impl Device for AhciController { unsafe fn init(&'static self) -> Result<(), Error> { + // Do the init in background runtime::spawn(self.late_init())?; Ok(()) } @@ -479,8 +168,6 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; info.config_space.set_command(cmd.bits()); - log::info!("AHCI SATA Controller {:?}", version); - Ok(Box::leak(Box::new(AhciController { regs: IrqSafeSpinlock::new(regs), ports: OneTimeInit::new(), diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs new file mode 100644 index 00000000..059367cd --- /dev/null +++ b/driver/block/ahci/src/port.rs @@ -0,0 +1,337 @@ +use core::{ + mem::MaybeUninit, + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use alloc::{boxed::Box, string::String}; +use bytemuck::Zeroable; +use futures_util::{task::AtomicWaker, Future}; +use kernel_util::{ + block, + mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, + runtime::{self, QueueWaker}, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; +use tock_registers::interfaces::{Readable, Writeable}; +use ygg_driver_block::{cache::BlockCache, BlockDevice}; +use yggdrasil_abi::{error::Error, io::DeviceRequest}; + +use crate::{ + command::{ + AtaCommand, AtaIdentify, AtaReadDmaEx, CommandListEntry, CommandTable, ReceivedFis, + COMMAND_LIST_LENGTH, + }, + error::AhciError, + regs::{CommandState, PortRegs, TFD}, + AhciController, MAX_COMMANDS, SECTOR_SIZE, +}; + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum PortType { + Sata, +} + +struct PortInner { + regs: DeviceMemoryIo<'static, PortRegs>, + + #[allow(unused)] + received_fis: PageBox, + command_list: PageBox<[CommandListEntry]>, +} + +pub struct PortInfo { + pub model: String, + pub serial: String, + pub lba_count: u64, +} + +#[allow(unused)] +pub struct AhciPort { + inner: IrqSafeSpinlock, + ahci: &'static AhciController, + ty: PortType, + index: usize, + info: OneTimeInit, + cache: IrqSafeSpinlock>, + + command_allocation: IrqSafeSpinlock, + // One command index can only be waited for by one task, so this approach is usable + command_completion: [AtomicWaker; COMMAND_LIST_LENGTH], + command_available: QueueWaker, +} + +impl PortInner { + fn submit_command( + &mut self, + index: usize, + command: &C, + ) -> Result<(), AhciError> { + let list_entry = &mut self.command_list[index]; + let mut table_entry = + PageBox::new(CommandTable::zeroed()).map_err(AhciError::MemoryError)?; + + table_entry.setup_command(command)?; + *list_entry = CommandListEntry::new( + unsafe { table_entry.as_physical_address() }, + command.regions().len(), + )?; + + // Sync before send + unsafe { + core::arch::asm!("wbinvd"); + } + + // TODO deal with this async way + while self.regs.TFD.matches_any(TFD::BSY::SET + TFD::DRQ::SET) { + core::hint::spin_loop(); + } + + let ci = self.regs.CI.get(); + assert_eq!(ci & (1 << index), 0); + self.regs.CI.set(ci | (1 << index)); + + Ok(()) + } +} + +impl AhciPort { + pub async fn create( + regs: DeviceMemoryIo<'static, PortRegs>, + ahci: &'static AhciController, + index: usize, + ) -> Result<&'static Self, AhciError> { + log::debug!("Initialize port {}", index); + regs.stop()?; + + if !ahci.has_64_bit { + todo!("Handle controllers incapable of 64 bit"); + } + + let received_fis = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; + let command_list = PageBox::new_slice(CommandListEntry::zeroed(), COMMAND_LIST_LENGTH) + .map_err(AhciError::MemoryError)?; + + regs.set_received_fis_address_64(unsafe { received_fis.as_physical_address() }); + regs.set_command_list_address_64(unsafe { command_list.as_physical_address() }); + + regs.start()?; + + let inner = PortInner { + regs, + command_list, + received_fis, + }; + let command_completion = [const { AtomicWaker::new() }; MAX_COMMANDS]; + let command_available = QueueWaker::new(); + let command_allocation = IrqSafeSpinlock::new(0); + + let port = Box::leak(Box::new(Self { + inner: IrqSafeSpinlock::new(inner), + ty: PortType::Sata, + info: OneTimeInit::new(), + cache: IrqSafeSpinlock::new(BlockCache::new(SECTOR_SIZE)), + ahci, + index, + + command_completion, + command_allocation, + command_available, + })); + + runtime::spawn(port.poll_task()).unwrap(); + + let identify = port.perform_command(AtaIdentify::create()?).await?; + let model = identify.model_number.to_string(); + let serial = identify.serial_number.to_string(); + let lba_count = identify.logical_sector_count(); + + // TODO can sector size be different from 512 in ATA? + // should logical sector size be accounted for? + // TODO test for ReadDmaEx capability (?) + + port.info.init(PortInfo { + model, + serial, + lba_count, + }); + + Ok(port) + } + + pub fn info(&self) -> Option<&PortInfo> { + self.info.try_get() + } + + async fn perform_command(&self, command: C) -> Result { + let slot = self.allocate_command().await?; + log::trace!( + "Submit command on port {}, cmd index = {}", + self.index, + slot + ); + self.inner.lock().submit_command(slot, &command)?; + self.wait_for_completion(slot).await?; + self.free_command(slot); + Ok(unsafe { command.into_response() }) + } + + async fn read_block( + &self, + lba: u64, + block: &mut PageBox<[MaybeUninit]>, + ) -> Result<(), AhciError> { + log::debug!("read_block port={}, lba={}", self.index, lba); + self.perform_command(AtaReadDmaEx::new(lba, block)).await + } + + fn allocate_command<'r>(&'r self) -> impl Future> + 'r { + struct F<'f> { + waker: &'f QueueWaker, + state: &'f IrqSafeSpinlock, + } + + impl<'f> Future for F<'f> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.waker.register(cx.waker()); + + let mut state = self.state.lock(); + + if *state != u32::MAX { + self.waker.remove(cx.waker()); + for i in 0..MAX_COMMANDS { + if *state & (1 << i) == 0 { + *state |= 1 << i; + return Poll::Ready(Ok(i)); + } + } + + panic!("Unreachable"); + } else { + Poll::Pending + } + } + } + + log::debug!("allocate_command"); + + let waker = &self.command_available; + let state = &self.command_allocation; + + F { waker, state } + } + + fn wait_for_completion<'r>( + &'r self, + index: usize, + ) -> impl Future> + 'r { + struct F<'f> { + waker: &'f AtomicWaker, + inner: &'f IrqSafeSpinlock, + index: usize, + } + + impl<'f> Future for F<'f> { + type Output = Result<(), AhciError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let inner = self.inner.lock(); + + if let CommandState::Ready(status) = inner.regs.command_status(self.index) { + return Poll::Ready(status.into()); + } + + self.waker.register(cx.waker()); + + match inner.regs.command_status(self.index) { + CommandState::Pending => Poll::Pending, + CommandState::Ready(status) => Poll::Ready(status.into()), + } + } + } + + log::debug!("wait_for_completion {}", index); + + let waker = &self.command_completion[index]; + let inner = &self.inner; + + F { + waker, + inner, + index, + } + } + + fn free_command(&self, index: usize) { + { + let mut alloc = self.command_allocation.lock(); + assert_ne!(*alloc & (1 << index), 0); + *alloc &= !(1 << index); + } + self.command_available.wake_one(); + } + + // TODO use MSI + async fn poll_task(&self) { + loop { + for i in 0..MAX_COMMANDS { + self.command_completion[i].wake(); + } + + runtime::sleep(Duration::from_millis(50)).await; + } + } +} + +impl BlockDevice for AhciPort { + fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { + let info = self.info.try_get().ok_or(Error::PermissionDenied)?; + + let mut cache = self.cache.lock(); + let mut rem = buf.len(); + let mut off = 0; + + while rem != 0 { + let lba = pos / SECTOR_SIZE as u64; + + if lba >= info.lba_count { + break; + } + + let block_offset = (pos % SECTOR_SIZE as u64) as usize; + let count = core::cmp::min(SECTOR_SIZE - block_offset, rem); + + let block = cache.get_or_fetch_with(lba, |block| { + block! { + self.read_block(lba, block).await + }? + .map_err(|_| Error::InvalidOperation) + })?; + + buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); + + rem -= count; + off += count; + pos += count as u64; + } + + Ok(off) + } + + fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { + todo!() + } + + fn size(&self) -> Result { + let info = self.info.try_get().ok_or(Error::PermissionDenied)?; + Ok(info.lba_count * SECTOR_SIZE as u64) + } + + fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { + todo!() + } +} diff --git a/driver/block/ahci/src/regs.rs b/driver/block/ahci/src/regs.rs index 09e4ebfd..e437c6aa 100644 --- a/driver/block/ahci/src/regs.rs +++ b/driver/block/ahci/src/regs.rs @@ -184,7 +184,7 @@ impl Into> for CommandStatus { fn into(self) -> Result<(), AhciError> { match self { Self::Success => Ok(()), - Self::TaskFileError => Err(todo!()), + Self::TaskFileError => Err(AhciError::TaskFileError), } } } diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index ea782c78..c01b1dbb 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -6,7 +6,7 @@ extern crate alloc; use core::fmt; use acpi::mcfg::McfgEntry; -use alloc::{rc::Rc, vec::Vec}; +use alloc::vec::Vec; use bitflags::bitflags; use device_api::Device; use kernel_util::{ From c870d7290f75f006924fce8b8a56f6668ccfff28 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 13 Dec 2023 00:18:59 +0200 Subject: [PATCH 126/211] block/ahci: implement MSI-driven updates --- driver/block/ahci/src/command.rs | 5 +- driver/block/ahci/src/error.rs | 2 +- driver/block/ahci/src/lib.rs | 73 +++++++++++++++++---- driver/block/ahci/src/port.rs | 102 +++++++++++++++++------------- driver/block/ahci/src/regs.rs | 57 ++++++++++------- driver/bus/pci/src/capability.rs | 64 +++++++++++++++++++ lib/kernel-util/src/mem/device.rs | 7 ++ lib/kernel-util/src/util.rs | 8 +-- 8 files changed, 232 insertions(+), 86 deletions(-) diff --git a/driver/block/ahci/src/command.rs b/driver/block/ahci/src/command.rs index 35eda27a..4fb26e08 100644 --- a/driver/block/ahci/src/command.rs +++ b/driver/block/ahci/src/command.rs @@ -235,9 +235,7 @@ impl AtaCommand for AtaReadDmaEx { &self.regions } - unsafe fn into_response(self) -> Self::Response { - () - } + unsafe fn into_response(self) -> Self::Response {} } impl CommandTable { @@ -362,6 +360,7 @@ impl AtaString where Assert<{ N % 2 == 0 }>: IsTrue, { + #[allow(clippy::inherent_to_string)] pub fn to_string(&self) -> String { let mut buf = [0; N]; diff --git a/driver/block/ahci/src/error.rs b/driver/block/ahci/src/error.rs index d9325ca6..85f3b418 100644 --- a/driver/block/ahci/src/error.rs +++ b/driver/block/ahci/src/error.rs @@ -4,5 +4,5 @@ use yggdrasil_abi::error::Error; pub enum AhciError { MemoryError(Error), RegionTooLarge, - TaskFileError, + DeviceError, } diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 6aff475b..0d46cf65 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -5,7 +5,10 @@ extern crate alloc; use alloc::{boxed::Box, format, vec, vec::Vec}; -use device_api::Device; +use device_api::{ + interrupt::{InterruptAffinity, MsiHandler}, + Device, +}; use error::AhciError; use kernel_fs::devfs; use kernel_util::{ @@ -13,14 +16,17 @@ use kernel_util::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIo, }, - runtime, + message_interrupt_controller, runtime, sync::IrqSafeSpinlock, util::OneTimeInit, }; use port::AhciPort; use regs::{PortRegs, Regs}; -use tock_registers::interfaces::{ReadWriteable, Readable}; -use ygg_driver_pci::{PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use ygg_driver_pci::{ + capability::MsiCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, + PciDeviceInfo, +}; use yggdrasil_abi::error::Error; use crate::regs::{Version, CAP, GHC, SSTS}; @@ -61,17 +67,18 @@ impl AhciController { regs.GHC.modify(GHC::AE::SET); } - // TODO configure interrupts - let pi = regs.PI.get(); let mut ports = vec![]; + drop(regs); + for i in 0..self.max_port_count { if pi & (1 << i) == 0 { continue; } + let regs = self.regs.lock(); let port = ®s.PORTS[i]; if !port.SSTS.matches_all(SSTS::DET::Online + SSTS::IPM::Active) { @@ -90,7 +97,9 @@ impl AhciController { let port = unsafe { regs.extract(|regs| ®s.PORTS[i]) }; - let port = match AhciPort::create(port, self, i).await { + drop(regs); + + let port = match AhciPort::create(port, self, i) { Ok(port) => port, Err(error) => { log::warn!("Port {} init error: {:?}", i, error); @@ -101,6 +110,17 @@ impl AhciController { ports.push(port); } + let ports = self.ports.init(ports); + + // Enable global HC interrupts + self.regs.lock().GHC.modify(GHC::IE::SET); + + // Setup the detected ports + for (i, &port) in ports.iter().enumerate() { + log::info!("Init port {}", i); + port.init().await?; + } + // Dump info about the drives for (i, &port) in ports.iter().enumerate() { let info = port.info().unwrap(); @@ -128,14 +148,32 @@ impl AhciController { } } - self.ports.init(ports); - log::debug!("All ports initialized"); Ok(()) } } +impl MsiHandler for AhciController { + fn handle_msi(&self, _vector: usize) -> bool { + let regs = self.regs.lock(); + + let is = regs.IS.get(); + if is != 0 { + // Clear global interrupt status + regs.IS.set(u32::MAX); + + for &port in self.ports.get() { + if is & (1 << port.index) != 0 { + port.handle_pending_interrupts(); + } + } + } + + false + } +} + impl Device for AhciController { unsafe fn init(&'static self) -> Result<(), Error> { // Do the init in background @@ -156,6 +194,12 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { return Err(Error::InvalidOperation); }; + // TODO support regular PCI interrupts (ACPI dependency) + let Some(mut msi) = info.config_space.capability::() else { + log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)"); + return Err(Error::InvalidOperation); + }; + // Map the registers let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar5)) }?; let version = Version::try_from(regs.VS.get())?; @@ -168,12 +212,19 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; info.config_space.set_command(cmd.bits()); - Ok(Box::leak(Box::new(AhciController { + // TODO extract Number of Command Slots + + let ahci = Box::leak(Box::new(AhciController { regs: IrqSafeSpinlock::new(regs), ports: OneTimeInit::new(), version, max_port_count, ahci_only, has_64_bit, - }))) + })); + + // TODO use multiple vectors if capable + msi.register(message_interrupt_controller(), InterruptAffinity::Any, ahci)?; + + Ok(ahci) } diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index 059367cd..f79d9663 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -1,8 +1,8 @@ use core::{ mem::MaybeUninit, pin::Pin, + sync::atomic::{AtomicU32, Ordering}, task::{Context, Poll}, - time::Duration, }; use alloc::{boxed::Box, string::String}; @@ -11,7 +11,7 @@ use futures_util::{task::AtomicWaker, Future}; use kernel_util::{ block, mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, - runtime::{self, QueueWaker}, + runtime::QueueWaker, sync::IrqSafeSpinlock, util::OneTimeInit, }; @@ -25,7 +25,7 @@ use crate::{ COMMAND_LIST_LENGTH, }, error::AhciError, - regs::{CommandState, PortRegs, TFD}, + regs::{CommandState, CommandStatus, PortRegs, IE, TFD}, AhciController, MAX_COMMANDS, SECTOR_SIZE, }; @@ -53,13 +53,13 @@ pub struct AhciPort { inner: IrqSafeSpinlock, ahci: &'static AhciController, ty: PortType, - index: usize, + pub(crate) index: usize, info: OneTimeInit, cache: IrqSafeSpinlock>, command_allocation: IrqSafeSpinlock, // One command index can only be waited for by one task, so this approach is usable - command_completion: [AtomicWaker; COMMAND_LIST_LENGTH], + command_completion: [(AtomicWaker, AtomicU32); COMMAND_LIST_LENGTH], command_available: QueueWaker, } @@ -98,7 +98,7 @@ impl PortInner { } impl AhciPort { - pub async fn create( + pub fn create( regs: DeviceMemoryIo<'static, PortRegs>, ahci: &'static AhciController, index: usize, @@ -117,6 +117,16 @@ impl AhciPort { regs.set_received_fis_address_64(unsafe { received_fis.as_physical_address() }); regs.set_command_list_address_64(unsafe { command_list.as_physical_address() }); + regs.IE.write( + IE::DPE::SET + + IE::IFE::SET + + IE::OFE::SET + + IE::HBDE::SET + + IE::HBFE::SET + + IE::TFEE::SET + + IE::DHRE::SET, + ); + regs.start()?; let inner = PortInner { @@ -124,11 +134,11 @@ impl AhciPort { command_list, received_fis, }; - let command_completion = [const { AtomicWaker::new() }; MAX_COMMANDS]; + let command_completion = [const { (AtomicWaker::new(), AtomicU32::new(0)) }; MAX_COMMANDS]; let command_available = QueueWaker::new(); let command_allocation = IrqSafeSpinlock::new(0); - let port = Box::leak(Box::new(Self { + Ok(Box::leak(Box::new(Self { inner: IrqSafeSpinlock::new(inner), ty: PortType::Sata, info: OneTimeInit::new(), @@ -139,11 +149,11 @@ impl AhciPort { command_completion, command_allocation, command_available, - })); + }))) + } - runtime::spawn(port.poll_task()).unwrap(); - - let identify = port.perform_command(AtaIdentify::create()?).await?; + pub async fn init(&'static self) -> Result<(), AhciError> { + let identify = self.perform_command(AtaIdentify::create()?).await?; let model = identify.model_number.to_string(); let serial = identify.serial_number.to_string(); let lba_count = identify.logical_sector_count(); @@ -152,13 +162,13 @@ impl AhciPort { // should logical sector size be accounted for? // TODO test for ReadDmaEx capability (?) - port.info.init(PortInfo { + self.info.init(PortInfo { model, serial, lba_count, }); - Ok(port) + Ok(()) } pub fn info(&self) -> Option<&PortInfo> { @@ -187,7 +197,7 @@ impl AhciPort { self.perform_command(AtaReadDmaEx::new(lba, block)).await } - fn allocate_command<'r>(&'r self) -> impl Future> + 'r { + fn allocate_command(&self) -> impl Future> + '_ { struct F<'f> { waker: &'f QueueWaker, state: &'f IrqSafeSpinlock, @@ -203,6 +213,7 @@ impl AhciPort { if *state != u32::MAX { self.waker.remove(cx.waker()); + for i in 0..MAX_COMMANDS { if *state & (1 << i) == 0 { *state |= 1 << i; @@ -217,53 +228,44 @@ impl AhciPort { } } - log::debug!("allocate_command"); - let waker = &self.command_available; let state = &self.command_allocation; F { waker, state } } - fn wait_for_completion<'r>( - &'r self, + fn wait_for_completion( + &self, index: usize, - ) -> impl Future> + 'r { + ) -> impl Future> + '_ { struct F<'f> { waker: &'f AtomicWaker, - inner: &'f IrqSafeSpinlock, - index: usize, + status: &'f AtomicU32, } impl<'f> Future for F<'f> { type Output = Result<(), AhciError>; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let inner = self.inner.lock(); - - if let CommandState::Ready(status) = inner.regs.command_status(self.index) { - return Poll::Ready(status.into()); + match self.status.load(Ordering::Acquire) { + 0 => (), + 1 => return Poll::Ready(Ok(())), + _ => return Poll::Ready(Err(AhciError::DeviceError)), } self.waker.register(cx.waker()); - match inner.regs.command_status(self.index) { - CommandState::Pending => Poll::Pending, - CommandState::Ready(status) => Poll::Ready(status.into()), + match self.status.load(Ordering::Acquire) { + 0 => Poll::Pending, + 1 => Poll::Ready(Ok(())), + _ => Poll::Ready(Err(AhciError::DeviceError)), } } } - log::debug!("wait_for_completion {}", index); + let (waker, status) = &self.command_completion[index]; - let waker = &self.command_completion[index]; - let inner = &self.inner; - - F { - waker, - inner, - index, - } + F { status, waker } } fn free_command(&self, index: usize) { @@ -275,15 +277,25 @@ impl AhciPort { self.command_available.wake_one(); } - // TODO use MSI - async fn poll_task(&self) { - loop { - for i in 0..MAX_COMMANDS { - self.command_completion[i].wake(); - } + pub fn handle_pending_interrupts(&self) -> bool { + let inner = self.inner.lock(); - runtime::sleep(Duration::from_millis(50)).await; + for i in 0..MAX_COMMANDS { + match inner.regs.clear_state(i) { + CommandState::Pending => (), + CommandState::Ready(status) => { + // TODO better error handling? + let val = match status { + CommandStatus::Success => 1, + _ => 2, + }; + + self.command_completion[i].1.store(val, Ordering::Release); + self.command_completion[i].0.wake(); + } + } } + true } } diff --git a/driver/block/ahci/src/regs.rs b/driver/block/ahci/src/regs.rs index e437c6aa..8f24649c 100644 --- a/driver/block/ahci/src/regs.rs +++ b/driver/block/ahci/src/regs.rs @@ -22,12 +22,22 @@ register_bitfields! { AE OFFSET(31) NUMBITS(1) [], ], + // Read/write 1 to clear pub IS [ TFES OFFSET(30) NUMBITS(1) [], - DHRS OFFSET(0) NUMBITS(1) [], + HBFS OFFSET(29) NUMBITS(1) [], + HBDS OFFSET(28) NUMBITS(1) [], + IFS OFFSET(27) NUMBITS(1) [], + OFS OFFSET(24) NUMBITS(1) [], ], pub IE [ - DHRS OFFSET(0) NUMBITS(1) [], + TFEE OFFSET(30) NUMBITS(1) [], + HBFE OFFSET(29) NUMBITS(1) [], + HBDE OFFSET(28) NUMBITS(1) [], + IFE OFFSET(27) NUMBITS(1) [], + OFE OFFSET(24) NUMBITS(1) [], + DPE OFFSET(5) NUMBITS(1) [], + DHRE OFFSET(0) NUMBITS(1) [], ], pub CMD [ CR OFFSET(15) NUMBITS(1) [], @@ -59,7 +69,7 @@ register_structs! { pub Regs { (0x0000 => pub CAP: ReadOnly), (0x0004 => pub GHC: ReadWrite), - (0x0008 => pub IS: ReadOnly), + (0x0008 => pub IS: ReadWrite), (0x000C => pub PI: ReadOnly), (0x0010 => pub VS: ReadOnly), (0x0014 => _0), @@ -75,7 +85,7 @@ register_structs! { (0x04 => pub CLBU: ReadWrite), (0x08 => pub FB: ReadWrite), (0x0C => pub FBU: ReadWrite), - (0x10 => pub IS: ReadOnly), + (0x10 => pub IS: ReadWrite), (0x14 => pub IE: ReadWrite), (0x18 => pub CMD: ReadWrite), (0x1C => _0), @@ -151,16 +161,28 @@ impl PortRegs { self.CLBU.set((address >> 32) as u32); } - pub fn command_status(&self, index: usize) -> CommandState { - if self.CI.get() & (1 << index) == 0 { - CommandState::Ready(if self.IS.matches_all(IS::TFES::SET) { - CommandStatus::TaskFileError - } else { - CommandStatus::Success - }) - } else { - CommandState::Pending + pub fn clear_state(&self, index: usize) -> CommandState { + let is = self.IS.extract(); + let ci = self.CI.get(); + + if is.get() == 0 { + return CommandState::Pending; } + + // Clear everything + self.IS.set(0xFFFFFFFF); + + if is.matches_any(IS::HBDS::SET + IS::HBFS::SET) { + todo!("Host communication error unhandled"); + } + + assert_eq!(ci & (1 << index), 0); + + if is.matches_any(IS::TFES::SET + IS::IFS::SET + IS::OFS::SET) { + return CommandState::Ready(CommandStatus::TaskFileError); + } + + CommandState::Ready(CommandStatus::Success) } } @@ -179,12 +201,3 @@ impl TryFrom for Version { } } } - -impl Into> for CommandStatus { - fn into(self) -> Result<(), AhciError> { - match self { - Self::Success => Ok(()), - Self::TaskFileError => Err(AhciError::TaskFileError), - } - } -} diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index de335793..cb62c932 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -14,6 +14,9 @@ use crate::PciBaseAddress; /// MSI-X capability query pub struct MsiXCapability; +/// MSI capability query +pub struct MsiCapability; + /// Represents an entry in MSI-X vector table #[repr(C)] pub struct MsiXEntry { @@ -35,6 +38,12 @@ pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> { offset: usize, } +/// MSI capability data structure +pub struct MsiData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + impl PciCapability for MsiXCapability { const ID: PciCapabilityId = PciCapabilityId::MsiX; type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>; @@ -47,6 +56,18 @@ impl PciCapability for MsiXCapability { } } +impl PciCapability for MsiCapability { + const ID: PciCapabilityId = PciCapabilityId::Msi; + type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiData<'a, S>; + + fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( + space: &'s S, + offset: usize, + ) -> Self::CapabilityData<'s, S> { + MsiData { space, offset } + } +} + impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { // TODO use pending bits as well /// Maps and returns the vector table associated with the device's MSI-X capability @@ -149,3 +170,46 @@ impl MsiXEntry { } } } + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> { + pub fn register( + &mut self, + ic: &C, + affinity: InterruptAffinity, + handler: &'static dyn MsiHandler, + ) -> Result { + let info = ic.register_msi(affinity, handler)?; + + let mut w0 = self.space.read_u16(self.offset + 2); + // Enable the vector first + w0 |= 1 << 0; + + // Reset to one vector + w0 &= !(0x7 << 4); + + self.space.write_u16(self.offset + 2, w0); + + if info.value > u16::MAX as u32 { + log::warn!("Could not setup a MSI: value={:#x} > u16", info.value); + return Err(Error::InvalidOperation); + } + + if info.address > u32::MAX as usize { + if w0 & (1 << 7) == 0 { + log::warn!( + "Could not setup a MSI: address={:#x} and MSI is not 64 bit capable", + info.address + ); + return Err(Error::InvalidOperation); + } + + self.space + .write_u32(self.offset + 8, (info.address >> 32) as u32); + } + self.space.write_u32(self.offset + 4, info.address as u32); + + self.space.write_u16(self.offset + 12, info.value as u16); + + Ok(info) + } +} diff --git a/lib/kernel-util/src/mem/device.rs b/lib/kernel-util/src/mem/device.rs index 822d33ba..7d5433ce 100644 --- a/lib/kernel-util/src/mem/device.rs +++ b/lib/kernel-util/src/mem/device.rs @@ -155,6 +155,13 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { } impl<'a, T: ?Sized> DeviceMemoryIo<'a, T> { + /// Extracts an inner reference to `U` from within `T`. + /// + /// # Safety + /// + /// To use this safely, the caller must guarantee the "extracted" reference will be the only + /// valid reference and that `&U` won't be usable again through its parent `T`. The caller must + /// also guarantee that `&U` is, in fact, contained within `T`. pub unsafe fn extract<'b, U: ?Sized + 'b, F: FnOnce(&'a T) -> *const U>( &'a self, f: F, diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util.rs index 8f505750..0b46941b 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util.rs @@ -44,7 +44,7 @@ impl OneTimeInit { /// Sets the underlying value of the [OneTimeInit]. If already initialized, panics. #[track_caller] - pub fn init(&self, value: T) { + pub fn init(&self, value: T) -> &T { // Transition to "initializing" state if self .state @@ -62,9 +62,7 @@ impl OneTimeInit { ); } - unsafe { - (*self.value.get()).write(value); - } + let value = unsafe { (*self.value.get()).write(value) }; // Transition to "initialized" state. This must not fail self.state @@ -75,6 +73,8 @@ impl OneTimeInit { Ordering::Relaxed, ) .unwrap(); + + value } /// Returns an immutable reference to the underlying value and panics if it hasn't yet been From 789b8974342c945c4727097bd1bd12778f51aec8 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 13 Dec 2023 00:38:18 +0200 Subject: [PATCH 127/211] block/ahci: split data structures --- driver/block/ahci/src/command.rs | 276 ++----------------------------- driver/block/ahci/src/data.rs | 254 ++++++++++++++++++++++++++++ driver/block/ahci/src/lib.rs | 1 + driver/block/ahci/src/port.rs | 6 +- lib/kernel-util/src/util.rs | 5 + 5 files changed, 280 insertions(+), 262 deletions(-) create mode 100644 driver/block/ahci/src/data.rs diff --git a/driver/block/ahci/src/command.rs b/driver/block/ahci/src/command.rs index 4fb26e08..3263d519 100644 --- a/driver/block/ahci/src/command.rs +++ b/driver/block/ahci/src/command.rs @@ -1,111 +1,29 @@ use core::mem::{size_of, MaybeUninit}; -use alloc::string::String; -use bytemuck::{Pod, Zeroable}; use kernel_util::mem::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; -use static_assertions::const_assert_eq; use tock_registers::register_structs; -use crate::{error::AhciError, MAX_PRD_SIZE}; +use crate::{data::AtaString, error::AhciError, SECTOR_SIZE}; -pub const COMMAND_LIST_LENGTH: usize = 32; - -const AHCI_FIS_REG_H2D_COMMAND: u8 = 1 << 7; -const AHCI_FIS_REG_H2D: u8 = 0x27; - -pub enum Assert {} -pub trait IsTrue {} - -impl IsTrue for Assert {} - -#[repr(C)] -pub struct AtaString -where - Assert<{ N % 2 == 0 }>: IsTrue, -{ - data: [u8; N], +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[repr(u8)] +pub enum AtaCommandId { + Identify = 0xEC, + ReadDmaEx = 0x25, } -#[derive(Debug, Clone, Copy, Zeroable)] -#[repr(C)] -pub struct PhysicalRegionDescriptor { - buffer_address: u64, - _0: u32, - dbc: u32, -} +pub trait AtaCommand { + type Response; -#[derive(Debug, Clone, Copy, Zeroable)] -#[repr(C)] -pub struct CommandListEntry { - attr: u16, - prdtl: u16, - prdbc: u32, - ctba: u64, - _0: [u32; 4], -} + const COMMAND_ID: AtaCommandId; -#[derive(Clone, Copy, Zeroable)] -#[repr(C)] -pub union SentFis { - reg_h2d: RegisterHostToDeviceFis, - raw: RawFis, -} - -#[derive(Clone, Copy, Zeroable)] -#[repr(C)] -pub struct RegisterHostToDeviceFis { - pub ty: u8, - pub cmd_port: u8, - pub cmd: u8, - pub feature_low: u8, - - pub lba0: u8, - pub lba1: u8, - pub lba2: u8, - pub device: u8, - - pub lba3: u8, - pub lba4: u8, - pub lba5: u8, - pub feature_high: u8, - - pub count: u16, - pub icc: u8, - pub control: u8, - - _0: u32, -} - -#[derive(Clone, Copy, Zeroable, Pod)] -#[repr(C)] -pub struct RawFis { - pub bytes: [u8; 64], -} - -#[derive(Clone, Copy)] -#[repr(C)] -pub struct CommandTable { - fis: SentFis, // 0x00..0x40 - _0: [u8; 16], // 0x40..0x50 - _1: [u8; 48], // 0x50..0x80 - prdt: [PhysicalRegionDescriptor; 248], // 0x80..0x1000 -} - -#[derive(Clone, Copy, Zeroable, Pod)] -#[repr(C)] -pub struct ReceivedFis { - _dsfis: [u8; 0x1C], // 0x00..0x1C - _0: [u8; 0x04], // 0x1C..0x20 - _psfis: [u8; 0x14], // 0x20..0x34 - _1: [u8; 0x0C], // 0x34..0x40 - _rfis: [u8; 0x14], // 0x40..0x54 - _2: [u8; 0x04], // 0x54..0x58 - _sdbfis: [u8; 0x08], // 0x58..0x60 - _ufis: [u8; 0x40], // 0x60..0xA0 - _3: [u8; 0x60], // 0xA0..0x100 + fn lba(&self) -> u64; + fn sector_count(&self) -> usize; + fn regions(&self) -> &[(PhysicalAddress, usize)]; + unsafe fn into_response(self) -> Self::Response; } register_structs! { @@ -137,11 +55,6 @@ register_structs! { } } -const_assert_eq!(size_of::(), 0x40); -const_assert_eq!(size_of::(), 0x1000); -const_assert_eq!(size_of::(), 32); -const_assert_eq!(size_of::(), 0x100); - pub struct AtaIdentify { buffer: PageBox>, regions: [(PhysicalAddress, usize); 1], @@ -153,17 +66,6 @@ pub struct AtaReadDmaEx { regions: [(PhysicalAddress, usize); 1], } -pub trait AtaCommand { - type Response; - - const COMMAND_ID: u8; - - fn lba(&self) -> u64; - fn sector_count(&self) -> usize; - fn regions(&self) -> &[(PhysicalAddress, usize)]; - unsafe fn into_response(self) -> Self::Response; -} - impl AtaIdentify { pub fn create() -> Result { PageBox::new_uninit() @@ -184,10 +86,10 @@ impl AtaIdentify { impl AtaReadDmaEx { pub fn new(lba: u64, buffer: &PageBox<[MaybeUninit]>) -> Self { - assert_eq!(buffer.len() % 512, 0); + assert_eq!(buffer.len() % SECTOR_SIZE, 0); assert_ne!(buffer.len(), 0); - let sector_count = buffer.len() / 512; + let sector_count = buffer.len() / SECTOR_SIZE; Self { lba, sector_count, @@ -199,7 +101,7 @@ impl AtaReadDmaEx { impl AtaCommand for AtaIdentify { type Response = PageBox; - const COMMAND_ID: u8 = 0xEC; + const COMMAND_ID: AtaCommandId = AtaCommandId::Identify; fn lba(&self) -> u64 { 0 @@ -221,7 +123,7 @@ impl AtaCommand for AtaIdentify { impl AtaCommand for AtaReadDmaEx { type Response = (); - const COMMAND_ID: u8 = 0x25; + const COMMAND_ID: AtaCommandId = AtaCommandId::ReadDmaEx; fn lba(&self) -> u64 { self.lba @@ -237,145 +139,3 @@ impl AtaCommand for AtaReadDmaEx { unsafe fn into_response(self) -> Self::Response {} } - -impl CommandTable { - pub fn setup_command(&mut self, command: &C) -> Result<(), AhciError> { - let lba = command.lba(); - assert_eq!(lba & !0xFFFFFFFFFF, 0); - let count = command.sector_count().try_into().unwrap(); - - if C::COMMAND_ID == AtaIdentify::COMMAND_ID { - self.fis = SentFis { - reg_h2d: RegisterHostToDeviceFis { - ty: AHCI_FIS_REG_H2D, - cmd_port: AHCI_FIS_REG_H2D_COMMAND, - cmd: C::COMMAND_ID, - - ..RegisterHostToDeviceFis::zeroed() - }, - }; - } else { - self.fis = SentFis { - reg_h2d: RegisterHostToDeviceFis { - ty: AHCI_FIS_REG_H2D, - cmd_port: AHCI_FIS_REG_H2D_COMMAND, - cmd: C::COMMAND_ID, - device: 1 << 6, // LBA mode - lba0: lba as u8, - lba1: (lba >> 8) as u8, - lba2: (lba >> 16) as u8, - lba3: (lba >> 24) as u8, - lba4: (lba >> 32) as u8, - lba5: (lba >> 40) as u8, - count, - - ..RegisterHostToDeviceFis::zeroed() - }, - }; - } - - let regions = command.regions(); - for (i, &(base, size)) in regions.iter().enumerate() { - let last = i == regions.len() - 1; - self.prdt[i] = PhysicalRegionDescriptor::new(base, size, last)?; - } - - Ok(()) - } -} - -impl CommandListEntry { - pub fn new(command_table_entry: PhysicalAddress, prd_count: usize) -> Result { - if prd_count > 0xFFFF { - todo!() - } - Ok(Self { - // attr = FIS size in dwords - attr: (size_of::() / size_of::()) as _, - prdtl: prd_count as _, - prdbc: 0, - ctba: command_table_entry.into_raw(), - _0: [0; 4], - }) - } -} - -unsafe impl Zeroable for CommandTable { - fn zeroed() -> Self { - Self { - fis: SentFis::zeroed(), - _0: [0; 16], - _1: [0; 48], - prdt: [PhysicalRegionDescriptor::zeroed(); 248], - } - } -} - -impl PhysicalRegionDescriptor { - pub fn new( - address: PhysicalAddress, - byte_count: usize, - is_last: bool, - ) -> Result { - if byte_count >= MAX_PRD_SIZE { - return Err(AhciError::RegionTooLarge); - } - - let dbc_mask = (is_last as u32) << 31; - Ok(Self { - buffer_address: address.into_raw(), - _0: 0, - dbc: ((byte_count as u32 - 1) << 1) | 1 | dbc_mask, - }) - } -} - -impl AtaIdentifyResponse { - pub fn logical_sector_count(&self) -> u64 { - // If logical_sector_count_28 == 0x0FFFFFFF, and logical_sector_count_qw >= 0x0FFFFFFF, - // then ACCEESSIBLE CAPACITY (?) field contains the total number of user addressable - // LBAs (see 4.1) - // bit 3 in additional_features -> logical_sector_count_qw: - // 0 -> max value = 0xFFFFFFFFFFFF (48) - // 1 -> max value = 0xFFFFFFFF (32) - // If bit 3 in additional_features is set, ext_logical_sector_count_qw contains - // the maximum addressable LBA count. Max value = 0xFFFFFFFFFFFF (48) - - if self.command_sets[1] & (1 << 10) != 0 { - // 48-bit supported - if self.additional_features & (1 << 3) != 0 { - // Use ext_logical_sector_count_qw - todo!() - } else { - // Use logical_sector_count_qw - self.logical_sector_count_qw - } - } else { - todo!() - } - } -} - -impl AtaString -where - Assert<{ N % 2 == 0 }>: IsTrue, -{ - #[allow(clippy::inherent_to_string)] - pub fn to_string(&self) -> String { - let mut buf = [0; N]; - - for i in (0..N).step_by(2) { - buf[i] = self.data[i + 1]; - buf[i + 1] = self.data[i]; - } - let mut len = 0; - for i in (0..N).rev() { - if buf[i] != b' ' { - len = i + 1; - break; - } - } - - String::from(core::str::from_utf8(&buf[..len]).unwrap()) - } -} diff --git a/driver/block/ahci/src/data.rs b/driver/block/ahci/src/data.rs new file mode 100644 index 00000000..ff15f5ca --- /dev/null +++ b/driver/block/ahci/src/data.rs @@ -0,0 +1,254 @@ +use core::mem::size_of; + +use alloc::string::String; +use bytemuck::{Pod, Zeroable}; +use kernel_util::{ + mem::address::{IntoRaw, PhysicalAddress}, + util::{ConstAssert, IsTrue}, +}; +use static_assertions::const_assert_eq; + +use crate::{ + command::{AtaCommand, AtaIdentify, AtaIdentifyResponse}, + error::AhciError, + MAX_PRD_SIZE, +}; + +pub const COMMAND_LIST_LENGTH: usize = 32; + +const AHCI_FIS_REG_H2D_COMMAND: u8 = 1 << 7; +const AHCI_FIS_REG_H2D: u8 = 0x27; + +#[repr(C)] +pub struct AtaString +where + ConstAssert<{ N % 2 == 0 }>: IsTrue, +{ + data: [u8; N], +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub struct PhysicalRegionDescriptor { + buffer_address: u64, + _0: u32, + dbc: u32, +} + +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub struct CommandListEntry { + attr: u16, + prdtl: u16, + prdbc: u32, + ctba: u64, + _0: [u32; 4], +} + +#[derive(Clone, Copy, Zeroable)] +#[repr(C)] +pub union SentFis { + reg_h2d: RegisterHostToDeviceFis, + raw: RawFis, +} + +#[derive(Clone, Copy, Zeroable)] +#[repr(C)] +pub struct RegisterHostToDeviceFis { + pub ty: u8, + pub cmd_port: u8, + pub cmd: u8, + pub feature_low: u8, + + pub lba0: u8, + pub lba1: u8, + pub lba2: u8, + pub device: u8, + + pub lba3: u8, + pub lba4: u8, + pub lba5: u8, + pub feature_high: u8, + + pub count: u16, + pub icc: u8, + pub control: u8, + + _0: u32, +} + +#[derive(Clone, Copy, Zeroable, Pod)] +#[repr(C)] +pub struct RawFis { + pub bytes: [u8; 64], +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub struct CommandTable { + fis: SentFis, // 0x00..0x40 + _0: [u8; 16], // 0x40..0x50 + _1: [u8; 48], // 0x50..0x80 + prdt: [PhysicalRegionDescriptor; 248], // 0x80..0x1000 +} + +#[derive(Clone, Copy, Zeroable, Pod)] +#[repr(C)] +pub struct ReceivedFis { + _dsfis: [u8; 0x1C], // 0x00..0x1C + _0: [u8; 0x04], // 0x1C..0x20 + _psfis: [u8; 0x14], // 0x20..0x34 + _1: [u8; 0x0C], // 0x34..0x40 + _rfis: [u8; 0x14], // 0x40..0x54 + _2: [u8; 0x04], // 0x54..0x58 + _sdbfis: [u8; 0x08], // 0x58..0x60 + _ufis: [u8; 0x40], // 0x60..0xA0 + _3: [u8; 0x60], // 0xA0..0x100 +} + +const_assert_eq!(size_of::(), 0x40); +const_assert_eq!(size_of::(), 0x1000); +const_assert_eq!(size_of::(), 32); +const_assert_eq!(size_of::(), 0x100); + +impl CommandTable { + pub fn setup_command(&mut self, command: &C) -> Result<(), AhciError> { + let lba = command.lba(); + assert_eq!(lba & !0xFFFFFFFFFF, 0); + let count = command.sector_count().try_into().unwrap(); + + if C::COMMAND_ID == AtaIdentify::COMMAND_ID { + self.fis = SentFis { + reg_h2d: RegisterHostToDeviceFis { + ty: AHCI_FIS_REG_H2D, + cmd_port: AHCI_FIS_REG_H2D_COMMAND, + cmd: C::COMMAND_ID as _, + + ..RegisterHostToDeviceFis::zeroed() + }, + }; + } else { + self.fis = SentFis { + reg_h2d: RegisterHostToDeviceFis { + ty: AHCI_FIS_REG_H2D, + cmd_port: AHCI_FIS_REG_H2D_COMMAND, + cmd: C::COMMAND_ID as _, + device: 1 << 6, // LBA mode + lba0: lba as u8, + lba1: (lba >> 8) as u8, + lba2: (lba >> 16) as u8, + lba3: (lba >> 24) as u8, + lba4: (lba >> 32) as u8, + lba5: (lba >> 40) as u8, + count, + + ..RegisterHostToDeviceFis::zeroed() + }, + }; + } + + let regions = command.regions(); + for (i, &(base, size)) in regions.iter().enumerate() { + let last = i == regions.len() - 1; + self.prdt[i] = PhysicalRegionDescriptor::new(base, size, last)?; + } + + Ok(()) + } +} + +impl CommandListEntry { + pub fn new(command_table_entry: PhysicalAddress, prd_count: usize) -> Result { + if prd_count > 0xFFFF { + todo!() + } + Ok(Self { + // attr = FIS size in dwords + attr: (size_of::() / size_of::()) as _, + prdtl: prd_count as _, + prdbc: 0, + ctba: command_table_entry.into_raw(), + _0: [0; 4], + }) + } +} + +unsafe impl Zeroable for CommandTable { + fn zeroed() -> Self { + Self { + fis: SentFis::zeroed(), + _0: [0; 16], + _1: [0; 48], + prdt: [PhysicalRegionDescriptor::zeroed(); 248], + } + } +} + +impl PhysicalRegionDescriptor { + pub fn new( + address: PhysicalAddress, + byte_count: usize, + is_last: bool, + ) -> Result { + if byte_count >= MAX_PRD_SIZE { + return Err(AhciError::RegionTooLarge); + } + + let dbc_mask = (is_last as u32) << 31; + Ok(Self { + buffer_address: address.into_raw(), + _0: 0, + dbc: ((byte_count as u32 - 1) << 1) | 1 | dbc_mask, + }) + } +} + +impl AtaIdentifyResponse { + pub fn logical_sector_count(&self) -> u64 { + // If logical_sector_count_28 == 0x0FFFFFFF, and logical_sector_count_qw >= 0x0FFFFFFF, + // then ACCEESSIBLE CAPACITY (?) field contains the total number of user addressable + // LBAs (see 4.1) + // bit 3 in additional_features -> logical_sector_count_qw: + // 0 -> max value = 0xFFFFFFFFFFFF (48) + // 1 -> max value = 0xFFFFFFFF (32) + // If bit 3 in additional_features is set, ext_logical_sector_count_qw contains + // the maximum addressable LBA count. Max value = 0xFFFFFFFFFFFF (48) + + if self.command_sets[1] & (1 << 10) != 0 { + // 48-bit supported + if self.additional_features & (1 << 3) != 0 { + // Use ext_logical_sector_count_qw + todo!() + } else { + // Use logical_sector_count_qw + self.logical_sector_count_qw + } + } else { + todo!() + } + } +} + +impl AtaString +where + ConstAssert<{ N % 2 == 0 }>: IsTrue, +{ + #[allow(clippy::inherent_to_string)] + pub fn to_string(&self) -> String { + let mut buf = [0; N]; + + for i in (0..N).step_by(2) { + buf[i] = self.data[i + 1]; + buf[i + 1] = self.data[i]; + } + let mut len = 0; + for i in (0..N).rev() { + if buf[i] != b' ' { + len = i + 1; + break; + } + } + + String::from(core::str::from_utf8(&buf[..len]).unwrap()) + } +} diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 0d46cf65..99dbab8a 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -32,6 +32,7 @@ use yggdrasil_abi::error::Error; use crate::regs::{Version, CAP, GHC, SSTS}; mod command; +mod data; mod error; mod port; mod regs; diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index f79d9663..354c5cfb 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -20,10 +20,8 @@ use ygg_driver_block::{cache::BlockCache, BlockDevice}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::{ - command::{ - AtaCommand, AtaIdentify, AtaReadDmaEx, CommandListEntry, CommandTable, ReceivedFis, - COMMAND_LIST_LENGTH, - }, + command::{AtaCommand, AtaIdentify, AtaReadDmaEx}, + data::{CommandListEntry, CommandTable, ReceivedFis, COMMAND_LIST_LENGTH}, error::AhciError, regs::{CommandState, CommandStatus, PortRegs, IE, TFD}, AhciController, MAX_COMMANDS, SECTOR_SIZE, diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util.rs index 0b46941b..c5abb833 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util.rs @@ -7,6 +7,11 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; +pub enum ConstAssert {} +pub trait IsTrue {} + +impl IsTrue for ConstAssert {} + /// Statically-allocated "dynamic" vector pub struct StaticVector { data: [MaybeUninit; N], From d624505fb4920fa6f5d8a562b742c765220a4c2c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 18 Dec 2023 23:25:33 +0200 Subject: [PATCH 128/211] block/ahci: add partition enumeration --- driver/block/ahci/src/command.rs | 3 +- driver/block/ahci/src/lib.rs | 9 +- driver/block/ahci/src/port.rs | 158 +++++++++++++++++++++---------- driver/block/core/src/device.rs | 108 +++++++++++++++------ driver/block/nvme/src/drive.rs | 4 +- driver/fs/kernel-fs/src/devfs.rs | 3 +- 6 files changed, 201 insertions(+), 84 deletions(-) diff --git a/driver/block/ahci/src/command.rs b/driver/block/ahci/src/command.rs index 3263d519..9665d152 100644 --- a/driver/block/ahci/src/command.rs +++ b/driver/block/ahci/src/command.rs @@ -85,11 +85,10 @@ impl AtaIdentify { } impl AtaReadDmaEx { - pub fn new(lba: u64, buffer: &PageBox<[MaybeUninit]>) -> Self { + pub fn new(lba: u64, sector_count: usize, buffer: &PageBox<[MaybeUninit]>) -> Self { assert_eq!(buffer.len() % SECTOR_SIZE, 0); assert_ne!(buffer.len(), 0); - let sector_count = buffer.len() / SECTOR_SIZE; Self { lba, sector_count, diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 99dbab8a..aed1d4b2 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -23,6 +23,7 @@ use kernel_util::{ use port::AhciPort; use regs::{PortRegs, Regs}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use ygg_driver_block::{probe_partitions, NgBlockDeviceWrapper}; use ygg_driver_pci::{ capability::MsiCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, @@ -145,7 +146,13 @@ impl AhciController { lock.push(port); let name = format!("sd{}", (n + b'a') as char); - devfs::add_named_block_device(port, name).unwrap(); + + let blk = NgBlockDeviceWrapper::new(port); + devfs::add_named_block_device(blk, name.clone()).ok(); + probe_partitions(blk, move |index, partition| { + devfs::add_block_device_partition(name.clone(), index, partition) + }) + .ok(); } } diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index 354c5cfb..8b1da03c 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -1,5 +1,4 @@ use core::{ - mem::MaybeUninit, pin::Pin, sync::atomic::{AtomicU32, Ordering}, task::{Context, Poll}, @@ -9,15 +8,14 @@ use alloc::{boxed::Box, string::String}; use bytemuck::Zeroable; use futures_util::{task::AtomicWaker, Future}; use kernel_util::{ - block, mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, runtime::QueueWaker, sync::IrqSafeSpinlock, util::OneTimeInit, }; use tock_registers::interfaces::{Readable, Writeable}; -use ygg_driver_block::{cache::BlockCache, BlockDevice}; -use yggdrasil_abi::{error::Error, io::DeviceRequest}; +use ygg_driver_block::{IoOperation, IoRequest, IoSubmissionId, NgBlockDevice}; +use yggdrasil_abi::error::Error; use crate::{ command::{AtaCommand, AtaIdentify, AtaReadDmaEx}, @@ -53,7 +51,6 @@ pub struct AhciPort { ty: PortType, pub(crate) index: usize, info: OneTimeInit, - cache: IrqSafeSpinlock>, command_allocation: IrqSafeSpinlock, // One command index can only be waited for by one task, so this approach is usable @@ -140,7 +137,6 @@ impl AhciPort { inner: IrqSafeSpinlock::new(inner), ty: PortType::Sata, info: OneTimeInit::new(), - cache: IrqSafeSpinlock::new(BlockCache::new(SECTOR_SIZE)), ahci, index, @@ -186,15 +182,6 @@ impl AhciPort { Ok(unsafe { command.into_response() }) } - async fn read_block( - &self, - lba: u64, - block: &mut PageBox<[MaybeUninit]>, - ) -> Result<(), AhciError> { - log::debug!("read_block port={}, lba={}", self.index, lba); - self.perform_command(AtaReadDmaEx::new(lba, block)).await - } - fn allocate_command(&self) -> impl Future> + '_ { struct F<'f> { waker: &'f QueueWaker, @@ -297,51 +284,120 @@ impl AhciPort { } } -impl BlockDevice for AhciPort { - fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { - let info = self.info.try_get().ok_or(Error::PermissionDenied)?; +impl NgBlockDevice for AhciPort { + type CompletionNotify = AtomicWaker; - let mut cache = self.cache.lock(); - let mut rem = buf.len(); - let mut off = 0; + fn bus_id(&self) -> u32 { + 0 + } - while rem != 0 { - let lba = pos / SECTOR_SIZE as u64; + fn unit_id(&self) -> u32 { + self.index as u32 + } - if lba >= info.lba_count { - break; + fn block_size(&self) -> u64 { + SECTOR_SIZE as _ + } + + fn block_count(&self) -> u64 { + self.info.get().lba_count + } + + fn max_blocks_per_request(&self) -> u64 { + // TODO + 1 + } + + async fn submit_request(&self, request: IoRequest<'_>) -> Result { + // TODO better error handling + let slot = self.allocate_command().await.unwrap(); + log::trace!( + "Submit command on port {}, cmd index = {}", + self.index, + slot + ); + + match request.operation { + IoOperation::Read { lba, count } => { + self.inner + .lock() + .submit_command(slot, &AtaReadDmaEx::new(lba, count, request.data)) + .unwrap(); } - - let block_offset = (pos % SECTOR_SIZE as u64) as usize; - let count = core::cmp::min(SECTOR_SIZE - block_offset, rem); - - let block = cache.get_or_fetch_with(lba, |block| { - block! { - self.read_block(lba, block).await - }? - .map_err(|_| Error::InvalidOperation) - })?; - - buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); - - rem -= count; - off += count; - pos += count as u64; + IoOperation::Write { .. } => todo!(), } - Ok(off) + Ok(IoSubmissionId { + queue_id: self.index, + command_id: slot, + }) } - fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { - todo!() + fn poll_completion(&self, id: IoSubmissionId) -> Poll> { + let (_, status) = &self.command_completion[id.command_id]; + + match status.load(Ordering::Acquire) { + 0 => Poll::Pending, + 1 => { + self.free_command(id.command_id); + log::debug!("COMMAND FINISHED"); + Poll::Ready(Ok(())) + } + _ => todo!(), // Poll::Ready(Err(AhciError::DeviceError)), + } } - fn size(&self) -> Result { - let info = self.info.try_get().ok_or(Error::PermissionDenied)?; - Ok(info.lba_count * SECTOR_SIZE as u64) - } - - fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { - todo!() + fn completion_notify(&self, id: IoSubmissionId) -> &Self::CompletionNotify { + let (notify, _) = &self.command_completion[id.command_id]; + notify } } + +// impl BlockDevice for AhciPort { +// fn read(&'static self, mut pos: u64, buf: &mut [u8]) -> Result { +// let info = self.info.try_get().ok_or(Error::PermissionDenied)?; +// +// let mut cache = self.cache.lock(); +// let mut rem = buf.len(); +// let mut off = 0; +// +// while rem != 0 { +// let lba = pos / SECTOR_SIZE as u64; +// +// if lba >= info.lba_count { +// break; +// } +// +// let block_offset = (pos % SECTOR_SIZE as u64) as usize; +// let count = core::cmp::min(SECTOR_SIZE - block_offset, rem); +// +// let block = cache.get_or_fetch_with(lba, |block| { +// block! { +// self.read_block(lba, block).await +// }? +// .map_err(|_| Error::InvalidOperation) +// })?; +// +// buf[off..off + count].copy_from_slice(&block[block_offset..block_offset + count]); +// +// rem -= count; +// off += count; +// pos += count as u64; +// } +// +// Ok(off) +// } +// +// fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { +// todo!() +// } +// +// fn size(&self) -> Result { +// let info = self.info.try_get().ok_or(Error::PermissionDenied)?; +// Ok(info.lba_count * SECTOR_SIZE as u64) +// } +// +// fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { +// todo!() +// } +// } diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index 71e3b86e..ba6106c0 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -5,7 +5,7 @@ use core::{ }; use alloc::boxed::Box; -use futures_util::Future; +use futures_util::{task::AtomicWaker, Future}; use kernel_util::{block, mem::PageBox, runtime::QueueWaker}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; @@ -14,7 +14,17 @@ use crate::{ BlockDevice, }; +pub trait CompletionNotify { + fn wait_for_completion<'a, D: NgBlockDevice + 'a>( + &'a self, + device: &'a D, + id: IoSubmissionId, + ) -> impl Future> + Send + '_; +} + pub trait NgBlockDevice: Sync { + type CompletionNotify: CompletionNotify; + fn bus_id(&self) -> u32; // HBA, controller ID, etc. fn unit_id(&self) -> u32; // Drive, slot, connector ID, etc. @@ -28,7 +38,7 @@ pub trait NgBlockDevice: Sync { ) -> impl Future> + Send; fn poll_completion(&self, id: IoSubmissionId) -> Poll>; - fn completion_notify(&self, id: IoSubmissionId) -> &QueueWaker; + fn completion_notify(&self, id: IoSubmissionId) -> &Self::CompletionNotify; fn wait_for_completion( &self, @@ -37,32 +47,7 @@ pub trait NgBlockDevice: Sync { where Self: Sized, { - struct F<'f, D: NgBlockDevice + 'f> { - device: &'f D, - notify: &'f QueueWaker, - id: IoSubmissionId, - } - - impl<'f, D: NgBlockDevice + 'f> Future for F<'f, D> { - type Output = Result<(), Error>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.notify.register(cx.waker()); - match self.device.poll_completion(self.id) { - Poll::Ready(result) => { - self.notify.remove(cx.waker()); - Poll::Ready(result) - } - Poll::Pending => Poll::Pending, - } - } - } - - F { - device: self, - notify: self.completion_notify(id), - id, - } + self.completion_notify(id).wait_for_completion(self, id) } } @@ -93,6 +78,73 @@ struct BlockChunkIter { max_blocks_per_request: u64, } +impl CompletionNotify for QueueWaker { + fn wait_for_completion<'a, D: NgBlockDevice + 'a>( + &'a self, + device: &'a D, + id: IoSubmissionId, + ) -> impl Future> + Send + '_ { + struct F<'f, D: NgBlockDevice + 'f> { + device: &'f D, + notify: &'f QueueWaker, + id: IoSubmissionId, + } + + impl<'f, D: NgBlockDevice + 'f> Future for F<'f, D> { + type Output = Result<(), Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.notify.register(cx.waker()); + match self.device.poll_completion(self.id) { + Poll::Ready(result) => { + self.notify.remove(cx.waker()); + Poll::Ready(result) + } + Poll::Pending => Poll::Pending, + } + } + } + + F { + notify: self, + device, + id, + } + } +} + +impl CompletionNotify for AtomicWaker { + fn wait_for_completion<'a, D: NgBlockDevice + 'a>( + &'a self, + device: &'a D, + id: IoSubmissionId, + ) -> impl Future> + Send + '_ { + struct F<'f, D: NgBlockDevice + 'f> { + device: &'f D, + notify: &'f AtomicWaker, + id: IoSubmissionId, + } + + impl<'f, D: NgBlockDevice + 'f> Future for F<'f, D> { + type Output = Result<(), Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if let Poll::Ready(result) = self.device.poll_completion(self.id) { + return Poll::Ready(result); + } + self.notify.register(cx.waker()); + self.device.poll_completion(self.id) + } + } + + F { + notify: self, + device, + id, + } + } +} + impl BlockChunk { pub fn block_range(&self) -> Range { self.lba_offset..self.lba_offset + self.byte_count diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index d0f1cfa3..37fe394a 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -51,7 +51,7 @@ impl NvmeDrive { let blk = NgBlockDeviceWrapper::new(dev); devfs::add_named_block_device(blk, node_name.clone()).ok(); probe_partitions(blk, move |index, partition| { - devfs::add_block_device_partition(node_name.clone(), index, partition) + devfs::add_block_device_partition(format!("{}p", node_name), index, partition) }) .ok(); @@ -60,6 +60,8 @@ impl NvmeDrive { } impl NgBlockDevice for NvmeDrive { + type CompletionNotify = QueueWaker; + fn bus_id(&self) -> u32 { (*self.controller.controller_id.get()) as _ } diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 6b348da4..1adc7203 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -60,7 +60,8 @@ pub fn add_block_device_partition>( index: usize, partition: &'static dyn BlockDevice, ) -> Result<(), Error> { - let name = format!("{}p{}", base_name.into(), index + 1); + let base_name = base_name.into(); + let name = format!("{}{}", base_name, index + 1); log::info!("Add partition: {}", name); let node = Node::block(partition, NodeFlags::IN_MEMORY_PROPS); From b4c535ef936413c05265fa1a3a81f9c693cabaa9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 18 Dec 2023 23:26:51 +0200 Subject: [PATCH 129/211] readme: add ahci --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 779207b5..72fe1ac9 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ x86_64-specific: (no plans for legacy boot) * PCIe, with plans to extend to aarch64 as well * NVMe drive support (read/write) + * AHCI SATA drive support (read/write) * I/O and Local APIC IRQ controllers * PS/2 keyboard, * i8253-based timer (got some problems with HPET on From b763e391214c8502d91962b94d86c5be544ff904 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 22 Dec 2023 11:24:47 +0200 Subject: [PATCH 130/211] fs/poll: FdPoll WIP impl --- driver/block/core/src/device.rs | 65 +++++--- driver/block/core/src/lib.rs | 105 +++++++----- lib/kernel-util/src/api.rs | 1 + lib/kernel-util/src/runtime/executor.rs | 24 +-- lib/kernel-util/src/sync/fence.rs | 44 +++++ lib/kernel-util/src/sync/guard.rs | 19 +++ lib/kernel-util/src/sync/mod.rs | 36 +++++ lib/kernel-util/src/sync/mutex.rs | 133 +++++++++++++++ .../src/{sync.rs => sync/spinlock.rs} | 79 +-------- lib/vfs/src/device.rs | 9 +- lib/vfs/src/file/device.rs | 18 ++- lib/vfs/src/file/mod.rs | 117 +++++++++++++- lib/vfs/src/lib.rs | 6 +- lib/vfs/src/poll.rs | 76 +++++++++ lib/vfs/src/traits.rs | 12 ++ src/device/tty.rs | 151 ++++++++++++++++-- src/fs/mod.rs | 1 + src/fs/poll.rs | 14 ++ src/init.rs | 7 +- src/proc/io.rs | 53 +----- src/syscall/mod.rs | 87 ++++++++-- src/task/runtime/executor.rs | 65 -------- src/task/runtime/macros.rs | 35 ---- src/task/runtime/mod.rs | 58 ------- src/task/runtime/task.rs | 45 ------ src/task/runtime/task_queue.rs | 74 --------- src/task/runtime/waker.rs | 69 -------- src/task/thread.rs | 24 +-- src/util/ring.rs | 146 ++++++++++------- 29 files changed, 894 insertions(+), 679 deletions(-) create mode 100644 lib/kernel-util/src/sync/fence.rs create mode 100644 lib/kernel-util/src/sync/guard.rs create mode 100644 lib/kernel-util/src/sync/mod.rs create mode 100644 lib/kernel-util/src/sync/mutex.rs rename lib/kernel-util/src/{sync.rs => sync/spinlock.rs} (63%) create mode 100644 lib/vfs/src/poll.rs create mode 100644 src/fs/poll.rs delete mode 100644 src/task/runtime/executor.rs delete mode 100644 src/task/runtime/macros.rs delete mode 100644 src/task/runtime/mod.rs delete mode 100644 src/task/runtime/task.rs delete mode 100644 src/task/runtime/task_queue.rs delete mode 100644 src/task/runtime/waker.rs diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index ba6106c0..3efdcac9 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -236,35 +236,48 @@ impl<'a, D: NgBlockDevice + 'a> NgBlockDeviceWrapper<'a, D> { } impl<'a, D: NgBlockDevice + 'a> BlockDevice for NgBlockDeviceWrapper<'a, D> { - fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { - // TODO block cache - block! { - let mut bytes_read = 0; - - for chunk in - BlockChunkIter::new(pos, buf.len(), self.block_size, self.max_blocks_per_request) - { - log::debug!( - "Read chunk: lba_start={}, lba_count={}", - chunk.lba_start, - chunk.lba_count - ); - - let block = self.read_range_inner(chunk.lba_start, chunk.lba_count).await?; - - buf[chunk.buffer_range()].copy_from_slice(&block[chunk.block_range()]); - - bytes_read += chunk.byte_count; - } - - Ok(bytes_read) - }? - } - - fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { + fn poll_read( + &self, + cx: &mut Context<'_>, + pos: u64, + buf: &mut [u8], + ) -> Poll> { todo!() } + fn poll_write(&self, cx: &mut Context<'_>, pos: u64, buf: &[u8]) -> Poll> { + todo!() + } + + // fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { + // // TODO block cache + // block! { + // let mut bytes_read = 0; + + // for chunk in + // BlockChunkIter::new(pos, buf.len(), self.block_size, self.max_blocks_per_request) + // { + // log::debug!( + // "Read chunk: lba_start={}, lba_count={}", + // chunk.lba_start, + // chunk.lba_count + // ); + + // let block = self.read_range_inner(chunk.lba_start, chunk.lba_count).await?; + + // buf[chunk.buffer_range()].copy_from_slice(&block[chunk.block_range()]); + + // bytes_read += chunk.byte_count; + // } + + // Ok(bytes_read) + // }? + // } + + // fn write(&'static self, _pos: u64, _buf: &[u8]) -> Result { + // todo!() + // } + fn size(&self) -> Result { Ok(self.block_size * self.block_count) } diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index cca39db7..a8b1264f 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -2,18 +2,23 @@ extern crate alloc; +use core::{ + pin::Pin, + task::{Context, Poll}, +}; + use alloc::{boxed::Box, vec::Vec}; use kernel_util::runtime; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; -mod partition; +// mod partition; pub mod request; pub use device::{NgBlockDevice, NgBlockDeviceWrapper}; pub use request::{IoOperation, IoRequest, IoSubmissionId}; -use crate::partition::Partition; +// use crate::partition::Partition; pub fn probe_partitions< D: NgBlockDevice + 'static, @@ -22,50 +27,64 @@ pub fn probe_partitions< dev: &'static NgBlockDeviceWrapper, callback: F, ) -> Result<(), Error> { - async fn probe_table( - dev: &'static NgBlockDeviceWrapper<'static, D>, - ) -> Result>>, Error> { - if let Some(partitions) = partition::probe_gpt(dev)? { - return Ok(Some(partitions)); - } + Ok(()) + // async fn probe_table( + // dev: &'static NgBlockDeviceWrapper<'static, D>, + // ) -> Result>>, Error> { + // if let Some(partitions) = partition::probe_gpt(dev)? { + // return Ok(Some(partitions)); + // } - Ok(None) - } + // Ok(None) + // } - runtime::spawn(async move { - match probe_table(dev).await { - Ok(Some(partitions)) => { - // Create block devices for the partitions - for (i, partition) in partitions.into_iter().enumerate() { - let partition_blkdev = Box::leak(Box::new(partition)); + // runtime::spawn(async move { + // match probe_table(dev).await { + // Ok(Some(partitions)) => { + // // Create block devices for the partitions + // for (i, partition) in partitions.into_iter().enumerate() { + // let partition_blkdev = Box::leak(Box::new(partition)); - if let Err(error) = callback(i, partition_blkdev) { - log::warn!("Could not add partition {}: {:?}", i, error); - } - } - } - Ok(None) => { - log::warn!("Unknown or missing partition table"); - } - Err(error) => { - log::warn!("Could not probe partition table: {:?}", error); - } - } - }) + // if let Err(error) = callback(i, partition_blkdev) { + // log::warn!("Could not add partition {}: {:?}", i, error); + // } + // } + // } + // Ok(None) => { + // log::warn!("Unknown or missing partition table"); + // } + // Err(error) => { + // log::warn!("Could not probe partition table: {:?}", error); + // } + // } + // }) } /// Block device interface #[allow(unused)] pub trait BlockDevice: Sync { - /// Reads data frmo the given offset of the device - fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { - Err(Error::NotImplemented) + fn poll_read( + &self, + cx: &mut Context<'_>, + pos: u64, + buf: &mut [u8], + ) -> Poll> { + Poll::Ready(Err(Error::NotImplemented)) } - /// Writes the data to the given offset of the device - fn write(&'static self, pos: u64, buf: &[u8]) -> Result { - Err(Error::NotImplemented) + + fn poll_write(&self, cx: &mut Context<'_>, pos: u64, buf: &[u8]) -> Poll> { + Poll::Ready(Err(Error::NotImplemented)) } + // /// Reads data frmo the given offset of the device + // fn read(&'static self, pos: u64, buf: &mut [u8]) -> Result { + // Err(Error::NotImplemented) + // } + // /// Writes the data to the given offset of the device + // fn write(&'static self, pos: u64, buf: &[u8]) -> Result { + // Err(Error::NotImplemented) + // } + /// Returns the size of the block device in bytes fn size(&self) -> Result { Err(Error::NotImplemented) @@ -85,12 +104,12 @@ pub trait BlockDevice: Sync { Err(Error::NotImplemented) } - fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> { - let count = self.read(pos, buf)?; - if count == buf.len() { - Ok(()) - } else { - Err(Error::MissingData) - } - } + // fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> { + // let count = self.read(pos, buf)?; + // if count == buf.len() { + // Ok(()) + // } else { + // Err(Error::MissingData) + // } + // } } diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index 6771f96b..2b5acb2b 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -41,6 +41,7 @@ extern "Rust" { pub fn __current_thread() -> CurrentThread; pub fn __suspend_current(t: &CurrentThread) -> Result<(), Error>; pub fn __exit_current(t: &CurrentThread, code: ExitCode) -> !; + pub fn __yield(); pub fn __monotonic_timestamp() -> Result; diff --git a/lib/kernel-util/src/runtime/executor.rs b/lib/kernel-util/src/runtime/executor.rs index cd3295d1..f0d6c06e 100644 --- a/lib/kernel-util/src/runtime/executor.rs +++ b/lib/kernel-util/src/runtime/executor.rs @@ -4,7 +4,7 @@ use alloc::{boxed::Box, format, sync::Arc}; use futures_util::{task::waker_ref, Future}; use yggdrasil_abi::error::Error; -use crate::thread::Thread; +use crate::thread::{CurrentThread, Thread}; use super::{ task::{Task, Termination}, @@ -20,18 +20,16 @@ pub fn enqueue(task: Arc) -> Result<(), Error> { pub fn spawn_async_worker(index: usize) -> Result<(), Error> { let name = format!("[async-worker-{}]", index); - Thread::spawn(name, move || { - loop { - let task = task_queue::pop_task().unwrap(); // queue.block_pop().unwrap(); - let mut future_slot = task.future.lock(); + Thread::spawn(name, move || loop { + let task = task_queue::pop_task().unwrap(); + let mut future_slot = task.future.lock(); - if let Some(mut future) = future_slot.take() { - let waker = waker_ref(&task); - let context = &mut Context::from_waker(&waker); + if let Some(mut future) = future_slot.take() { + let waker = waker_ref(&task); + let context = &mut Context::from_waker(&waker); - if future.as_mut().poll(context).is_pending() { - *future_slot = Some(future); - } + if future.as_mut().poll(context).is_pending() { + *future_slot = Some(future); } } }) @@ -44,12 +42,14 @@ pub fn spawn + Send + 'static>( enqueue(Task::new(future)) } +struct CurrentThreadWaker(CurrentThread); + /// Runs a [Future] to its completion on the current thread pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { - let thread = Thread::current(); let mut future = Box::pin(future); loop { + let thread = Thread::current(); let waker = waker_ref(&thread); let context = &mut Context::from_waker(&waker); diff --git a/lib/kernel-util/src/sync/fence.rs b/lib/kernel-util/src/sync/fence.rs new file mode 100644 index 00000000..876690a3 --- /dev/null +++ b/lib/kernel-util/src/sync/fence.rs @@ -0,0 +1,44 @@ +use core::sync::atomic::{AtomicUsize, Ordering}; + +/// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is +/// met. +pub struct SpinFence { + value: AtomicUsize, +} + +// SpinFence impls +impl SpinFence { + /// Constructs a new [SpinFence] + pub const fn new() -> Self { + Self { + value: AtomicUsize::new(0), + } + } + + /// Resets a fence back to its original state + pub fn reset(&self) { + self.value.store(0, Ordering::Release); + } + + /// "Signals" a fence, incrementing its internal counter by one + pub fn signal(&self) { + self.value.fetch_add(1, Ordering::SeqCst); + } + + /// Waits until the fence is signalled at least the amount of times specified + pub fn wait_all(&self, count: usize) { + while self.value.load(Ordering::Acquire) < count { + core::hint::spin_loop(); + } + } + + /// Waits until the fence is signalled at least once + pub fn wait_one(&self) { + self.wait_all(1); + } + + /// Returns `true` if the fence has been signalled at least the amount of times specified + pub fn try_wait_all(&self, count: usize) -> bool { + self.value.load(Ordering::Acquire) >= count + } +} diff --git a/lib/kernel-util/src/sync/guard.rs b/lib/kernel-util/src/sync/guard.rs new file mode 100644 index 00000000..d2db9036 --- /dev/null +++ b/lib/kernel-util/src/sync/guard.rs @@ -0,0 +1,19 @@ +use crate::api; + +/// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation +/// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. +pub struct IrqGuard(bool); + +// IrqGuard impls +impl IrqGuard { + /// Saves the current IRQ state and masks them + pub fn acquire() -> Self { + Self(unsafe { api::__acquire_irq_guard() }) + } +} + +impl Drop for IrqGuard { + fn drop(&mut self) { + unsafe { api::__release_irq_guard(self.0) } + } +} diff --git a/lib/kernel-util/src/sync/mod.rs b/lib/kernel-util/src/sync/mod.rs new file mode 100644 index 00000000..242aaa84 --- /dev/null +++ b/lib/kernel-util/src/sync/mod.rs @@ -0,0 +1,36 @@ +//! Synchronization primitives +use core::sync::atomic::{AtomicBool, Ordering}; + +use yggdrasil_abi::error::Error; + +pub mod fence; +pub mod guard; +pub mod mutex; +pub mod spinlock; + +pub use fence::SpinFence; +pub use guard::IrqGuard; +pub use spinlock::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; + +// use crate::arch::{Architecture, ArchitectureImpl}; + +static LOCK_HACK: AtomicBool = AtomicBool::new(false); + +pub trait LockMethod<'q>: Sync { + type Guard<'a> + where + 'a: 'q, + Self: 'a; + + fn lock(&'q self) -> Result, Error>; + unsafe fn release(&self); +} + +/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. +/// +/// # Safety +/// +/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. +pub unsafe fn hack_locks() { + LOCK_HACK.store(true, Ordering::Release); +} diff --git a/lib/kernel-util/src/sync/mutex.rs b/lib/kernel-util/src/sync/mutex.rs new file mode 100644 index 00000000..2a97a404 --- /dev/null +++ b/lib/kernel-util/src/sync/mutex.rs @@ -0,0 +1,133 @@ +use core::{ + cell::UnsafeCell, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicU32, Ordering}, +}; + +use alloc::sync::Arc; +use crossbeam_queue::ArrayQueue; +use yggdrasil_abi::error::Error; + +use crate::{api, thread::Thread}; + +use super::LockMethod; + +struct ThreadedMutexInner { + queue: ArrayQueue>, + lock: AtomicU32, +} + +pub struct Mutex { + value: UnsafeCell, + lock: ThreadedMutexInner, +} + +pub struct MutexGuard<'a, T> { + value: &'a UnsafeCell, + lock: &'a ThreadedMutexInner, +} + +impl ThreadedMutexInner { + const UNLOCKED: u32 = 0; + const LOCKED: u32 = 1; + + pub fn new(queue_capacity: usize) -> Self { + Self { + queue: ArrayQueue::new(queue_capacity), + lock: AtomicU32::new(Self::UNLOCKED), + } + } + + fn try_lock(&self) -> bool { + self.lock + .compare_exchange( + Self::UNLOCKED, + Self::LOCKED, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_ok() + } +} + +impl<'q> LockMethod<'q> for ThreadedMutexInner { + type Guard<'a> = () where 'a: 'q, Self: 'a; + + fn lock(&self) -> Result<(), Error> { + // Fast-path + + loop { + if self.try_lock() { + return Ok(()); + } + + let thread = Thread::current(); + + self.queue + .push(thread.clone()) + .map_err(|_| Error::WouldBlock)?; + + thread.suspend()?; + } + } + + unsafe fn release(&self) { + if self.lock.swap(Self::UNLOCKED, Ordering::Release) == Self::LOCKED { + if let Some(t) = self.queue.pop() { + t.enqueue(); + + // Yield current thread to avoid congestion + unsafe { + api::__yield(); + } + } + } + } +} + +impl Mutex { + pub fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + lock: ThreadedMutexInner::new(32), + } + } +} + +impl<'q, T> LockMethod<'q> for Mutex { + type Guard<'a> = MutexGuard<'a, T> where 'a: 'q, Self: 'a; + + fn lock(&'q self) -> Result, Error> { + self.lock.lock()?; + Ok(MutexGuard { + lock: &self.lock, + value: &self.value, + }) + } + + unsafe fn release(&self) { + self.lock.release(); + } +} + +unsafe impl Sync for Mutex {} + +impl<'a, T> Deref for MutexGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.value.get() } + } +} + +impl<'a, T> DerefMut for MutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.value.get() } + } +} + +impl<'a, T> Drop for MutexGuard<'a, T> { + fn drop(&mut self) { + unsafe { self.lock.release() } + } +} diff --git a/lib/kernel-util/src/sync.rs b/lib/kernel-util/src/sync/spinlock.rs similarity index 63% rename from lib/kernel-util/src/sync.rs rename to lib/kernel-util/src/sync/spinlock.rs index ceb8cce3..ad1bcd12 100644 --- a/lib/kernel-util/src/sync.rs +++ b/lib/kernel-util/src/sync/spinlock.rs @@ -1,35 +1,11 @@ -//! Synchronization primitives use core::{ cell::UnsafeCell, mem, ops::{Deref, DerefMut}, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + sync::atomic::{AtomicBool, Ordering}, }; -use crate::api; - -// use crate::arch::{Architecture, ArchitectureImpl}; - -static LOCK_HACK: AtomicBool = AtomicBool::new(false); - -/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. -/// -/// # Safety -/// -/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. -pub unsafe fn hack_locks() { - LOCK_HACK.store(true, Ordering::Release); -} - -/// Simple spinloop-based fence guaranteeing that the execution resumes only after its condition is -/// met. -pub struct SpinFence { - value: AtomicUsize, -} - -/// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation -/// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. -pub struct IrqGuard(bool); +use super::{IrqGuard, LOCK_HACK}; struct SpinlockInner { value: UnsafeCell, @@ -174,54 +150,3 @@ impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { self.inner.deref_mut() } } - -// IrqGuard impls -impl IrqGuard { - /// Saves the current IRQ state and masks them - pub fn acquire() -> Self { - Self(unsafe { api::__acquire_irq_guard() }) - } -} - -impl Drop for IrqGuard { - fn drop(&mut self) { - unsafe { api::__release_irq_guard(self.0) } - } -} - -// SpinFence impls -impl SpinFence { - /// Constructs a new [SpinFence] - pub const fn new() -> Self { - Self { - value: AtomicUsize::new(0), - } - } - - /// Resets a fence back to its original state - pub fn reset(&self) { - self.value.store(0, Ordering::Release); - } - - /// "Signals" a fence, incrementing its internal counter by one - pub fn signal(&self) { - self.value.fetch_add(1, Ordering::SeqCst); - } - - /// Waits until the fence is signalled at least the amount of times specified - pub fn wait_all(&self, count: usize) { - while self.value.load(Ordering::Acquire) < count { - core::hint::spin_loop(); - } - } - - /// Waits until the fence is signalled at least once - pub fn wait_one(&self) { - self.wait_all(1); - } - - /// Returns `true` if the fence has been signalled at least the amount of times specified - pub fn try_wait_all(&self, count: usize) -> bool { - self.value.load(Ordering::Acquire) >= count - } -} diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs index 1015ed99..f940300d 100644 --- a/lib/vfs/src/device.rs +++ b/lib/vfs/src/device.rs @@ -1,11 +1,16 @@ +use futures_util::Future; +use kernel_util::runtime::QueueWaker; use ygg_driver_block::BlockDevice; use yggdrasil_abi::{error::Error, io::DeviceRequest}; -use crate::node::{CommonImpl, NodeRef}; +use crate::{ + node::{CommonImpl, NodeRef}, + traits::FileReadiness, +}; /// Character device interface #[allow(unused)] -pub trait CharDevice: Sync { +pub trait CharDevice: FileReadiness + Sync { /// Reads data from the device fn read(&'static self, buf: &mut [u8]) -> Result { Err(Error::NotImplemented) diff --git a/lib/vfs/src/file/device.rs b/lib/vfs/src/file/device.rs index 40c22b55..af2a261b 100644 --- a/lib/vfs/src/file/device.rs +++ b/lib/vfs/src/file/device.rs @@ -23,17 +23,19 @@ pub struct CharFile { impl BlockFile { pub fn read(&self, buf: &mut [u8]) -> Result { - let mut position = self.position.lock(); - let count = self.device.0.read(*position, buf)?; - *position += count as u64; - Ok(count) + todo!() + // let mut position = self.position.lock(); + // let count = self.device.0.read(*position, buf)?; + // *position += count as u64; + // Ok(count) } pub fn write(&self, buf: &[u8]) -> Result { - let mut position = self.position.lock(); - let count = self.device.0.write(*position, buf)?; - *position += count as u64; - Ok(count) + todo!() + // let mut position = self.position.lock(); + // let count = self.device.0.write(*position, buf)?; + // *position += count as u64; + // Ok(count) } pub fn seek(&self, from: SeekFrom) -> Result { diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 6ff08c9e..85e9e009 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -1,16 +1,27 @@ -use core::{any::Any, fmt, mem::MaybeUninit}; +use core::{ + any::Any, + fmt, + mem::MaybeUninit, + task::{Context, Poll}, +}; -use alloc::{boxed::Box, sync::Arc}; -use kernel_util::sync::IrqSafeSpinlock; +use alloc::{ + boxed::Box, + collections::{btree_map::Entry, BTreeMap}, + sync::Arc, +}; +use futures_util::{future::BoxFuture, FutureExt}; +use kernel_util::sync::{mutex::Mutex, IrqSafeSpinlock}; use yggdrasil_abi::{ error::Error, - io::{DirectoryEntry, OpenOptions, SeekFrom}, + io::{DirectoryEntry, OpenOptions, RawFd, SeekFrom}, }; use crate::{ device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, traits::{Read, Seek, Write}, + FdPoll, }; use self::{ @@ -48,6 +59,12 @@ pub enum File { Block(BlockFile), Char(CharFile), AnonymousPipe(PipeEnd), + Poll(FdPoll), +} + +/// Contains a per-process fd -> FileRef map +pub struct FileSet { + map: BTreeMap, } impl File { @@ -60,6 +77,11 @@ impl File { ) } + /// Constructs a new poll channel file + pub fn new_poll_channel() -> Arc { + Arc::new(Self::Poll(FdPoll::new())) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -145,7 +167,25 @@ impl File { Self::Regular(file) => Some(&file.node), Self::Block(file) => Some(&file.node), Self::Char(file) => Some(&file.node), - Self::AnonymousPipe(_) => None, + Self::AnonymousPipe(_) | Self::Poll(_) => None, + } + } + + /// + pub fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + match self { + Self::Char(f) => f.device.0.poll_read(cx), + // Polling not implemented, return ready immediately (XXX ?) + _ => Poll::Ready(Ok(())), + } + } + + /// + pub fn as_poll_channel(&self) -> Result<&FdPoll, Error> { + if let Self::Poll(poll) = self { + Ok(poll) + } else { + Err(Error::InvalidOperation) } } } @@ -157,6 +197,8 @@ impl Read for File { Self::Block(file) => file.read(buf), Self::Char(file) => file.read(buf), Self::AnonymousPipe(pipe) => pipe.read(buf), + // TODO maybe allow reading FDs from poll channels as if they were regular streams? + Self::Poll(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -169,6 +211,8 @@ impl Write for File { Self::Block(file) => file.write(buf), Self::Char(file) => file.write(buf), Self::AnonymousPipe(pipe) => pipe.write(buf), + // TODO maybe allow adding FDs to poll channels this way + Self::Poll(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -179,7 +223,7 @@ impl Seek for File { match self { Self::Regular(file) => Ok(*file.position.lock()), Self::Block(file) => Ok(*file.position.lock()), - Self::Char(_) | Self::AnonymousPipe(_) => Err(Error::InvalidOperation), + Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -188,7 +232,7 @@ impl Seek for File { match self { Self::Regular(file) => file.seek(from), Self::Block(file) => file.seek(from), - Self::Char(_) | Self::AnonymousPipe(_) => Err(Error::InvalidOperation), + Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -216,10 +260,69 @@ impl fmt::Debug for File { .finish_non_exhaustive(), Self::Directory(_) => f.debug_struct("DirectoryFile").finish_non_exhaustive(), Self::AnonymousPipe(_) => f.debug_struct("AnonymousPipe").finish_non_exhaustive(), + Self::Poll(_) => f.debug_struct("Poll").finish_non_exhaustive(), } } } +impl FileSet { + /// Creates an empty [FileSet] + pub fn new() -> Self { + Self { + map: BTreeMap::new(), + } + } + + /// Returns the [FileRef] associated with given `fd` or an error if it does not exist + pub fn file(&self, fd: RawFd) -> Result<&FileRef, Error> { + self.map.get(&fd).ok_or(Error::InvalidFile) + } + + /// Associates a `file` with `fd`, returning an error if requested `fd` is already taken + pub fn set_file(&mut self, fd: RawFd, file: FileRef) -> Result<(), Error> { + if self.map.contains_key(&fd) { + return Err(Error::AlreadyExists); + } + self.map.insert(fd, file); + Ok(()) + } + + /// Associates a `file` with any available [RawFd] and returns it + pub fn place_file(&mut self, file: FileRef) -> Result { + for idx in 0..64 { + let fd = RawFd::from(idx); + + if let Entry::Vacant(e) = self.map.entry(fd) { + e.insert(file); + return Ok(fd); + } + } + + // TODO OutOfFiles + Err(Error::OutOfMemory) + } + + /// Removes and closes a [FileRef] from the struct + pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { + // Do nothing, file will be dropped and closed + if self.map.remove(&fd).is_some() { + Ok(()) + } else { + Err(Error::InvalidFile) + } + } + + /// Removes all [FileRef]s from the struct which do not pass the `predicate` check + pub fn retain bool>(&mut self, predicate: F) { + self.map.retain(predicate); + } + + /// Closes all of the files + pub fn close_all(&mut self) { + self.map.clear(); + } +} + #[cfg(test)] mod tests { use core::{mem::MaybeUninit, str::FromStr}; diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 8b836512..b4fed41c 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -15,13 +15,15 @@ pub(crate) mod file; pub(crate) mod ioctx; pub(crate) mod node; pub(crate) mod path; +pub(crate) mod poll; pub(crate) mod traits; pub use device::CharDevice; -pub use file::{DirectoryOpenPosition, File, FileRef, InstanceData}; +pub use file::{DirectoryOpenPosition, File, FileRef, FileSet, InstanceData}; pub use ioctx::{Action, IoContext}; pub use node::{ impls, AccessToken, CommonImpl, CreateInfo, DirectoryImpl, Metadata, Node, NodeFlags, NodeRef, RegularImpl, SymlinkImpl, }; -pub use traits::{Read, Seek, Write}; +pub use poll::FdPoll; +pub use traits::{FileReadiness, Read, Seek, Write}; diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs new file mode 100644 index 00000000..fe070ffa --- /dev/null +++ b/lib/vfs/src/poll.rs @@ -0,0 +1,76 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use alloc::{collections::BTreeMap, vec::Vec}; + +use futures_util::{future::BoxFuture, Future, FutureExt}; +use kernel_util::{ + runtime, + sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, +}; +use yggdrasil_abi::{error::Error, io::RawFd}; + +use crate::{FileRef, FileSet}; + +/// +pub struct FdPoll { + fds: Mutex>, +} + +struct FdPollFuture<'a> { + fds: &'a Mutex>, + timeout: Option>, +} + +impl FdPoll { + /// Creates a new poll channel + pub fn new() -> Self { + Self { + fds: Mutex::new(BTreeMap::new()), + } + } + + /// Adds a file descriptor of interest to the poll channel + pub fn add(&self, fd: RawFd, file: FileRef) { + self.fds.lock().unwrap().insert(fd, file); + } + + fn begin(&self, timeout: Option) -> FdPollFuture<'_> { + let timeout = timeout.map(|t| runtime::sleep(t).boxed()); + + FdPollFuture { + fds: &self.fds, + timeout, + } + } + + /// Polls the channel once, returning either a file descriptor or timeout + pub async fn poll_once(&self, timeout: Option) -> Option { + self.begin(timeout).await + } +} + +impl<'a> Future for FdPollFuture<'a> { + type Output = Option; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if let Some(timeout) = self.timeout.as_mut() + && timeout.as_mut().poll(cx).is_ready() + { + // Timeout + return Poll::Ready(None); + } + + for (&fd, file) in self.fds.lock().unwrap().iter() { + if file.poll_read(cx).is_ready() { + return Poll::Ready(Some(fd)); + } + } + + // Wait + Poll::Pending + } +} diff --git a/lib/vfs/src/traits.rs b/lib/vfs/src/traits.rs index 96f39940..73a5804b 100644 --- a/lib/vfs/src/traits.rs +++ b/lib/vfs/src/traits.rs @@ -1,3 +1,9 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, +}; + +use futures_util::Future; use yggdrasil_abi::{error::Error, io::SeekFrom}; /// Immutable read interface for VFS objects @@ -35,3 +41,9 @@ pub trait Seek { self.seek(SeekFrom::Current(0)) } } + +/// +pub trait FileReadiness: Sync { + /// + fn poll_read(&self, cx: &mut Context<'_>) -> Poll>; +} diff --git a/src/device/tty.rs b/src/device/tty.rs index dc2b5da9..b5ffc042 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -1,28 +1,122 @@ //! Terminal driver implementation +use core::{ + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, + task::{Context, Poll}, +}; + use abi::{ error::Error, io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, process::Signal, }; use device_api::serial::SerialDevice; -use kernel_util::sync::IrqSafeSpinlock; +use futures_util::Future; +use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; +use vfs::FileReadiness; use crate::{ task::process::{Process, ProcessId}, - util::ring::AsyncRing, + util::ring::RingBuffer, }; +/// +pub struct TerminalRing { + buffer: IrqSafeSpinlock>, + eof: AtomicBool, + notify: QueueWaker, +} + +impl TerminalRing { + const CAPACITY: usize = 128; + + /// + pub const fn new() -> Self { + Self { + buffer: IrqSafeSpinlock::new(RingBuffer::new(0)), + eof: AtomicBool::new(false), + notify: QueueWaker::new(), + } + } + + /// + pub fn write(&self, ch: u8, signal: bool) { + self.buffer.lock().write(ch); + if signal { + self.notify.wake_one(); + } + } + + /// + pub fn signal(&self) { + self.notify.wake_all(); + } + + /// + pub fn signal_eof(&self) { + self.eof.store(true, Ordering::Release); + self.notify.wake_all(); + } + + /// + pub fn read_blocking<'a>(&'a self) -> impl Future> + 'a { + struct F<'f> { + ring: &'f TerminalRing, + } + + impl<'f> Future for F<'f> { + type Output = Option; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.ring.notify.register(cx.waker()); + + if self + .ring + .eof + .compare_exchange(true, false, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + self.ring.notify.remove(cx.waker()); + return Poll::Ready(None); + } + + let mut lock = self.ring.buffer.lock(); + + if lock.is_readable() { + self.ring.notify.remove(cx.waker()); + Poll::Ready(Some(lock.read_single_unchecked())) + } else { + Poll::Pending + } + } + } + + F { ring: self } + } + + fn can_read_now(&self, canonical: bool) -> bool { + let lock = self.buffer.lock(); + if canonical { + lock.contains(b'\n') + } else { + lock.is_readable() + } + } +} + #[cfg(feature = "fb_console")] pub mod combined { //! Combined console + keyboard terminal device + use core::task::{Context, Poll}; + use crate::task::process::ProcessId; use abi::{ error::Error, - io::{DeviceRequest, TerminalSize}, + io::{DeviceRequest, TerminalLineOptions, TerminalSize}, }; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; - use kernel_util::block; - use vfs::CharDevice; + use kernel_util::{block, runtime::QueueWaker}; + use vfs::{CharDevice, FileReadiness}; // use vfs::CharDevice; use crate::device::{ @@ -109,6 +203,22 @@ pub mod combined { } } } + + impl FileReadiness for CombinedTerminal { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + let inner = self.context.inner.lock(); + let canonical = inner.config.line.contains(TerminalLineOptions::CANONICAL); + + self.context.ring.notify.register(cx.waker()); + + if self.context.ring.can_read_now(canonical) { + self.context.ring.notify.remove(cx.waker()); + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + } } #[cfg(feature = "fb_console")] @@ -121,7 +231,7 @@ struct TtyContextInner { /// Represents the context of a terminal device pub struct TtyContext { - ring: AsyncRing, + ring: TerminalRing, // AsyncRing, inner: IrqSafeSpinlock, } @@ -196,6 +306,8 @@ pub trait TtyDevice: SerialDevice { { let pgrp = inner.process_group; + cx.signal(); + if let Some(pgrp) = pgrp { drop(inner); Process::signal_group(pgrp, Signal::Interrupted); @@ -205,8 +317,10 @@ pub trait TtyDevice: SerialDevice { } } + let canonical = inner.config.line.contains(TerminalLineOptions::CANONICAL); + drop(inner); - cx.putc(byte); + cx.putc(byte, !canonical || byte == b'\n' || byte == b'\r'); } /// Reads and processes data from the terminal @@ -220,7 +334,9 @@ pub trait TtyDevice: SerialDevice { if !inner.config.is_canonical() { drop(inner); - let byte = cx.getc().await; + let Some(byte) = cx.getc().await else { + return Ok(0); + }; data[0] = byte; Ok(1) } else { @@ -230,7 +346,9 @@ pub trait TtyDevice: SerialDevice { // Run until either end of buffer or return condition is reached while rem != 0 { drop(inner); - let byte = cx.getc().await; + let Some(byte) = cx.getc().await else { + break; + }; inner = cx.inner.lock(); if inner.config.is_canonical() { @@ -284,7 +402,7 @@ impl TtyContext { /// Constructs a new [TtyContext] pub fn new() -> Self { Self { - ring: AsyncRing::new(0), + ring: TerminalRing::new(), // AsyncRing::new(0), inner: IrqSafeSpinlock::new(TtyContextInner { config: TerminalOptions::const_default(), process_group: None, @@ -292,14 +410,19 @@ impl TtyContext { } } + /// + pub fn signal(&self) { + self.ring.signal() + } + /// Writes a single character to the terminal - pub fn putc(&self, ch: u8) { - self.ring.try_write(ch).unwrap(); + pub fn putc(&self, ch: u8, signal: bool) { + self.ring.write(ch, signal); } /// Performs a blocking read of a single character from the terminal - pub async fn getc(&self) -> u8 { - self.ring.read().await + pub async fn getc(&self) -> Option { + self.ring.read_blocking().await } /// Changes the configuration of the terminal diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 22811030..2e4f26f6 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -11,6 +11,7 @@ use yggdrasil_abi::{error::Error, io::MountOptions}; use crate::{mem::phys, proc::random}; // pub mod devfs; +pub mod poll; pub mod sysfs; /// Describes in-memory filesystem image used as initial root diff --git a/src/fs/poll.rs b/src/fs/poll.rs new file mode 100644 index 00000000..800edb7b --- /dev/null +++ b/src/fs/poll.rs @@ -0,0 +1,14 @@ +//! + +use core::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use abi::{error::Error, io::RawFd}; +use alloc::vec::Vec; +use futures_util::{future::BoxFuture, Future, FutureExt}; +use kernel_util::{runtime, sync::IrqSafeSpinlock}; + +use crate::proc::io::ProcessIo; diff --git a/src/init.rs b/src/init.rs index dd41b086..bbc08f8c 100644 --- a/src/init.rs +++ b/src/init.rs @@ -1,4 +1,5 @@ //! Kernel main process implementation: filesystem initialization and userspace init start + use abi::{ error::Error, io::{OpenOptions, RawFd}, @@ -60,9 +61,9 @@ pub fn kinit() -> Result<(), Error> { let mut io = user_init.io.lock(); io.set_ioctx(ioctx); - io.set_file(RawFd::STDIN, stdin)?; - io.set_file(RawFd::STDOUT, stdout)?; - io.set_file(RawFd::STDERR, stderr)?; + io.files.set_file(RawFd::STDIN, stdin)?; + io.files.set_file(RawFd::STDOUT, stdout)?; + io.files.set_file(RawFd::STDERR, stderr)?; drop(io); user_init.set_session_terminal(console); diff --git a/src/proc/io.rs b/src/proc/io.rs index cc156865..52eb2528 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -2,13 +2,14 @@ use abi::{error::Error, io::RawFd}; use alloc::collections::{btree_map::Entry, BTreeMap}; -use vfs::{FileRef, IoContext}; +use vfs::{FileSet, IoContext}; /// I/O context of a process, contains information like root, current directory and file /// descriptor table pub struct ProcessIo { ioctx: Option, - files: BTreeMap, + /// Process' set of files + pub files: FileSet, } impl ProcessIo { @@ -16,15 +17,10 @@ impl ProcessIo { pub fn new() -> Self { Self { ioctx: None, - files: BTreeMap::new(), + files: FileSet::new(), } } - /// Returns the [FileRef] associated with given `fd` or an error if it does not exist - pub fn file(&self, fd: RawFd) -> Result<&FileRef, Error> { - self.files.get(&fd).ok_or(Error::InvalidFile) - } - /// Returns the associated [IoContext] reference pub fn ioctx(&mut self) -> &mut IoContext { self.ioctx.as_mut().unwrap() @@ -35,47 +31,8 @@ impl ProcessIo { self.ioctx = Some(ioctx); } - /// Associates a `file` with `fd`, returning an error if requested `fd` is already taken - pub fn set_file(&mut self, fd: RawFd, file: FileRef) -> Result<(), Error> { - if self.files.contains_key(&fd) { - return Err(Error::AlreadyExists); - } - self.files.insert(fd, file); - Ok(()) - } - - /// Associates a `file` with any available [RawFd] and returns it - pub fn place_file(&mut self, file: FileRef) -> Result { - for idx in 0..64 { - let fd = RawFd::from(idx); - - if let Entry::Vacant(e) = self.files.entry(fd) { - e.insert(file); - return Ok(fd); - } - } - - // TODO OutOfFiles - Err(Error::OutOfMemory) - } - - /// Removes and closes a [FileRef] from the struct - pub fn close_file(&mut self, fd: RawFd) -> Result<(), Error> { - // Do nothing, file will be dropped and closed - if self.files.remove(&fd).is_some() { - Ok(()) - } else { - Err(Error::InvalidFile) - } - } - - /// Removes all [FileRef]s from the struct which do not pass the `predicate` check - pub fn retain bool>(&mut self, predicate: F) { - self.files.retain(predicate); - } - /// Handles process exit by closing all of its files pub fn handle_exit(&mut self) { - self.files.clear(); + self.files.close_all(); } } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 98f25d35..1fe103fe 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -3,12 +3,18 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, - io::{DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, RawFd, SeekFrom}, + io::{ + DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, PollControl, RawFd, + SeekFrom, + }, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; use alloc::sync::Arc; -use kernel_util::{block, runtime, sync::IrqSafeSpinlockGuard}; +use kernel_util::{ + block, runtime, + sync::{IrqSafeSpinlockGuard, LockMethod}, +}; use vfs::{File, IoContext, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; @@ -42,7 +48,8 @@ fn run_with_io_at< let io = proc.io.lock(); let at = at .map(|fd| { - io.file(fd) + io.files + .file(fd) .and_then(|f| f.node().ok_or(Error::InvalidFile).cloned()) }) .transpose()?; @@ -140,7 +147,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result process.set_session_terminal(node.clone()); } - let fd = io.place_file(file)?; + let fd = io.files.place_file(file)?; Ok(fd.0 as usize) }) } @@ -152,7 +159,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let node = io.ioctx().find(at, path, true, true)?; let access = io.ioctx().check_access(vfs::Action::Read, &node)?; let file = node.open_directory(access)?; - let fd = io.place_file(file)?; + let fd = io.files.place_file(file)?; Ok(fd.0 as usize) }) @@ -161,19 +168,21 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - run_with_io(process, |io| io.file(fd)?.read(data)) + run_with_io(process, |io| io.files.file(fd)?.read(data)) } SyscallFunction::Write => { let fd = RawFd(args[0] as u32); let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - run_with_io(process, |io| io.file(fd)?.write(data)) + run_with_io(process, |io| io.files.file(fd)?.write(data)) } SyscallFunction::Seek => { let fd = RawFd(args[0] as u32); let pos = SeekFrom::from(args[1]); - run_with_io(process, |io| io.file(fd)?.seek(pos).map(|v| v as usize)) + run_with_io(process, |io| { + io.files.file(fd)?.seek(pos).map(|v| v as usize) + }) } SyscallFunction::ReadDirectory => { let fd = RawFd(args[0] as u32); @@ -182,13 +191,13 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result args[2] as usize, )?; - run_with_io(process, |io| io.file(fd)?.read_dir(buffer)) + run_with_io(process, |io| io.files.file(fd)?.read_dir(buffer)) } SyscallFunction::Close => { let fd = RawFd(args[0] as u32); run_with_io(process, |mut io| { - let res = io.close_file(fd); + let res = io.files.close_file(fd); match res { Err(Error::InvalidFile) => { @@ -219,7 +228,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let req = arg_user_mut::(args[1] as usize)?; run_with_io(process, |io| { - let file = io.file(fd)?; + let file = io.files.file(fd)?; let node = file.node().ok_or(Error::InvalidFile)?; node.device_request(req)?; Ok(0) @@ -278,8 +287,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io(process, |mut io| { let (read, write) = File::new_pipe_pair(256); - let read_fd = io.place_file(read)?; - let write_fd = io.place_file(write)?; + let read_fd = io.files.place_file(read)?; + let write_fd = io.files.place_file(write)?; infoln!("Read end: {:?}", read_fd); infoln!("Write end: {:?}", write_fd); @@ -290,6 +299,50 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) }) } + SyscallFunction::CreatePoll => run_with_io(process, |mut io| { + let poll = File::new_poll_channel(); + let fd = io.files.place_file(poll)?; + + Ok(fd.0 as usize) + }), + SyscallFunction::PollControl => { + let poll_fd = RawFd::from(args[0] as u32); + let control = + PollControl::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + let fd = RawFd::from(args[2] as u32); + + run_with_io(process, |io| { + let poll_file = io.files.file(poll_fd)?; + let poll = poll_file.as_poll_channel()?; + + match control { + PollControl::AddFd => { + let polled_file = io.files.file(fd)?.clone(); + poll.add(fd, polled_file); + } + } + + Ok(0) + }) + } + SyscallFunction::PollWait => { + let poll_fd = RawFd::from(args[0] as u32); + let timeout = arg_user_ref::>(args[1] as usize)?; + let output = arg_user_mut::>(args[2] as usize)?; + + debugln!("PollWait {:?}", timeout); + + run_with_io(process, |io| { + let poll_file = io.files.file(poll_fd)?; + let poll = poll_file.as_poll_channel()?; + + *output = block! { + poll.poll_once(*timeout).await + }?; + + Ok(0) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; @@ -316,8 +369,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result for opt in options.optional { match opt { &SpawnOption::InheritFile { source, child } => { - let src_file = io.file(source)?; - child_io.set_file(child, src_file.clone())?; + let src_file = io.files.file(source)?; + child_io.files.set_file(child, src_file.clone())?; } &SpawnOption::SetProcessGroup(pgroup) => { child_process.set_group_id(pgroup.into()); @@ -334,7 +387,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } }) { debugln!("{} requested terminal {:?}", pid, fd); - let file = child_io.file(fd)?; + let file = child_io.files.file(fd)?; let node = file.node().ok_or(Error::InvalidFile)?; let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); @@ -414,7 +467,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result if let Some(ctty) = session_terminal { // Drop all FDs referring to the old session terminal run_with_io(process, |mut io| { - io.retain(|_, f| { + io.files.retain(|_, f| { f.node() .map(|node| !Arc::ptr_eq(node, &ctty)) .unwrap_or(true) diff --git a/src/task/runtime/executor.rs b/src/task/runtime/executor.rs deleted file mode 100644 index 5dab913c..00000000 --- a/src/task/runtime/executor.rs +++ /dev/null @@ -1,65 +0,0 @@ -use core::task::{Context, Poll}; - -use abi::error::Error; -use alloc::{boxed::Box, format, sync::Arc}; -use futures_util::{task::waker_ref, Future}; - -use crate::task::{spawn_kernel_closure, thread::Thread}; - -use super::{ - task::{Task, Termination}, - task_queue::{self}, -}; - -/// Pushes a task into the executor's queue -pub fn enqueue(task: Arc) -> Result<(), Error> { - task_queue::push_task(task) -} - -/// Spawns a background worker to execute the tasks from the global queue -pub fn spawn_async_worker(index: usize) -> Result<(), Error> { - let name = format!("[async-worker-{}]", index); - - spawn_kernel_closure(name, move || { - loop { - let task = task_queue::pop_task().unwrap(); // queue.block_pop().unwrap(); - let mut future_slot = task.future.lock(); - - if let Some(mut future) = future_slot.take() { - let waker = waker_ref(&task); - let context = &mut Context::from_waker(&waker); - - if future.as_mut().poll(context).is_pending() { - *future_slot = Some(future); - } - } - } - }) -} - -/// Creates a new task for the [Future] and queues it for execution in background -pub fn spawn + Send + 'static>( - future: F, -) -> Result<(), Error> { - enqueue(Task::new(future)) -} - -/// Runs a [Future] to its completion on the current thread -pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { - let thread = Thread::current(); - let mut future = Box::pin(future); - - loop { - let waker = waker_ref(&thread); - let context = &mut Context::from_waker(&waker); - - match future.as_mut().poll(context) { - Poll::Ready(value) => break Ok(value), - Poll::Pending => { - if let Err(error) = thread.suspend() { - break Err(error); - } - } - } - } -} diff --git a/src/task/runtime/macros.rs b/src/task/runtime/macros.rs deleted file mode 100644 index cdb0db3b..00000000 --- a/src/task/runtime/macros.rs +++ /dev/null @@ -1,35 +0,0 @@ -/// Runs a [futures_util::Future] to its completion -#[macro_export] -macro_rules! block { - ($($stmt:tt)*) => { - $crate::task::runtime::run_to_completion(alloc::boxed::Box::pin(async move { - $($stmt)* - })) - }; -} - -/// Runs two Futures, returning the result of whichever finishes first -#[macro_export] -macro_rules! any { - ($fut0:ident = $pat0:pat => $body0:expr, $fut1:ident = $pat1:pat => $body1:expr) => { - use futures_util::Future; - - let mut pin0 = alloc::boxed::Box::pin($fut0); - let mut pin1 = alloc::boxed::Box::pin($fut1); - - $crate::task::runtime::poll_fn(move |cx| { - match (pin0.as_mut().poll(cx), pin1.as_mut().poll(cx)) { - (core::task::Poll::Ready($pat0), _) => { - $body0; - core::task::Poll::Ready(()) - } - (_, core::task::Poll::Ready($pat1)) => { - $body1; - core::task::Poll::Ready(()) - } - (_, _) => core::task::Poll::Pending, - } - }) - .await; - }; -} diff --git a/src/task/runtime/mod.rs b/src/task/runtime/mod.rs deleted file mode 100644 index e40af2a1..00000000 --- a/src/task/runtime/mod.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Async/await runtime implementation - -// use core::{ -// pin::Pin, -// task::{Context, Poll}, -// time::Duration, -// }; -// -// use futures_util::Future; -// -// use crate::arch::{Architecture, ARCHITECTURE}; -// -// mod executor; -// mod macros; -// mod task; -// mod task_queue; -// mod waker; -// -// pub use executor::{run_to_completion, spawn, spawn_async_worker}; -// pub use task_queue::init_task_queue; -// pub use waker::QueueWaker; -// -// /// [Future] implementation that wraps a poll-function -// pub struct PollFn { -// f: F, -// } -// -// impl Future for PollFn -// where -// F: FnMut(&mut Context) -> Poll, -// { -// type Output = T; -// -// fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { -// #[allow(clippy::needless_borrow)] -// (unsafe { &mut self.get_unchecked_mut().f })(cx) -// } -// } -// -// /// Constructs a [PollFn] from given poll-function -// pub fn poll_fn(f: F) -> PollFn -// where -// F: FnMut(&mut Context) -> Poll, -// { -// PollFn { f } -// } -// - -use core::{ - pin::Pin, - task::{Context, Poll}, - time::Duration, -}; - -use futures_util::Future; -use kernel_util::runtime::QueueWaker; - -use crate::arch::{Architecture, ARCHITECTURE}; diff --git a/src/task/runtime/task.rs b/src/task/runtime/task.rs deleted file mode 100644 index f28e034d..00000000 --- a/src/task/runtime/task.rs +++ /dev/null @@ -1,45 +0,0 @@ -use core::fmt; - -use alloc::sync::Arc; -use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt}; -use kernel_util::sync::IrqSafeSpinlock; - -use super::executor; - -pub trait Termination { - fn print(&self); -} - -pub struct Task { - pub(super) future: IrqSafeSpinlock>>, -} - -impl ArcWake for Task { - fn wake_by_ref(arc_self: &Arc) { - executor::enqueue(arc_self.clone()).unwrap(); - } -} - -impl Task { - pub fn new + Send + 'static>(future: F) -> Arc { - let future = IrqSafeSpinlock::new(Some( - async move { - future.await.print(); - } - .boxed(), - )); - Arc::new(Self { future }) - } -} - -impl Termination for () { - fn print(&self) {} -} - -impl Termination for Result { - fn print(&self) { - if let Err(error) = self { - errorln!("A task finished with an error: {:?}", error); - } - } -} diff --git a/src/task/runtime/task_queue.rs b/src/task/runtime/task_queue.rs deleted file mode 100644 index 306f61cd..00000000 --- a/src/task/runtime/task_queue.rs +++ /dev/null @@ -1,74 +0,0 @@ -use abi::error::Error; -use alloc::sync::Arc; -use crossbeam_queue::ArrayQueue; -use kernel_util::{sync::IrqGuard, util::OneTimeInit}; - -use crate::{ - arch::{Architecture, ArchitectureImpl}, - task::thread::Thread, -}; - -use super::task::Task; - -pub(super) static TASK_QUEUE: OneTimeInit = OneTimeInit::new(); - -pub(super) struct TaskQueue { - // Queue of workers waiting for an item - pending_workers: ArrayQueue>, - task_queue: ArrayQueue>, -} - -impl TaskQueue { - pub fn new(task_capacity: usize) -> Self { - assert!(task_capacity > 0); - Self { - pending_workers: ArrayQueue::new(16), - task_queue: ArrayQueue::new(task_capacity), - } - } - - fn wakeup_one(&self) { - if let Some(worker) = self.pending_workers.pop() { - worker.enqueue_somewhere(); - } - } - - pub fn enqueue(&self, task: Arc) -> Result<(), Error> { - let _irq = IrqGuard::acquire(); - if self.task_queue.push(task).is_err() { - todo!(); - } - self.wakeup_one(); - Ok(()) - } - - pub fn dequeue(&self) -> Result, Error> { - let thread = Thread::current(); - assert!(ArchitectureImpl::interrupt_mask()); - loop { - if let Some(task) = self.task_queue.pop() { - return Ok(task); - } - - if self.pending_workers.push(thread.clone()).is_err() { - panic!("Pending worker queue overflow"); - } - - // This must not fail. Signals must not be raised. - thread.suspend().unwrap(); - } - } -} - -/// Initializes the global async/await task queue -pub fn init_task_queue() { - TASK_QUEUE.init(TaskQueue::new(128)); -} - -pub(super) fn push_task(task: Arc) -> Result<(), Error> { - TASK_QUEUE.get().enqueue(task) -} - -pub(super) fn pop_task() -> Result, Error> { - TASK_QUEUE.get().dequeue() -} diff --git a/src/task/runtime/waker.rs b/src/task/runtime/waker.rs deleted file mode 100644 index 76aceda1..00000000 --- a/src/task/runtime/waker.rs +++ /dev/null @@ -1,69 +0,0 @@ -use core::task::Waker; - -use alloc::collections::VecDeque; -use kernel_util::sync::IrqSafeSpinlock; - -/// Async/await primitive to suspend and wake up tasks waiting on some shared resource -pub struct QueueWaker { - queue: IrqSafeSpinlock>, -} - -impl QueueWaker { - /// Constructs an empty [QueueWaker] - pub const fn new() -> Self { - Self { - queue: IrqSafeSpinlock::new(VecDeque::new()), - } - } - - /// Registers a [Waker] reference to be waken up by this [QueueWaker] - pub fn register(&self, waker: &Waker) { - let mut queue = self.queue.lock(); - - if queue.iter().any(|other| other.will_wake(waker)) { - return; - } - - queue.push_back(waker.clone()); - } - - /// Removes a [Waker] reference from this [QueueWaker] - pub fn remove(&self, waker: &Waker) -> bool { - let mut queue = self.queue.lock(); - let mut index = 0; - let mut removed = false; - - while index < queue.len() { - if queue[index].will_wake(waker) { - removed = true; - queue.remove(index); - } - index += 1; - } - - removed - } - - /// Wakes up up to `limit` tasks waiting on this queue - pub fn wake_some(&self, limit: usize) -> usize { - let mut queue = self.queue.lock(); - let mut count = 0; - - while count < limit && let Some(item) = queue.pop_front() { - item.wake(); - count += 1; - } - - count - } - - /// Wakes up a single task waiting on this queue - pub fn wake_one(&self) -> bool { - self.wake_some(1) != 0 - } - - /// Wakes up all tasks waiting on this queue - pub fn wake_all(&self) -> usize { - self.wake_some(usize::MAX) - } -} diff --git a/src/task/thread.rs b/src/task/thread.rs index cfc9b0d0..ceb24d26 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -78,7 +78,6 @@ pub struct Thread { name: Option, id: ThreadId, - cpu_id: AtomicU32, pub(super) state: AtomicThreadState, process: Option>, @@ -104,7 +103,6 @@ impl Thread { name, id, - cpu_id: AtomicU32::new(0), state: AtomicThreadState::new(ThreadState::Suspended), process, @@ -281,12 +279,9 @@ impl Thread { ThreadState::Suspended => (), ThreadState::Terminated => panic!("Thread is terminated"), ThreadState::Running => { - let cpu_id = self.cpu_id.load(Ordering::Acquire); - let local_cpu_id = Cpu::local_id(); let queue = Cpu::local().queue(); - if cpu_id == local_cpu_id { - assert_eq!(queue as *const _, proc_queue as *const _, "Thread queue mismatch: process says cpu{}, queue {:p}, actual cpu{}, queue {:p}", cpu_id, proc_queue, local_cpu_id, queue); + if core::ptr::eq(queue, proc_queue) { drop(inner); // Suspending a process running on local CPU unsafe { queue.yield_cpu() } @@ -303,15 +298,9 @@ impl Thread { /// /// Only meant to be called from within the scheduler. pub unsafe fn set_running(&self, cpu: u32) { - self.cpu_id.store(cpu, Ordering::Release); self.state.store(ThreadState::Running, Ordering::Release); } - /// Suspends the thread and removes it from the queue until it is "waken up" again - pub fn suspend(&self) { - self.dequeue(ThreadState::Suspended); - } - // Accounting /// Returns the thread with given [ThreadId], if it exists @@ -389,12 +378,7 @@ impl CurrentThread { process.handle_thread_exit(self.id(), code); } - let mut inner = self.inner.lock(); - let proc_queue = inner.queue.take().unwrap(); let queue = Cpu::local().queue(); - assert_eq!(proc_queue.index(), queue.index()); - - drop(inner); queue.dequeue(self.id()); unsafe { queue.yield_cpu() } @@ -580,3 +564,9 @@ fn __exit_current(t: &CurrentThread, code: ExitCode) -> ! { t.exit(code); unreachable!(); } + +#[no_mangle] +unsafe fn __yield() { + let _irq = IrqGuard::acquire(); + Cpu::local().queue().yield_cpu() +} diff --git a/src/util/ring.rs b/src/util/ring.rs index c9605d44..c49c4c92 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -17,12 +17,13 @@ pub struct RingBuffer { data: [T; N], } -/// Ring buffer with async read support -pub struct AsyncRing { - inner: Arc>>, - read_waker: Arc, -} - +// /// Ring buffer with async read support +// pub struct AsyncRing { +// inner: Arc>>, +// /// +// pub read_waker: Arc, +// } +// impl RingBuffer { /// Constructs a new [RingBuffer] and fills it with `value` pub const fn new(value: T) -> Self { @@ -33,8 +34,17 @@ impl RingBuffer { } } + const fn readable_count_at(&self, at: usize) -> usize { + if at <= self.wr { + self.wr - at + } else { + self.wr + (N - at) + } + } + + /// #[inline] - const fn is_readable(&self) -> bool { + pub const fn is_readable(&self) -> bool { self.is_readable_at(self.rd) } @@ -46,8 +56,25 @@ impl RingBuffer { } } + /// + pub fn contains(&self, t: T) -> bool + where + T: PartialEq, + { + let count = self.readable_count_at(self.rd); + + for i in 0..count { + if self.data[(self.rd + i) % N] == t { + return true; + } + } + + false + } + + /// #[inline] - fn read_single_unchecked(&mut self) -> T { + pub fn read_single_unchecked(&mut self) -> T { let res = self.data[self.rd]; self.rd = (self.rd + 1) % N; res @@ -73,52 +100,57 @@ impl RingBuffer { } } -impl AsyncRing { - /// Constructs a new [AsyncRing] and fills it with `value` - pub fn new(value: T) -> Self { - Self { - inner: Arc::new(IrqSafeSpinlock::new(RingBuffer::new(value))), - read_waker: Arc::new(QueueWaker::new()), - } - } - - /// Writes a single entry to the buffer and signals readers - pub fn try_write(&self, item: T) -> Result<(), Error> { - { - let mut lock = self.inner.lock(); - lock.write(item); - } - self.read_waker.wake_one(); - - Ok(()) - } - - /// Asynchronously reads an entry from the buffer - pub fn read(&self) -> impl Future { - struct ReadFuture { - inner: Arc>>, - read_waker: Arc, - } - - impl Future for ReadFuture { - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.read_waker.register(cx.waker()); - - let mut inner = self.inner.lock(); - if inner.is_readable() { - self.read_waker.remove(cx.waker()); - Poll::Ready(inner.read_single_unchecked()) - } else { - Poll::Pending - } - } - } - - ReadFuture { - inner: self.inner.clone(), - read_waker: self.read_waker.clone(), - } - } -} +// impl AsyncRing { +// /// Constructs a new [AsyncRing] and fills it with `value` +// pub fn new(value: T) -> Self { +// Self { +// inner: Arc::new(IrqSafeSpinlock::new(RingBuffer::new(value))), +// read_waker: Arc::new(QueueWaker::new()), +// } +// } +// +// /// Writes a single entry to the buffer and signals readers +// pub fn try_write(&self, item: T) -> Result<(), Error> { +// { +// let mut lock = self.inner.lock(); +// lock.write(item); +// } +// self.read_waker.wake_one(); +// +// Ok(()) +// } +// +// /// Asynchronously reads an entry from the buffer +// pub fn read(&self) -> impl Future { +// struct ReadFuture { +// inner: Arc>>, +// read_waker: Arc, +// } +// +// impl Future for ReadFuture { +// type Output = T; +// +// fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { +// self.read_waker.register(cx.waker()); +// +// let mut inner = self.inner.lock(); +// if inner.is_readable() { +// self.read_waker.remove(cx.waker()); +// Poll::Ready(inner.read_single_unchecked()) +// } else { +// Poll::Pending +// } +// } +// } +// +// ReadFuture { +// inner: self.inner.clone(), +// read_waker: self.read_waker.clone(), +// } +// } +// +// /// +// pub fn can_read_now(&self) -> bool { +// self.inner.lock().is_readable() +// } +// } From 77e5fcc947b257ff032edc72db6c7bfd2beadf2d Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 22 Dec 2023 12:36:42 +0200 Subject: [PATCH 131/211] arch: nicer interface for accessing Cpu --- lib/kernel-util/src/sync/guard.rs | 13 +++ src/arch/mod.rs | 45 ++++++++-- src/arch/x86_64/apic/local.rs | 8 +- src/arch/x86_64/apic/mod.rs | 2 +- src/arch/x86_64/cpu.rs | 139 ++++++++++++++++++------------ src/arch/x86_64/exception.rs | 2 +- src/arch/x86_64/mod.rs | 4 +- src/arch/x86_64/smp.rs | 2 +- src/panic.rs | 19 ++-- src/task/mod.rs | 9 +- src/task/sched.rs | 10 +-- src/task/thread.rs | 14 +-- 12 files changed, 169 insertions(+), 98 deletions(-) diff --git a/lib/kernel-util/src/sync/guard.rs b/lib/kernel-util/src/sync/guard.rs index d2db9036..4695f82b 100644 --- a/lib/kernel-util/src/sync/guard.rs +++ b/lib/kernel-util/src/sync/guard.rs @@ -1,9 +1,14 @@ +use core::ops::Deref; + use crate::api; /// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation /// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. pub struct IrqGuard(bool); +/// +pub struct GuardedRef<'a, T: ?Sized>(pub &'a T, pub IrqGuard); + // IrqGuard impls impl IrqGuard { /// Saves the current IRQ state and masks them @@ -17,3 +22,11 @@ impl Drop for IrqGuard { unsafe { api::__release_irq_guard(self.0) } } } + +impl<'a, T: ?Sized> Deref for GuardedRef<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.0 + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index b14fc17b..4b7eab52 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,6 +1,9 @@ //! Provides architecture/platform-specific implementation details -use core::time::Duration; +use core::{ + ops::{Deref, DerefMut}, + time::Duration, +}; use abi::error::Error; @@ -30,11 +33,12 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use kernel_util::mem::{ - address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, +use kernel_util::{ + mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel}, + sync::IrqGuard, }; -use crate::{mem::phys::PhysicalMemoryRegion, task::Cpu}; +use crate::{mem::phys::PhysicalMemoryRegion, task::sched::CpuQueue}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -227,6 +231,36 @@ pub trait Architecture { } } +/// Interface for accessing local CPU (the CPU of the current context) +pub trait LocalCpuAccess: DerefMut { + /// Consumes the struct, returning the inner IrqGuard for further use + fn into_guard(self) -> IrqGuard; +} + +/// Interface for accessing CPU-local data +pub trait CpuAccess: Sized { + /// Safe wrapper for accessing local CPU + type Local: LocalCpuAccess; + + /// Returns the local CPU wrapper + fn local() -> Self::Local; + /// Returns the local CPU wrapper or None, if not yet initialized: + fn try_local() -> Option; + + /// Initializes the CPU's scheduling queue. + /// + /// # Panics + /// + /// Will panic, if the initialization has already been performed. + fn init_queue(&mut self, queue: &'static CpuQueue); + + /// Returns the CPU's scheduling queue + fn queue(&self) -> &'static CpuQueue; + + /// Returns the CPU index + fn id(&self) -> u32; +} + // External API for architecture specifics #[no_mangle] @@ -247,7 +281,8 @@ fn __release_irq_guard(mask: bool) { #[no_mangle] fn __cpu_index() -> usize { - Cpu::local_id() as _ + todo!() + // Cpu::local_id() as _ } #[no_mangle] diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 8e21a03f..b951b1eb 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -30,7 +30,7 @@ use crate::{ x86_64::{ apic::APIC_MSI_OFFSET, mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT, }, - CpuMessage, + CpuAccess, CpuMessage, }, task::Cpu, }; @@ -230,9 +230,11 @@ impl LocalInterruptController for LocalApic { // TODO use NMI or regular interrupt depending on severity of the message match target { IpiDeliveryTarget::OtherCpus => { - let local = Cpu::local_id(); + let local = Cpu::local(); + let local_id = local.id() as usize; + for i in 0..CPU_COUNT.load(Ordering::Acquire) { - if i != local as usize { + if i != local_id { Cpu::push_ipi_queue(i as u32, msg); } } diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index f82c2a2e..982422e0 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -5,7 +5,7 @@ use core::arch::global_asm; use static_assertions::{const_assert, const_assert_eq}; use crate::{ - arch::{x86_64::cpu::Cpu, Architecture}, + arch::{x86_64::cpu::Cpu, Architecture, CpuAccess}, task::thread::Thread, }; diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 740f5583..855d1d63 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -1,14 +1,21 @@ //! Per-CPU information and data structures -use core::{ptr::null_mut, sync::atomic::Ordering}; +use core::{ + ops::{Deref, DerefMut}, + ptr::null_mut, + sync::atomic::Ordering, +}; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + sync::{guard::GuardedRef, IrqGuard, IrqSafeSpinlock}, + util::OneTimeInit, +}; use tock_registers::interfaces::Writeable; use crate::{ arch::{ x86_64::{cpuid, gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, - CpuMessage, + CpuAccess, CpuMessage, LocalCpuAccess, }, task::sched::CpuQueue, }; @@ -31,6 +38,14 @@ pub struct Cpu { queue: OneTimeInit<&'static CpuQueue>, } +/// Handle allowing safe access to the local CPU. +/// +/// # Note +/// +/// Obtaining this handle prevents IRQs from being delivered to ensure the context having this +/// struct cannot be interrupted by another (which could obtain it as well). +pub struct LocalCpu(&'static mut Cpu, IrqGuard); + struct IpiQueue { data: IrqSafeSpinlock>, } @@ -63,7 +78,7 @@ impl Cpu { /// # Safety /// /// Only meant to be called once per each CPU during their init. - pub unsafe fn init_local(local_apic: LocalApic, id: u32) { + pub(super) unsafe fn init_local(local_apic: LocalApic, id: u32) { infoln!("Initialize CPU with id {}", id); // Initialize CPU features @@ -89,49 +104,11 @@ impl Cpu { syscall::init_syscall(); } - /// Returns this CPU's local data structure - #[inline(always)] - pub fn local<'a>() -> &'a Self { - Self::get_local().unwrap() - } - - /// Returns the system ID of the CPU - pub fn local_id() -> u32 { - if let Some(cpu) = Self::get_local() { - cpu.id() - } else { - 0 - } - } - - /// Returns this CPU's local data structure or None if it hasn't been set up yet - #[inline(always)] - pub fn get_local<'a>() -> Option<&'a Self> { - let mut addr: usize; - unsafe { - core::arch::asm!("movq %gs:(0), {0}", out(reg) addr, options(att_syntax)); - (addr as *const Cpu).as_ref() - } - } - - /// Returns this CPU's number - pub fn id(&self) -> u32 { - self.id - } - - /// Returns the local APIC associated with this CPU - pub fn local_apic(&self) -> &LocalApic { - &self.local_apic - } - - /// Returns the CPU's execution queue - pub fn queue(&self) -> &'static CpuQueue { - self.queue.get() - } - - /// Sets up the local CPU's execution queue - pub fn init_queue(&self, queue: &'static CpuQueue) { - self.queue.init(queue); + /// Sets up global list of interprocessor message queues + pub fn init_ipi_queues() { + IPI_QUEUES.init(Vec::from_iter( + (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), + )); } /// Inserts an IPI message to the back of the target CPU's message queue @@ -149,16 +126,66 @@ impl Cpu { let ipi_queue = &IPI_QUEUES.get()[self.id as usize]; ipi_queue.pop() } +} - /// Sets up global list of interprocessor message queues - pub fn init_ipi_queues() { - IPI_QUEUES.init(Vec::from_iter( - (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), - )); +impl LocalCpu { + /// Returns the local APIC associated with this CPU + pub fn local_apic(&self) -> &'static LocalApic { + // TODO find a way to safely represent this voodoo magic + let r = (&self.0.local_apic) as *const LocalApic; + unsafe { &*r } + } +} + +impl CpuAccess for Cpu { + type Local = LocalCpu; + + fn local() -> Self::Local { + Self::try_local().unwrap() } - // /// Gets an IPI message from the processor's queue and takes corresponding actions. If there is - // /// none, this is treated as a spurious IPI and ignored. See [CpuMessage]. - // pub fn handle_ipi(&self) { - // } + fn try_local() -> Option { + let mut addr: usize; + let guard = IrqGuard::acquire(); + unsafe { + core::arch::asm!("movq %gs:(0), {0}", out(reg) addr, options(att_syntax)); + if let Some(cpu) = (addr as *mut Cpu).as_mut() { + Some(LocalCpu(cpu, guard)) + } else { + None + } + } + } + + fn id(&self) -> u32 { + self.id + } + + fn init_queue(&mut self, queue: &'static CpuQueue) { + self.queue.init(queue); + } + + fn queue(&self) -> &'static CpuQueue { + self.queue.get() + } +} + +impl LocalCpuAccess for LocalCpu { + fn into_guard(self) -> IrqGuard { + self.1 + } +} + +impl Deref for LocalCpu { + type Target = Cpu; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl DerefMut for LocalCpu { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + } } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 9cb92d8b..bf7ea3f0 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -4,7 +4,7 @@ use core::{arch::global_asm, mem::size_of_val}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use crate::{ - arch::x86_64::apic, + arch::{x86_64::apic, CpuAccess}, task::{context::TaskFrame, thread::Thread, Cpu}, }; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 6e4613a5..f186fe91 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -18,7 +18,7 @@ use kernel_util::{ device::RawDeviceMemoryMapping, table::EntryLevelExt, }, - sync::SpinFence, + sync::{guard::GuardedRef, SpinFence}, util::OneTimeInit, }; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; @@ -77,7 +77,7 @@ use self::{ smp::CPU_COUNT, }; -use super::{Architecture, CpuMessage}; +use super::{Architecture, CpuAccess, CpuMessage}; /// x86-64-specific interrupt number #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index fa69e85f..f8a08ae1 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -17,7 +17,7 @@ use crate::{ KERNEL_TABLES, }, }, - Architecture, ArchitectureImpl, + Architecture, ArchitectureImpl, CpuAccess, }, mem::phys, task::Cpu, diff --git a/src/panic.rs b/src/panic.rs index 2900e0fe..9661fdf3 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -5,7 +5,7 @@ use device_api::interrupt::IpiDeliveryTarget; use kernel_util::sync::{hack_locks, SpinFence}; use crate::{ - arch::{Architecture, ArchitectureImpl, CpuMessage, ARCHITECTURE}, + arch::{Architecture, ArchitectureImpl, CpuAccess, CpuMessage, ARCHITECTURE}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, task::Cpu, @@ -19,15 +19,13 @@ static PANIC_SEQUENCE: AtomicU32 = AtomicU32::new(0); /// Panic handler for CPUs other than the one that initiated it pub fn panic_secondary() -> ! { - let id = Cpu::local_id(); - unsafe { - ArchitectureImpl::set_interrupt_mask(true); - } + // Will also mask IRQs in this section + let cpu = Cpu::local(); PANIC_HANDLED_FENCE.signal(); PANIC_FINISHED_FENCE.wait_one(); - while PANIC_SEQUENCE.load(Ordering::Acquire) != id { + while PANIC_SEQUENCE.load(Ordering::Acquire) != cpu.id() { core::hint::spin_loop(); } @@ -43,9 +41,7 @@ pub fn panic_secondary() -> ! { #[panic_handler] fn panic_handler(pi: &core::panic::PanicInfo) -> ! { - unsafe { - ArchitectureImpl::set_interrupt_mask(true); - } + let cpu = Cpu::local(); static PANIC_HAPPENED: AtomicBool = AtomicBool::new(false); @@ -55,7 +51,6 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() { - let id = Cpu::local_id(); // Let other CPUs know we're screwed unsafe { ARCHITECTURE @@ -71,7 +66,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { } log_print_raw!(LogLevel::Fatal, "--- BEGIN PANIC ---\n"); - log_print_raw!(LogLevel::Fatal, "In CPU {}\n", Cpu::local_id()); + log_print_raw!(LogLevel::Fatal, "In CPU {}\n", cpu.id()); log_print_raw!(LogLevel::Fatal, "Kernel panic "); if let Some(location) = pi.location() { @@ -95,7 +90,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { log_print_raw!(LogLevel::Fatal, "--- END PANIC ---\n"); PANIC_FINISHED_FENCE.signal(); - while PANIC_SEQUENCE.load(Ordering::Acquire) != id { + while PANIC_SEQUENCE.load(Ordering::Acquire) != cpu.id() { core::hint::spin_loop(); } diff --git a/src/task/mod.rs b/src/task/mod.rs index 6e21f0b4..fa1491d4 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -7,7 +7,7 @@ use alloc::{string::String, vec::Vec}; use kernel_util::{runtime, sync::SpinFence, thread::Termination}; use crate::{ - arch::{Architecture, ArchitectureImpl}, + arch::{Architecture, ArchitectureImpl, CpuAccess}, task::{sched::CpuQueue, thread::Thread}, }; @@ -59,17 +59,16 @@ pub fn init() -> Result<(), Error> { pub unsafe fn enter() -> ! { static AP_CAN_ENTER: SpinFence = SpinFence::new(); - let cpu_id = Cpu::local_id(); + let mut cpu = Cpu::local(); - if cpu_id != 0 { + if cpu.id() != 0 { // Wait until BSP allows us to enter AP_CAN_ENTER.wait_one(); } else { AP_CAN_ENTER.signal(); } - let queue = CpuQueue::for_cpu(cpu_id as usize); - let cpu = Cpu::local(); + let queue = CpuQueue::for_cpu(cpu.id() as usize); cpu.init_queue(queue); queue.enter() diff --git a/src/task/sched.rs b/src/task/sched.rs index 995082d2..3c839fcb 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -7,7 +7,7 @@ use cfg_if::cfg_if; use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::{ - arch::{Architecture, ArchitectureImpl}, + arch::{Architecture, ArchitectureImpl, CpuAccess}, task::thread::ThreadState, }; @@ -108,7 +108,7 @@ impl CpuQueueInner { impl CpuQueue { /// Constructs an empty queue with its own idle task pub fn new(index: usize) -> Self { - let idle = TaskContext::kernel(__idle, Cpu::local_id() as usize) + let idle = TaskContext::kernel(__idle, Cpu::local().id() as usize) .expect("Could not construct an idle task"); Self { @@ -144,8 +144,8 @@ impl CpuQueue { /// The function is only meant to be called from kernel threads (e.g. if they want to yield /// CPU execution to wait for something) or interrupt handlers. pub unsafe fn yield_cpu(&self) { - // Make sure the scheduling process doesn't get interrupted - ArchitectureImpl::set_interrupt_mask(true); + // Will also disable interrupts in this section + let cpu = Cpu::local(); let mut inner = self.inner.lock(); @@ -185,7 +185,7 @@ impl CpuQueue { }; let (to, _to_rc) = if let Some(next_t) = next_t.as_ref() { - next_t.set_running(Cpu::local_id()); + next_t.set_running(cpu.id()); (next_t.context(), Arc::strong_count(next_t)) } else { (&self.idle, 0) diff --git a/src/task/thread.rs b/src/task/thread.rs index ceb24d26..2e863e7a 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -5,7 +5,7 @@ use core::{ mem::size_of, ops::Deref, pin::Pin, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, + sync::atomic::{AtomicU64, Ordering}, task::{Context, Poll}, }; @@ -27,6 +27,7 @@ use kernel_util::{ }; use crate::{ + arch::{CpuAccess, LocalCpuAccess}, mem::{process::ProcessAddressSpace, ForeignPointer}, task::{context::TaskContextImpl, Cpu}, }; @@ -198,12 +199,11 @@ impl Thread { /// Returns the current thread on the CPU, if any is present pub fn get_current() -> Option { - let guard = IrqGuard::acquire(); - Cpu::local() - .queue() - .current_id() - .and_then(Self::get) - .map(|t| CurrentThread(t, guard)) + // IrqGuard is held throughout + let cpu = Cpu::local(); + let thread = cpu.queue().current_id().and_then(Self::get); + + thread.map(|t| CurrentThread(t, cpu.into_guard())) } /// Enqueues the thread onto any (usually the least loaded) CPU queue and returns its index. From cdd401552ea0dedf06a645378e2607c5ac30f618 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 22 Dec 2023 12:41:45 +0200 Subject: [PATCH 132/211] arch/x86-64: implement IrqFrame signal frame --- src/arch/x86_64/exception.rs | 39 +++++++++++++++++++++++++++--------- src/arch/x86_64/mod.rs | 2 +- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index bf7ea3f0..7ebff9d7 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -136,7 +136,26 @@ struct Pointer { impl TaskFrame for IrqFrame { fn store(&self) -> SavedFrame { - todo!() + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.rdx, + rbx: self.rbx, + rsi: self.rsi, + rdi: self.rdi, + rbp: self.rbp, + r8: self.r8, + r9: self.r9, + r10: self.r10, + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.rip, + user_sp: self.rsp, + rflags: self.rflags, + } } fn restore(&mut self, _saved: &SavedFrame) { @@ -144,31 +163,31 @@ impl TaskFrame for IrqFrame { } fn argument(&self) -> u64 { - todo!(); + self.rdi as _ } fn user_ip(&self) -> usize { - todo!() + self.rip as _ } fn user_sp(&self) -> usize { - todo!() + self.rsp as _ } fn set_argument(&mut self, value: u64) { self.rdi = value; } - fn set_return_value(&mut self, _value: u64) { - todo!() + fn set_return_value(&mut self, value: u64) { + self.rax = value; } - fn set_user_ip(&mut self, _value: usize) { - todo!() + fn set_user_ip(&mut self, value: usize) { + self.rip = value as _; } - fn set_user_sp(&mut self, _value: usize) { - todo!() + fn set_user_sp(&mut self, value: usize) { + self.rsp = value as _; } } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index f186fe91..da60007a 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -18,7 +18,7 @@ use kernel_util::{ device::RawDeviceMemoryMapping, table::EntryLevelExt, }, - sync::{guard::GuardedRef, SpinFence}, + sync::SpinFence, util::OneTimeInit, }; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; From 369668dbfdc855640906e7450f3e7caf9b1bc4c4 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 22 Dec 2023 13:10:12 +0200 Subject: [PATCH 133/211] fs/poll: properly return poll error if the file reports it --- lib/vfs/src/poll.rs | 14 +++++++------- src/fs/mod.rs | 1 - src/fs/poll.rs | 14 -------------- src/syscall/mod.rs | 2 +- 4 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 src/fs/poll.rs diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index fe070ffa..1becf376 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -4,16 +4,16 @@ use core::{ time::Duration, }; -use alloc::{collections::BTreeMap, vec::Vec}; +use alloc::collections::BTreeMap; use futures_util::{future::BoxFuture, Future, FutureExt}; use kernel_util::{ runtime, - sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, + sync::{mutex::Mutex, LockMethod}, }; use yggdrasil_abi::{error::Error, io::RawFd}; -use crate::{FileRef, FileSet}; +use crate::FileRef; /// pub struct FdPoll { @@ -48,13 +48,13 @@ impl FdPoll { } /// Polls the channel once, returning either a file descriptor or timeout - pub async fn poll_once(&self, timeout: Option) -> Option { + pub async fn poll_once(&self, timeout: Option) -> Option<(RawFd, Result<(), Error>)> { self.begin(timeout).await } } impl<'a> Future for FdPollFuture<'a> { - type Output = Option; + type Output = Option<(RawFd, Result<(), Error>)>; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { if let Some(timeout) = self.timeout.as_mut() @@ -65,8 +65,8 @@ impl<'a> Future for FdPollFuture<'a> { } for (&fd, file) in self.fds.lock().unwrap().iter() { - if file.poll_read(cx).is_ready() { - return Poll::Ready(Some(fd)); + if let Poll::Ready(result) = file.poll_read(cx) { + return Poll::Ready(Some((fd, result))); } } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 2e4f26f6..22811030 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -11,7 +11,6 @@ use yggdrasil_abi::{error::Error, io::MountOptions}; use crate::{mem::phys, proc::random}; // pub mod devfs; -pub mod poll; pub mod sysfs; /// Describes in-memory filesystem image used as initial root diff --git a/src/fs/poll.rs b/src/fs/poll.rs deleted file mode 100644 index 800edb7b..00000000 --- a/src/fs/poll.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! - -use core::{ - pin::Pin, - task::{Context, Poll}, - time::Duration, -}; - -use abi::{error::Error, io::RawFd}; -use alloc::vec::Vec; -use futures_util::{future::BoxFuture, Future, FutureExt}; -use kernel_util::{runtime, sync::IrqSafeSpinlock}; - -use crate::proc::io::ProcessIo; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 1fe103fe..a5775ff2 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -328,7 +328,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::PollWait => { let poll_fd = RawFd::from(args[0] as u32); let timeout = arg_user_ref::>(args[1] as usize)?; - let output = arg_user_mut::>(args[2] as usize)?; + let output = arg_user_mut::)>>(args[2] as usize)?; debugln!("PollWait {:?}", timeout); From e2bcbee3bbddea9687e2fb7af1f14f2460b81201 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 22 Dec 2023 13:33:52 +0200 Subject: [PATCH 134/211] proc/mem: add a stub for mapping files to memory --- src/syscall/mod.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index a5775ff2..ef4716a2 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -7,6 +7,7 @@ use abi::{ DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, PollControl, RawFd, SeekFrom, }, + mem::MappingSource, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; @@ -90,6 +91,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Resource management SyscallFunction::MapMemory => { let len = args[1] as usize; + let source = arg_user_ref::(args[2] as usize)?; let space = thread.address_space(); @@ -97,12 +99,19 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result todo!(); } - space.allocate( - None, - len, - |_| phys::alloc_page(), - MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - ) + match source { + MappingSource::Anonymous => space.allocate( + None, + len, + |_| phys::alloc_page(), + MapAttributes::USER_WRITE + | MapAttributes::USER_READ + | MapAttributes::NON_GLOBAL, + ), + MappingSource::File(fd, offset) => run_with_io(process, |io| { + todo!(); + }), + } } SyscallFunction::UnmapMemory => { let addr = args[0] as usize; From 8df5dc487e4a560bfb09045f552640217f5682af Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 26 Dec 2023 22:12:47 +0200 Subject: [PATCH 135/211] proc: add fb0 device + message channels --- driver/block/core/src/lib.rs | 6 +- lib/kernel-util/src/sync/guard.rs | 13 --- lib/vfs/src/channel.rs | 172 ++++++++++++++++++++++++++++++ lib/vfs/src/file/mod.rs | 50 +++++++-- lib/vfs/src/ioctx.rs | 17 +++ lib/vfs/src/lib.rs | 1 + lib/vfs/src/node/access.rs | 1 + lib/vfs/src/node/ops.rs | 25 ++++- lib/vfs/src/poll.rs | 3 +- lib/vfs/src/traits.rs | 10 +- src/arch/x86_64/mod.rs | 1 + src/device/display/fb_console.rs | 8 +- src/device/display/linear_fb.rs | 99 ++++++++++++++--- src/device/tty.rs | 23 ++-- src/fs/mod.rs | 17 ++- src/init.rs | 4 +- src/main.rs | 2 - src/mem/process.rs | 3 +- src/proc/random.rs | 12 ++- src/syscall/mod.rs | 72 +++++++++++-- src/util/ring.rs | 84 ++------------- 21 files changed, 463 insertions(+), 160 deletions(-) create mode 100644 lib/vfs/src/channel.rs diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index a8b1264f..aa12f1af 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -8,7 +8,7 @@ use core::{ }; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::runtime; +use kernel_util::{mem::address::PhysicalAddress, runtime}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; @@ -104,6 +104,10 @@ pub trait BlockDevice: Sync { Err(Error::NotImplemented) } + fn get_page(&self, offset: u64) -> Result { + Err(Error::NotImplemented) + } + // fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> { // let count = self.read(pos, buf)?; // if count == buf.len() { diff --git a/lib/kernel-util/src/sync/guard.rs b/lib/kernel-util/src/sync/guard.rs index 4695f82b..d2db9036 100644 --- a/lib/kernel-util/src/sync/guard.rs +++ b/lib/kernel-util/src/sync/guard.rs @@ -1,14 +1,9 @@ -use core::ops::Deref; - use crate::api; /// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation /// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. pub struct IrqGuard(bool); -/// -pub struct GuardedRef<'a, T: ?Sized>(pub &'a T, pub IrqGuard); - // IrqGuard impls impl IrqGuard { /// Saves the current IRQ state and masks them @@ -22,11 +17,3 @@ impl Drop for IrqGuard { unsafe { api::__release_irq_guard(self.0) } } } - -impl<'a, T: ?Sized> Deref for GuardedRef<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.0 - } -} diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs new file mode 100644 index 00000000..322f595e --- /dev/null +++ b/lib/vfs/src/channel.rs @@ -0,0 +1,172 @@ +use core::{ + pin::Pin, + task::{Context, Poll}, +}; + +use alloc::{ + boxed::Box, + collections::{BTreeMap, VecDeque}, + string::String, + sync::Arc, + vec::Vec, +}; +use futures_util::{task::AtomicWaker, Future}; +use kernel_util::{ + block, + sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, +}; +use yggdrasil_abi::error::Error; + +use crate::FileReadiness; + +pub struct Channel { + subscriptions: Mutex>>, +} + +pub struct Message { + data: Box<[u8]>, +} + +pub struct Subscription { + queue: Mutex>>, + notify: AtomicWaker, +} + +pub struct ChannelDescriptor { + tx: Arc, + rx: Option>, +} + +impl ChannelDescriptor { + pub fn open(name: &str, subscribe: bool) -> ChannelDescriptor { + let tx = Channel::get_or_create(name.into()); + let rx = if subscribe { + Some(tx.subscribe()) + } else { + None + }; + + Self { tx, rx } + } + + pub fn receive_message(&self, buf: &mut [u8]) -> Result { + let Some(rx) = self.rx.as_ref() else { + return Err(Error::InvalidOperation); + }; + + let message = rx.receive_message_inner()?; + let len = message.data.len(); + + if buf.len() < len { + return Err(Error::MissingData); + } + + buf[..len].copy_from_slice(&message.data); + + Ok(len) + } + + pub fn send_message(&self, msg: &[u8]) -> Result<(), Error> { + let message = Arc::new(Message { + data: Box::from(msg), + }); + + let lock = self.tx.subscriptions.lock()?; + for sub in lock.iter() { + sub.push_message(message.clone())?; + } + + Ok(()) + } +} + +impl Channel { + fn new() -> Arc { + Arc::new(Self { + subscriptions: Mutex::new(Vec::new()), + }) + } + + pub fn get_or_create(name: String) -> Arc { + let mut channels = CHANNELS.lock(); + + channels.entry(name).or_insert_with(Self::new).clone() + } + + pub fn subscribe(&self) -> Arc { + let mut lock = self.subscriptions.lock().unwrap(); + + let sub = Arc::new(Subscription { + queue: Mutex::new(VecDeque::new()), + notify: AtomicWaker::new(), + }); + + lock.push(sub.clone()); + + sub + } +} + +impl Subscription { + fn receive_message_async(&self) -> impl Future, Error>> + '_ { + struct F<'f> { + rx: &'f Subscription, + } + + impl<'f> Future for F<'f> { + type Output = Result, Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut lock = self.rx.queue.lock()?; + if let Some(msg) = lock.pop_front() { + return Poll::Ready(Ok(msg)); + } + drop(lock); + + self.rx.notify.register(cx.waker()); + + let mut lock = self.rx.queue.lock()?; + if let Some(msg) = lock.pop_front() { + Poll::Ready(Ok(msg)) + } else { + Poll::Pending + } + } + } + + F { rx: self } + } + + fn receive_message_inner(&self) -> Result, Error> { + block!(self.receive_message_async().await)? + } + + fn push_message(&self, msg: Arc) -> Result<(), Error> { + self.queue.lock()?.push_back(msg); + self.notify.wake(); + Ok(()) + } +} + +impl FileReadiness for ChannelDescriptor { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + let Some(rx) = self.rx.as_ref() else { + return Poll::Ready(Err(Error::InvalidOperation)); + }; + + if !rx.queue.lock()?.is_empty() { + return Poll::Ready(Ok(())); + } + + rx.notify.register(cx.waker()); + + if !rx.queue.lock()?.is_empty() { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } +} + +static CHANNELS: IrqSafeSpinlock>> = + IrqSafeSpinlock::new(BTreeMap::new()); diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 85e9e009..f76726b7 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -10,18 +10,18 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use futures_util::{future::BoxFuture, FutureExt}; -use kernel_util::sync::{mutex::Mutex, IrqSafeSpinlock}; +use kernel_util::{mem::address::PhysicalAddress, sync::IrqSafeSpinlock}; use yggdrasil_abi::{ error::Error, io::{DirectoryEntry, OpenOptions, RawFd, SeekFrom}, }; use crate::{ + channel::{Channel, ChannelDescriptor}, device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, traits::{Read, Seek, Write}, - FdPoll, + FdPoll, FileReadiness, }; use self::{ @@ -60,6 +60,7 @@ pub enum File { Char(CharFile), AnonymousPipe(PipeEnd), Poll(FdPoll), + Channel(ChannelDescriptor), } /// Contains a per-process fd -> FileRef map @@ -82,6 +83,12 @@ impl File { Arc::new(Self::Poll(FdPoll::new())) } + /// Opens a new message channel, optionally subscribing to it as well + pub fn new_message_channel(name: &str, with_sub: bool) -> Arc { + let channel = ChannelDescriptor::open(name, with_sub); + Arc::new(Self::Channel(channel)) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -167,20 +174,29 @@ impl File { Self::Regular(file) => Some(&file.node), Self::Block(file) => Some(&file.node), Self::Char(file) => Some(&file.node), - Self::AnonymousPipe(_) | Self::Poll(_) => None, + Self::AnonymousPipe(_) | Self::Poll(_) | Self::Channel(_) => None, } } - /// + /// Polls a file for "read-readiness" pub fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { match self { Self::Char(f) => f.device.0.poll_read(cx), + Self::Channel(ch) => ch.poll_read(cx), // Polling not implemented, return ready immediately (XXX ?) _ => Poll::Ready(Ok(())), } } - /// + /// Gets a page from a file to be mapped into virtual memory + pub fn get_page(&self, offset: u64) -> Result { + match self { + Self::Block(f) => f.device.0.get_page(offset), + _ => Err(Error::NotImplemented), + } + } + + /// Interprets the file as a poll channel pub fn as_poll_channel(&self) -> Result<&FdPoll, Error> { if let Self::Poll(poll) = self { Ok(poll) @@ -188,6 +204,15 @@ impl File { Err(Error::InvalidOperation) } } + + /// Interprets the file as a message channel + pub fn as_message_channel(&self) -> Result<&ChannelDescriptor, Error> { + if let Self::Channel(ch) = self { + Ok(ch) + } else { + Err(Error::InvalidOperation) + } + } } impl Read for File { @@ -199,6 +224,8 @@ impl Read for File { Self::AnonymousPipe(pipe) => pipe.read(buf), // TODO maybe allow reading FDs from poll channels as if they were regular streams? Self::Poll(_) => Err(Error::InvalidOperation), + // TODO maybe allow reading messages from Channels? + Self::Channel(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -213,6 +240,8 @@ impl Write for File { Self::AnonymousPipe(pipe) => pipe.write(buf), // TODO maybe allow adding FDs to poll channels this way Self::Poll(_) => Err(Error::InvalidOperation), + // TODO maybe allow writing messages to Channels? + Self::Channel(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -223,7 +252,9 @@ impl Seek for File { match self { Self::Regular(file) => Ok(*file.position.lock()), Self::Block(file) => Ok(*file.position.lock()), - Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) => Err(Error::InvalidOperation), + Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) | Self::Channel(_) => { + Err(Error::InvalidOperation) + } Self::Directory(_) => Err(Error::IsADirectory), } } @@ -232,7 +263,9 @@ impl Seek for File { match self { Self::Regular(file) => file.seek(from), Self::Block(file) => file.seek(from), - Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) => Err(Error::InvalidOperation), + Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) | Self::Channel(_) => { + Err(Error::InvalidOperation) + } Self::Directory(_) => Err(Error::IsADirectory), } } @@ -261,6 +294,7 @@ impl fmt::Debug for File { Self::Directory(_) => f.debug_struct("DirectoryFile").finish_non_exhaustive(), Self::AnonymousPipe(_) => f.debug_struct("AnonymousPipe").finish_non_exhaustive(), Self::Poll(_) => f.debug_struct("Poll").finish_non_exhaustive(), + Self::Channel(_) => f.debug_struct("Channel").finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index f40858d2..b6458655 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -241,6 +241,23 @@ impl IoContext { parent.create(&create_info, access) } + /// Creates an arbitrary node at given path + pub fn create_node>( + &mut self, + at: Option, + path: P, + node: NodeRef, + ) -> Result<(), Error> { + let path = path.as_ref(); + let (parent, name) = path.split_right(); + let parent = self.find(at, parent, true, true)?; + let access = self.check_access(Action::Write, &parent)?; + + parent.create_node(node, name, access)?; + + Ok(()) + } + fn remove_entry>( &mut self, at: Option, diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index b4fed41c..e1dbcd35 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -10,6 +10,7 @@ extern crate hosted_tests; extern crate alloc; +pub(crate) mod channel; pub(crate) mod device; pub(crate) mod file; pub(crate) mod ioctx; diff --git a/lib/vfs/src/node/access.rs b/lib/vfs/src/node/access.rs index dc928029..76a72430 100644 --- a/lib/vfs/src/node/access.rs +++ b/lib/vfs/src/node/access.rs @@ -8,6 +8,7 @@ use yggdrasil_abi::io::{FileMode, GroupId, UserId}; use super::Metadata; /// Zero-sized token type used to ensure checked access to node functions +#[derive(Clone)] pub struct AccessToken(PhantomData<()>); #[allow(missing_docs)] diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs index ff7860f3..827a9f87 100644 --- a/lib/vfs/src/node/ops.rs +++ b/lib/vfs/src/node/ops.rs @@ -1,5 +1,6 @@ use core::mem::MaybeUninit; +use alloc::string::String; use yggdrasil_abi::{ error::Error, io::{DeviceRequest, DirectoryEntry, FileMode, GroupId, OpenOptions, UserId}, @@ -82,15 +83,33 @@ impl Node { let node = directory.imp.create_node(self, info.ty)?; // Fill out the node info - node.set_access(Some(info.uid), Some(info.gid), Some(info.mode), check)?; + node.set_access( + Some(info.uid), + Some(info.gid), + Some(info.mode), + check.clone(), + )?; - match directory.imp.attach_node(self, &node, &info.name) { + self.create_node(node, &info.name, check) + } + + /// Attaches a pre-created node to its parent + pub fn create_node( + self: &NodeRef, + node: NodeRef, + name: &str, + _check: AccessToken, + ) -> Result { + let directory = self.as_directory()?; + + match directory.imp.attach_node(self, &node, name) { Ok(_) | Err(Error::NotImplemented) => (), Err(err) => return Err(err), } // Attach the created node to the directory in memory cache - self.add_child(&info.name, node.clone())?; + self.add_child(name, node.clone())?; + Ok(node) } diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index 1becf376..f96d9a83 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -15,7 +15,8 @@ use yggdrasil_abi::{error::Error, io::RawFd}; use crate::FileRef; -/// +/// Poll channel implementation. Allows blocking until a file descriptor signals an event or a +/// timeout is reached. pub struct FdPoll { fds: Mutex>, } diff --git a/lib/vfs/src/traits.rs b/lib/vfs/src/traits.rs index 73a5804b..be192eed 100644 --- a/lib/vfs/src/traits.rs +++ b/lib/vfs/src/traits.rs @@ -1,9 +1,5 @@ -use core::{ - pin::Pin, - task::{Context, Poll}, -}; +use core::task::{Context, Poll}; -use futures_util::Future; use yggdrasil_abi::{error::Error, io::SeekFrom}; /// Immutable read interface for VFS objects @@ -42,8 +38,8 @@ pub trait Seek { } } -/// +/// Interface for polling files for events pub trait FileReadiness: Sync { - /// + /// Polls file's "read-ready" status fn poll_read(&self, cx: &mut Context<'_>) -> Poll>; } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index da60007a..d222e732 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -514,6 +514,7 @@ impl X86_64 { self.tty.init(CombinedTerminal::new(self.fbconsole.get())); + devfs::add_named_block_device(self.framebuffer.get(), "fb0")?; devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular)?; console::add_console_autoflush(self.fbconsole.get()); diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 9336b750..33d5e19d 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -154,7 +154,9 @@ impl FramebufferConsole { impl Inner { #[optimize(speed)] fn draw_glyph(&mut self, font: PcScreenFont<'static>, g: DrawGlyph) { - let mut fb = unsafe { self.framebuffer.lock() }; + let Some(mut fb) = (unsafe { self.framebuffer.lock() }) else { + return; + }; let mut c = g.c as u32; if c >= font.len() { @@ -184,7 +186,9 @@ impl Inner { #[optimize(speed)] fn fill_rect(&mut self, x: u32, y: u32, w: u32, h: u32, val: u32) { - let mut fb = unsafe { self.framebuffer.lock() }; + let Some(mut fb) = (unsafe { self.framebuffer.lock() }) else { + return; + }; for i in 0..h { let row = &mut fb[i + y]; diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index f7e045e2..9b5d81aa 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -1,20 +1,29 @@ //! Abstract linear framebuffer device implementation -use core::ops::{Index, IndexMut}; +use core::{ + ops::{Index, IndexMut}, + task::{Context, Poll}, +}; -use abi::error::Error; +use abi::{error::Error, io::DeviceRequest}; use device_api::Device; use kernel_util::{ - mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}, + mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel}, sync::IrqSafeSpinlock, }; +use ygg_driver_block::BlockDevice; + +use crate::{ + arch::L3, + task::{process::ProcessId, thread::Thread}, +}; use super::{DisplayDevice, DisplayDimensions}; struct Inner { - dimensions: DisplayDimensions, base: usize, stride: usize, + holder: Option, } #[doc(hidden)] @@ -27,6 +36,9 @@ pub struct FramebufferAccess { /// Linear framebuffer wrapper pub struct LinearFramebuffer { inner: IrqSafeSpinlock, + phys_base: PhysicalAddress, + dimensions: DisplayDimensions, + size: usize, } impl LinearFramebuffer { @@ -36,26 +48,29 @@ impl LinearFramebuffer { /// /// Unsafe: the caller must ensure the validity of all the arguments. pub unsafe fn from_physical_bits( - base: PhysicalAddress, + phys_base: PhysicalAddress, size: usize, stride: usize, width: u32, height: u32, ) -> Result { - let base = unsafe { RawDeviceMemoryMapping::map(base, size) }?.leak(); + let base = unsafe { RawDeviceMemoryMapping::map(phys_base, size) }?.leak(); let inner = Inner { - dimensions: DisplayDimensions { width, height }, base, stride, + holder: None, }; let res = Self { inner: IrqSafeSpinlock::new(inner), + dimensions: DisplayDimensions { width, height }, + phys_base, + size, }; // Clear the screen - res.lock().fill_rows(0, height, 0); + res.lock().unwrap().fill_rows(0, height, 0); Ok(res) } @@ -66,13 +81,69 @@ impl LinearFramebuffer { /// /// Unsafe: access is not synchronized // TODO doesn't actually lock - pub unsafe fn lock(&self) -> FramebufferAccess { + pub unsafe fn lock(&self) -> Option { let inner = self.inner.lock(); - FramebufferAccess { - dimensions: inner.dimensions, - base: inner.base, - stride: inner.stride, + if inner.holder.is_some() { + None + } else { + Some(FramebufferAccess { + dimensions: self.dimensions, + base: inner.base, + stride: inner.stride, + }) + } + } +} + +impl BlockDevice for LinearFramebuffer { + fn size(&self) -> Result { + Ok(self.size as _) + } + + fn poll_read( + &self, + cx: &mut Context<'_>, + pos: u64, + buf: &mut [u8], + ) -> Poll> { + todo!() + } + + fn poll_write(&self, cx: &mut Context<'_>, pos: u64, buf: &[u8]) -> Poll> { + todo!() + } + + fn is_readable(&self) -> bool { + false + } + + fn is_writable(&self) -> bool { + false + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + let thread = Thread::current(); + match req { + DeviceRequest::AcquireDevice => { + self.inner.lock().holder.replace(thread.process().id()); + Ok(()) + } + _ => todo!(), + } + } + + fn get_page(&self, offset: u64) -> Result { + let offset = offset as usize; + if offset + L3::SIZE > self.size { + warnln!( + "Tried to map offset {:#x}, but size is {:#x}", + offset, + self.size + ); + Err(Error::InvalidMemoryOperation) + } else { + Ok(self.phys_base.add(offset)) } } } @@ -85,7 +156,7 @@ impl Device for LinearFramebuffer { impl DisplayDevice for LinearFramebuffer { fn dimensions(&self) -> DisplayDimensions { - self.inner.lock().dimensions + self.dimensions } } diff --git a/src/device/tty.rs b/src/device/tty.rs index b5ffc042..76f895f6 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -13,15 +13,13 @@ use abi::{ use device_api::serial::SerialDevice; use futures_util::Future; use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; -use vfs::FileReadiness; use crate::{ task::process::{Process, ProcessId}, util::ring::RingBuffer, }; -/// -pub struct TerminalRing { +struct TerminalRing { buffer: IrqSafeSpinlock>, eof: AtomicBool, notify: QueueWaker, @@ -30,8 +28,7 @@ pub struct TerminalRing { impl TerminalRing { const CAPACITY: usize = 128; - /// - pub const fn new() -> Self { + const fn new() -> Self { Self { buffer: IrqSafeSpinlock::new(RingBuffer::new(0)), eof: AtomicBool::new(false), @@ -39,27 +36,23 @@ impl TerminalRing { } } - /// - pub fn write(&self, ch: u8, signal: bool) { + fn write(&self, ch: u8, signal: bool) { self.buffer.lock().write(ch); if signal { self.notify.wake_one(); } } - /// - pub fn signal(&self) { + fn signal(&self) { self.notify.wake_all(); } - /// - pub fn signal_eof(&self) { + fn signal_eof(&self) { self.eof.store(true, Ordering::Release); self.notify.wake_all(); } - /// - pub fn read_blocking<'a>(&'a self) -> impl Future> + 'a { + fn read_blocking<'a>(&'a self) -> impl Future> + 'a { struct F<'f> { ring: &'f TerminalRing, } @@ -115,7 +108,7 @@ pub mod combined { io::{DeviceRequest, TerminalLineOptions, TerminalSize}, }; use device_api::{input::KeyboardConsumer, serial::SerialDevice}; - use kernel_util::{block, runtime::QueueWaker}; + use kernel_util::block; use vfs::{CharDevice, FileReadiness}; // use vfs::CharDevice; @@ -410,7 +403,7 @@ impl TtyContext { } } - /// + /// Signals an event on the terminal pub fn signal(&self) { self.ring.signal() } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 22811030..2c21f26f 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,14 +1,23 @@ //! Filesystem implementations -use core::ptr::NonNull; +use core::{ + pin::Pin, + ptr::NonNull, + task::{Context, Poll}, +}; +use alloc::boxed::Box; +use futures_util::Future; use kernel_fs::devfs; -use kernel_util::{mem::address::PhysicalAddress, util::OneTimeInit}; +use kernel_util::{ + block, mem::address::PhysicalAddress, runtime::QueueWaker, sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use memfs::block::{self, BlockAllocator}; -use vfs::{impls::read_fn_node, NodeRef}; +use vfs::{impls::read_fn_node, CharDevice, FileReadiness, Node, NodeFlags, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::{mem::phys, proc::random}; +use crate::{mem::phys, proc::random, util::ring::RingBuffer}; // pub mod devfs; pub mod sysfs; diff --git a/src/init.rs b/src/init.rs index bbc08f8c..60dc6826 100644 --- a/src/init.rs +++ b/src/init.rs @@ -10,7 +10,7 @@ use vfs::{Action, IoContext, NodeRef}; use crate::{ fs::{FileBlockAllocator, INITRD_DATA}, - proc, + proc::{self, random}, }; fn setup_root() -> Result { @@ -38,6 +38,8 @@ pub fn kinit() -> Result<(), Error> { })?; } + random::init(); + let root = setup_root()?; let mut ioctx = IoContext::new(root); diff --git a/src/main.rs b/src/main.rs index 69bafe83..3cc4eba6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -92,8 +92,6 @@ pub fn kernel_secondary_main() -> ! { pub fn kernel_main() -> ! { debugln!("Heap: {:#x?}", heap::heap_range()); - random::init(); - // Setup the sysfs sysfs::init(); fs::add_pseudo_devices().unwrap(); diff --git a/src/mem/process.rs b/src/mem/process.rs index cdd3d66b..6f1afffa 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -77,7 +77,7 @@ impl Inner { ) -> Result<(), (usize, Error)> { for i in 0..page_count { let virt = address + i * ProcessAddressSpaceImpl::PAGE_SIZE; - let phys = match get_page(virt) { + let phys = match get_page(i * ProcessAddressSpaceImpl::PAGE_SIZE) { Ok(page) => page, Err(err) => { return Err((i, err)); @@ -146,6 +146,7 @@ impl Inner { // TODO handle failures here? let phys = self.table.unmap_page(virt).unwrap(); + // TODO this will fail spectacularly for memory-mapped files and devices free_page(virt, phys); } diff --git a/src/proc/random.rs b/src/proc/random.rs index 6bba63de..b80c896e 100644 --- a/src/proc/random.rs +++ b/src/proc/random.rs @@ -4,8 +4,10 @@ use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::arch::{Architecture, ARCHITECTURE}; +const BUFFER_SIZE: usize = 1024; + struct RandomState { - data: [u8; 1024], + data: [u8; BUFFER_SIZE], pos: usize, last_state: u32, } @@ -13,8 +15,8 @@ struct RandomState { impl RandomState { fn new(seed: u32) -> Self { Self { - data: [0; 1024], - pos: 1024, + data: [0; BUFFER_SIZE], + pos: BUFFER_SIZE, last_state: seed, } } @@ -41,10 +43,10 @@ impl RandomState { let mut pos = 0; while rem != 0 { - if self.pos == 1024 { + if self.pos == BUFFER_SIZE { self.fill_buf(); } - let count = core::cmp::min(rem, 1024 - self.pos); + let count = core::cmp::min(rem, BUFFER_SIZE - self.pos); buf[pos..pos + count].copy_from_slice(&self.data[self.pos..self.pos + count]); diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index ef4716a2..ecf58520 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -13,15 +13,18 @@ use abi::{ }; use alloc::sync::Arc; use kernel_util::{ - block, runtime, + block, + mem::table::EntryLevelExt, + runtime, sync::{IrqSafeSpinlockGuard, LockMethod}, }; use vfs::{File, IoContext, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ + arch::L3, debug::LogLevel, - fs, + fs::{self, NamedPipe}, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ @@ -90,14 +93,15 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } // Resource management SyscallFunction::MapMemory => { - let len = args[1] as usize; + let mut len = args[1] as usize; let source = arg_user_ref::(args[2] as usize)?; let space = thread.address_space(); - if len & 0xFFF != 0 { - todo!(); - } + len = len.page_align_up::(); + // if len & 0xFFF != 0 { + // todo!(); + // } match source { MappingSource::Anonymous => space.allocate( @@ -108,8 +112,19 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, ), - MappingSource::File(fd, offset) => run_with_io(process, |io| { - todo!(); + &MappingSource::File(fd, offset) => run_with_io(process, |io| { + let file = io.files.file(fd)?; + log::info!("len = {:#x}", len); + + space.allocate( + None, + len, + |off| file.get_page(offset + off as u64), + // TODO make get_page return attributes for each page itself + MapAttributes::USER_WRITE + | MapAttributes::USER_READ + | MapAttributes::NON_GLOBAL, + ) }), } } @@ -352,6 +367,47 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) }) } + SyscallFunction::OpenChannel => { + let name = arg_user_str(args[0] as usize, args[1] as usize)?; + let with_sub = args[2] != 0; + + debugln!("OpenChannel {:?}, with_sub={}", name, with_sub); + + run_with_io(process, |mut io| { + let file = File::new_message_channel(name, with_sub); + let fd = io.files.place_file(file)?; + + Ok(fd.0 as usize) + }) + } + SyscallFunction::SendMessage => { + let fd = RawFd::from(args[0] as u32); + let buf = arg_buffer_ref(args[1] as usize, args[2] as usize)?; + + debugln!("SendMessage {:?}, len={}", fd, buf.len()); + + run_with_io(process, |io| { + let file = io.files.file(fd)?; + let channel = file.as_message_channel()?; + + channel.send_message(buf)?; + + Ok(0) + }) + } + SyscallFunction::ReceiveMessage => { + let fd = RawFd::from(args[0] as u32); + let buf = arg_buffer_mut(args[1] as usize, args[2] as usize)?; + + debugln!("ReceiveMessage {:?}, len = {}", fd, buf.len()); + + run_with_io(process, |io| { + let file = io.files.file(fd)?; + let channel = file.as_message_channel()?; + + channel.receive_message(buf) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; diff --git a/src/util/ring.rs b/src/util/ring.rs index c49c4c92..93471639 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -1,15 +1,5 @@ //! Ring buffer implementation -use core::{ - pin::Pin, - task::{Context, Poll}, -}; - -use abi::error::Error; -use alloc::sync::Arc; -use futures_util::Future; -use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; - /// Ring buffer base pub struct RingBuffer { rd: usize, @@ -17,13 +7,6 @@ pub struct RingBuffer { data: [T; N], } -// /// Ring buffer with async read support -// pub struct AsyncRing { -// inner: Arc>>, -// /// -// pub read_waker: Arc, -// } -// impl RingBuffer { /// Constructs a new [RingBuffer] and fills it with `value` pub const fn new(value: T) -> Self { @@ -42,7 +25,7 @@ impl RingBuffer { } } - /// + /// Returns `true` if the ring has data that can be read #[inline] pub const fn is_readable(&self) -> bool { self.is_readable_at(self.rd) @@ -56,7 +39,13 @@ impl RingBuffer { } } - /// + /// Returns `true` if the ring is not full + #[inline] + pub fn is_writable(&self) -> bool { + (self.wr + 1) % N != self.rd + } + + /// Returns `true` if the ring contains given element pub fn contains(&self, t: T) -> bool where T: PartialEq, @@ -72,7 +61,7 @@ impl RingBuffer { false } - /// + /// Reads a single value from the ring without checking if it's empty #[inline] pub fn read_single_unchecked(&mut self) -> T { let res = self.data[self.rd]; @@ -99,58 +88,3 @@ impl RingBuffer { self.wr = (self.wr + 1) % N; } } - -// impl AsyncRing { -// /// Constructs a new [AsyncRing] and fills it with `value` -// pub fn new(value: T) -> Self { -// Self { -// inner: Arc::new(IrqSafeSpinlock::new(RingBuffer::new(value))), -// read_waker: Arc::new(QueueWaker::new()), -// } -// } -// -// /// Writes a single entry to the buffer and signals readers -// pub fn try_write(&self, item: T) -> Result<(), Error> { -// { -// let mut lock = self.inner.lock(); -// lock.write(item); -// } -// self.read_waker.wake_one(); -// -// Ok(()) -// } -// -// /// Asynchronously reads an entry from the buffer -// pub fn read(&self) -> impl Future { -// struct ReadFuture { -// inner: Arc>>, -// read_waker: Arc, -// } -// -// impl Future for ReadFuture { -// type Output = T; -// -// fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { -// self.read_waker.register(cx.waker()); -// -// let mut inner = self.inner.lock(); -// if inner.is_readable() { -// self.read_waker.remove(cx.waker()); -// Poll::Ready(inner.read_single_unchecked()) -// } else { -// Poll::Pending -// } -// } -// } -// -// ReadFuture { -// inner: self.inner.clone(), -// read_waker: self.read_waker.clone(), -// } -// } -// -// /// -// pub fn can_read_now(&self) -> bool { -// self.inner.lock().is_readable() -// } -// } From b3e7d2848b04584678109e4f39b83474cf1ae1ce Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 26 Dec 2023 22:24:38 +0200 Subject: [PATCH 136/211] refactor: fix warnings --- driver/block/core/src/device.rs | 4 +++- driver/block/core/src/lib.rs | 15 +++++---------- lib/kernel-util/src/runtime/executor.rs | 4 +--- lib/kernel-util/src/sync/mod.rs | 6 ++++-- lib/vfs/src/device.rs | 2 -- lib/vfs/src/file/device.rs | 4 ++-- lib/vfs/src/file/mod.rs | 2 +- lib/vfs/src/lib.rs | 2 +- lib/vfs/src/node/ops.rs | 1 - src/arch/mod.rs | 5 +---- src/arch/x86_64/cpu.rs | 9 +++------ src/device/display/linear_fb.rs | 13 +++++++++---- src/device/tty.rs | 11 ++++++----- src/fs/mod.rs | 17 ++++------------- src/main.rs | 1 - src/proc/io.rs | 2 -- src/syscall/mod.rs | 16 ++++------------ src/task/thread.rs | 2 +- 18 files changed, 45 insertions(+), 71 deletions(-) diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index 3efdcac9..c0dd5579 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -1,3 +1,5 @@ +#![allow(unused)] + use core::{ ops::Range, pin::Pin, @@ -6,7 +8,7 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{block, mem::PageBox, runtime::QueueWaker}; +use kernel_util::{mem::PageBox, runtime::QueueWaker}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::{ diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index aa12f1af..1af33678 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -2,13 +2,9 @@ extern crate alloc; -use core::{ - pin::Pin, - task::{Context, Poll}, -}; +use core::task::{Context, Poll}; -use alloc::{boxed::Box, vec::Vec}; -use kernel_util::{mem::address::PhysicalAddress, runtime}; +use kernel_util::mem::address::PhysicalAddress; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; @@ -18,14 +14,13 @@ pub mod request; pub use device::{NgBlockDevice, NgBlockDeviceWrapper}; pub use request::{IoOperation, IoRequest, IoSubmissionId}; -// use crate::partition::Partition; - +// TODO pub fn probe_partitions< D: NgBlockDevice + 'static, F: Fn(usize, &'static dyn BlockDevice) -> Result<(), Error> + Send + 'static, >( - dev: &'static NgBlockDeviceWrapper, - callback: F, + _dev: &'static NgBlockDeviceWrapper, + _callback: F, ) -> Result<(), Error> { Ok(()) // async fn probe_table( diff --git a/lib/kernel-util/src/runtime/executor.rs b/lib/kernel-util/src/runtime/executor.rs index f0d6c06e..e0b053cd 100644 --- a/lib/kernel-util/src/runtime/executor.rs +++ b/lib/kernel-util/src/runtime/executor.rs @@ -4,7 +4,7 @@ use alloc::{boxed::Box, format, sync::Arc}; use futures_util::{task::waker_ref, Future}; use yggdrasil_abi::error::Error; -use crate::thread::{CurrentThread, Thread}; +use crate::thread::Thread; use super::{ task::{Task, Termination}, @@ -42,8 +42,6 @@ pub fn spawn + Send + 'static>( enqueue(Task::new(future)) } -struct CurrentThreadWaker(CurrentThread); - /// Runs a [Future] to its completion on the current thread pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { let mut future = Box::pin(future); diff --git a/lib/kernel-util/src/sync/mod.rs b/lib/kernel-util/src/sync/mod.rs index 242aaa84..b9df129b 100644 --- a/lib/kernel-util/src/sync/mod.rs +++ b/lib/kernel-util/src/sync/mod.rs @@ -12,8 +12,6 @@ pub use fence::SpinFence; pub use guard::IrqGuard; pub use spinlock::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; -// use crate::arch::{Architecture, ArchitectureImpl}; - static LOCK_HACK: AtomicBool = AtomicBool::new(false); pub trait LockMethod<'q>: Sync { @@ -23,6 +21,10 @@ pub trait LockMethod<'q>: Sync { Self: 'a; fn lock(&'q self) -> Result, Error>; + + /// # Safety + /// + /// Only meant to be called from Guard's [Drop] impl. unsafe fn release(&self); } diff --git a/lib/vfs/src/device.rs b/lib/vfs/src/device.rs index f940300d..c45a291e 100644 --- a/lib/vfs/src/device.rs +++ b/lib/vfs/src/device.rs @@ -1,5 +1,3 @@ -use futures_util::Future; -use kernel_util::runtime::QueueWaker; use ygg_driver_block::BlockDevice; use yggdrasil_abi::{error::Error, io::DeviceRequest}; diff --git a/lib/vfs/src/file/device.rs b/lib/vfs/src/file/device.rs index af2a261b..1c639ea3 100644 --- a/lib/vfs/src/file/device.rs +++ b/lib/vfs/src/file/device.rs @@ -22,7 +22,7 @@ pub struct CharFile { } impl BlockFile { - pub fn read(&self, buf: &mut [u8]) -> Result { + pub fn read(&self, _buf: &mut [u8]) -> Result { todo!() // let mut position = self.position.lock(); // let count = self.device.0.read(*position, buf)?; @@ -30,7 +30,7 @@ impl BlockFile { // Ok(count) } - pub fn write(&self, buf: &[u8]) -> Result { + pub fn write(&self, _buf: &[u8]) -> Result { todo!() // let mut position = self.position.lock(); // let count = self.device.0.write(*position, buf)?; diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index f76726b7..93e5f949 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -17,7 +17,7 @@ use yggdrasil_abi::{ }; use crate::{ - channel::{Channel, ChannelDescriptor}, + channel::ChannelDescriptor, device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, traits::{Read, Seek, Write}, diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index e1dbcd35..88ee5468 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,7 +1,7 @@ //! Virtual filesystem interfaces and driver implementation #![cfg_attr(not(test), no_std)] -#![allow(clippy::new_ret_no_self)] +#![allow(clippy::new_ret_no_self, clippy::new_without_default)] #![deny(missing_docs)] #![feature(if_let_guard, maybe_uninit_slice, trait_alias, let_chains)] diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs index 827a9f87..11c1b3fa 100644 --- a/lib/vfs/src/node/ops.rs +++ b/lib/vfs/src/node/ops.rs @@ -1,6 +1,5 @@ use core::mem::MaybeUninit; -use alloc::string::String; use yggdrasil_abi::{ error::Error, io::{DeviceRequest, DirectoryEntry, FileMode, GroupId, OpenOptions, UserId}, diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 4b7eab52..d92dc897 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,9 +1,6 @@ //! Provides architecture/platform-specific implementation details -use core::{ - ops::{Deref, DerefMut}, - time::Duration, -}; +use core::{ops::DerefMut, time::Duration}; use abi::error::Error; diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 855d1d63..f8689381 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -7,7 +7,7 @@ use core::{ use alloc::{boxed::Box, vec::Vec}; use kernel_util::{ - sync::{guard::GuardedRef, IrqGuard, IrqSafeSpinlock}, + sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; use tock_registers::interfaces::Writeable; @@ -149,11 +149,8 @@ impl CpuAccess for Cpu { let guard = IrqGuard::acquire(); unsafe { core::arch::asm!("movq %gs:(0), {0}", out(reg) addr, options(att_syntax)); - if let Some(cpu) = (addr as *mut Cpu).as_mut() { - Some(LocalCpu(cpu, guard)) - } else { - None - } + + (addr as *mut Cpu).as_mut().map(|cpu| LocalCpu(cpu, guard)) } } diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 9b5d81aa..6db637bc 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -103,14 +103,19 @@ impl BlockDevice for LinearFramebuffer { fn poll_read( &self, - cx: &mut Context<'_>, - pos: u64, - buf: &mut [u8], + _cx: &mut Context<'_>, + _pos: u64, + _buf: &mut [u8], ) -> Poll> { todo!() } - fn poll_write(&self, cx: &mut Context<'_>, pos: u64, buf: &[u8]) -> Poll> { + fn poll_write( + &self, + _cx: &mut Context<'_>, + _pos: u64, + _buf: &[u8], + ) -> Poll> { todo!() } diff --git a/src/device/tty.rs b/src/device/tty.rs index 76f895f6..e064556b 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -47,12 +47,13 @@ impl TerminalRing { self.notify.wake_all(); } - fn signal_eof(&self) { - self.eof.store(true, Ordering::Release); - self.notify.wake_all(); - } + // TODO EOF not yet implemented + // fn signal_eof(&self) { + // self.eof.store(true, Ordering::Release); + // self.notify.wake_all(); + // } - fn read_blocking<'a>(&'a self) -> impl Future> + 'a { + fn read_blocking(&self) -> impl Future> + '_ { struct F<'f> { ring: &'f TerminalRing, } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 2c21f26f..22811030 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,23 +1,14 @@ //! Filesystem implementations -use core::{ - pin::Pin, - ptr::NonNull, - task::{Context, Poll}, -}; +use core::ptr::NonNull; -use alloc::boxed::Box; -use futures_util::Future; use kernel_fs::devfs; -use kernel_util::{ - block, mem::address::PhysicalAddress, runtime::QueueWaker, sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use kernel_util::{mem::address::PhysicalAddress, util::OneTimeInit}; use memfs::block::{self, BlockAllocator}; -use vfs::{impls::read_fn_node, CharDevice, FileReadiness, Node, NodeFlags, NodeRef}; +use vfs::{impls::read_fn_node, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::{mem::phys, proc::random, util::ring::RingBuffer}; +use crate::{mem::phys, proc::random}; // pub mod devfs; pub mod sysfs; diff --git a/src/main.rs b/src/main.rs index 3cc4eba6..b20e03fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,7 +41,6 @@ use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, fs::sysfs, mem::heap, - proc::random, task::{spawn_kernel_closure, Cpu}, }; diff --git a/src/proc/io.rs b/src/proc/io.rs index 52eb2528..19a06351 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -1,7 +1,5 @@ //! Process I/O management -use abi::{error::Error, io::RawFd}; -use alloc::collections::{btree_map::Entry, BTreeMap}; use vfs::{FileSet, IoContext}; /// I/O context of a process, contains information like root, current directory and file diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index ecf58520..96715792 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -12,19 +12,14 @@ use abi::{ syscall::SyscallFunction, }; use alloc::sync::Arc; -use kernel_util::{ - block, - mem::table::EntryLevelExt, - runtime, - sync::{IrqSafeSpinlockGuard, LockMethod}, -}; +use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; use vfs::{File, IoContext, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ arch::L3, debug::LogLevel, - fs::{self, NamedPipe}, + fs, mem::{phys, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ @@ -223,11 +218,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io(process, |mut io| { let res = io.files.close_file(fd); - match res { - Err(Error::InvalidFile) => { - warnln!("Double close of fd {:?} in process {}", fd, process.id()); - } - _ => (), + if res == Err(Error::InvalidFile) { + warnln!("Double close of fd {:?} in process {}", fd, process.id()); } res?; diff --git a/src/task/thread.rs b/src/task/thread.rs index 2e863e7a..5beb802f 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -297,7 +297,7 @@ impl Thread { /// # Safety /// /// Only meant to be called from within the scheduler. - pub unsafe fn set_running(&self, cpu: u32) { + pub unsafe fn set_running(&self, _cpu: u32) { self.state.store(ThreadState::Running, Ordering::Release); } From 34a1b7b9974fd5b50a5ce81aaa6715d66d078008 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 28 Dec 2023 10:37:35 +0200 Subject: [PATCH 137/211] WIP: keyboard rework --- lib/device-api/src/input.rs | 8 -- lib/vfs/src/file/mod.rs | 5 +- src/arch/x86_64/mod.rs | 10 +- src/arch/x86_64/peripherals/ps2/codeset.rs | 159 +++++++++++++++++++-- src/arch/x86_64/peripherals/ps2/mod.rs | 127 +++++----------- src/debug.rs | 4 +- src/device/input.rs | 150 +++++++++++++++++++ src/device/mod.rs | 1 + src/device/tty.rs | 126 +--------------- src/init.rs | 30 ++-- src/main.rs | 5 +- src/proc/exec.rs | 13 +- src/syscall/mod.rs | 17 +-- src/util/ring.rs | 32 +++-- 14 files changed, 392 insertions(+), 295 deletions(-) create mode 100644 src/device/input.rs diff --git a/lib/device-api/src/input.rs b/lib/device-api/src/input.rs index 2df3fbc3..907357ac 100644 --- a/lib/device-api/src/input.rs +++ b/lib/device-api/src/input.rs @@ -1,11 +1,3 @@ use yggdrasil_abi::error::Error; use crate::Device; - -pub trait KeyboardConsumer { - fn handle_character(&self, data: u8) -> Result<(), Error>; -} - -pub trait KeyboardProducer: Device { - fn connect(&self, to: &'static dyn KeyboardConsumer); -} diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 93e5f949..049a2019 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -322,8 +322,9 @@ impl FileSet { } /// Associates a `file` with any available [RawFd] and returns it - pub fn place_file(&mut self, file: FileRef) -> Result { - for idx in 0..64 { + pub fn place_file(&mut self, file: FileRef, skip_stdio: bool) -> Result { + let start = if skip_stdio { 3 } else { 0 }; + for idx in start..64 { let fd = RawFd::from(idx); if let Entry::Vacant(e) = self.map.entry(fd) { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index d222e732..f4666f7d 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -5,7 +5,6 @@ use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use device_api::{ - input::KeyboardProducer, interrupt::{ExternalInterruptController, MessageInterruptController}, timer::MonotonicTimestampProviderDevice, Device, @@ -52,7 +51,6 @@ use crate::{ device::{ self, display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, - tty::CombinedTerminal, }, fs::{Initrd, INITRD_DATA}, mem::{ @@ -96,7 +94,6 @@ pub struct X86_64 { // Display framebuffer: OneTimeInit, fbconsole: OneTimeInit, - tty: OneTimeInit, ioapic: OneTimeInit, timer: OneTimeInit, @@ -111,7 +108,6 @@ pub static ARCHITECTURE: X86_64 = X86_64 { framebuffer: OneTimeInit::new(), fbconsole: OneTimeInit::new(), - tty: OneTimeInit::new(), ioapic: OneTimeInit::new(), timer: OneTimeInit::new(), @@ -438,7 +434,7 @@ impl X86_64 { } self.timer.get().init_irq()?; - ps2.connect(self.tty.get()); + // ps2.connect(self.tty.get()); ps2.init_irq()?; device::register_device(self.ioapic.get()); @@ -512,10 +508,10 @@ impl X86_64 { )?); debug::add_sink(self.fbconsole.get(), LogLevel::Info); - self.tty.init(CombinedTerminal::new(self.fbconsole.get())); + // self.tty.init(CombinedTerminal::new(self.fbconsole.get())); devfs::add_named_block_device(self.framebuffer.get(), "fb0")?; - devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular)?; + // devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular)?; console::add_console_autoflush(self.fbconsole.get()); Ok(()) diff --git a/src/arch/x86_64/peripherals/ps2/codeset.rs b/src/arch/x86_64/peripherals/ps2/codeset.rs index d3a3ca0f..714a2a8e 100644 --- a/src/arch/x86_64/peripherals/ps2/codeset.rs +++ b/src/arch/x86_64/peripherals/ps2/codeset.rs @@ -1,13 +1,148 @@ -pub static CODE_SET_1_00: &[u8] = &[ - 0x00, b'\x1b', b'1', b'2', b'3', b'4', b'5', b'6', b'7', b'8', b'9', b'0', b'-', b'=', b'\x7f', - b'\t', b'q', b'w', b'e', b'r', b't', b'y', b'u', b'i', b'o', b'p', b'[', b']', b'\n', 0x00, - b'a', b's', b'd', b'f', b'g', b'h', b'j', b'k', b'l', b';', b'\'', b'`', 0x00, b'\\', b'z', - b'x', b'c', b'v', b'b', b'n', b'm', b',', b'.', b'/', 0x00, b'*', 0x00, b' ', -]; +use abi::io::KeyboardKey as Key; -pub static CODE_SET_1_00_SHIFT: &[u8] = &[ - 0x00, b'\x1b', b'!', b'@', b'#', b'$', b'%', b'^', b'&', b'*', b'(', b')', b'_', b'+', b'\x7f', - b'\t', b'Q', b'W', b'E', b'R', b'T', b'Y', b'U', b'I', b'O', b'P', b'{', b'}', b'\n', 0x00, - b'A', b'S', b'D', b'F', b'G', b'H', b'J', b'K', b'L', b':', b'"', b'~', 0x00, b'|', b'Z', b'X', - b'C', b'V', b'B', b'N', b'M', b'<', b'>', b'?', 0x00, -]; +macro_rules! table { + { + $vis:vis static $name:ident: + [$ty:ty; $size:expr; $default:expr] = + [ $( $index:literal => $value:expr ),* $(,)? ]; + } => { + $vis static $name: [$ty; $size] = const { + let mut array = [$default; $size]; + + $( + array[$index] = $value; + )* + + array + }; + }; +} + +table! { + pub static CODE_SET_1_00: [Key; 128; Key::Unknown] = [ + 0x01 => Key::Escape, + 0x02 => Key::Char(b'1'), + 0x03 => Key::Char(b'2'), + + 0x04 => Key::Char(b'3'), + 0x05 => Key::Char(b'4'), + 0x06 => Key::Char(b'5'), + 0x07 => Key::Char(b'6'), + + 0x08 => Key::Char(b'7'), + 0x09 => Key::Char(b'8'), + 0x0A => Key::Char(b'9'), + 0x0B => Key::Char(b'0'), + + 0x0C => Key::Char(b'-'), + 0x0D => Key::Char(b'='), + 0x0E => Key::Backspace, + 0x0F => Key::Tab, + + 0x10 => Key::Char(b'q'), + 0x11 => Key::Char(b'w'), + 0x12 => Key::Char(b'e'), + 0x13 => Key::Char(b'r'), + + 0x14 => Key::Char(b't'), + 0x15 => Key::Char(b'y'), + 0x16 => Key::Char(b'u'), + 0x17 => Key::Char(b'i'), + + 0x18 => Key::Char(b'o'), + 0x19 => Key::Char(b'p'), + 0x1A => Key::Char(b'['), + 0x1B => Key::Char(b']'), + + 0x1C => Key::Enter, + 0x1D => Key::LControl, + 0x1E => Key::Char(b'a'), + 0x1F => Key::Char(b's'), + + 0x20 => Key::Char(b'd'), + 0x21 => Key::Char(b'f'), + 0x22 => Key::Char(b'g'), + 0x23 => Key::Char(b'h'), + + 0x24 => Key::Char(b'j'), + 0x25 => Key::Char(b'k'), + 0x26 => Key::Char(b'l'), + 0x27 => Key::Char(b';'), + + 0x28 => Key::Char(b'\''), + 0x29 => Key::Char(b'`'), + 0x2A => Key::LShift, + 0x2B => Key::Char(b'\\'), + + 0x2C => Key::Char(b'z'), + 0x2D => Key::Char(b'x'), + 0x2E => Key::Char(b'c'), + 0x2F => Key::Char(b'v'), + + 0x30 => Key::Char(b'b'), + 0x31 => Key::Char(b'n'), + 0x32 => Key::Char(b'm'), + 0x33 => Key::Char(b','), + + 0x34 => Key::Char(b'.'), + 0x35 => Key::Char(b'/'), + 0x36 => Key::RShift, + // 0x37 => Key::KeypadMul, + + 0x38 => Key::LAlt, + 0x39 => Key::Char(b' '), + 0x3A => Key::CapsLock, + 0x3B => Key::F(1), + + 0x3C => Key::F(2), + 0x3D => Key::F(3), + 0x3E => Key::F(4), + 0x3F => Key::F(5), + + 0x40 => Key::F(6), + 0x41 => Key::F(7), + 0x42 => Key::F(8), + 0x43 => Key::F(9), + + 0x44 => Key::F(10), + // 0x45 => Key::NumLock, + // 0x46 => Key::ScrollLock, + // 0x47 => Key::Keypad7, + + // 0x48 => Key::Keypad8, + // 0x49 => Key::Keypad9, + // 0x4A => Key::KeypadMinus, + // 0x4B => Key::Keypad4, + + // 0x4C => Key::Keypad5, + // 0x4D => Key::Keypad6, + // 0x4E => Key::KeypadPlus, + // 0x4F => Key::Keypad1, + + // 0x50 => Key::Keypad2, + // 0x51 => Key::Keypad3, + // 0x52 => Key::Keypad0, + // 0x53 => Key::KeypadPeriod, + + 0x57 => Key::F(11), + + 0x58 => Key::F(12), + ]; +} + +table! { + pub static CODE_SET_1_E0: [Key; 128; Key::Unknown] = [ + 0x1D => Key::RControl, + 0x38 => Key::RAlt, + // 0x47 => Key::Home, + // 0x48 => Key::Up, + // 0x49 => Key::PageUp, + // 0x4B => Key::Left, + // 0x4D => Key::Right, + // 0x4F => Key::End, + // 0x50 => Key::Down, + // 0x51 => Key::PageDown, + // 0x52 => Key::Insert, + // 0x53 => Key::Delete, + ]; +} diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index dccaa5f3..193fbd87 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -1,32 +1,28 @@ //! Intel 8042 PS/2 controller driver implemenation -use abi::error::Error; -use device_api::{ - input::{KeyboardConsumer, KeyboardProducer}, - interrupt::InterruptHandler, - Device, +use abi::{ + error::Error, + io::{KeyboardKey, KeyboardKeyEvent}, }; +use device_api::{interrupt::InterruptHandler, Device}; use kernel_util::sync::IrqSafeSpinlock; -use crate::arch::{ - x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - IrqNumber, +use crate::{ + arch::{ + x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, + IrqNumber, + }, + Architecture, ARCHITECTURE, }, - Architecture, ARCHITECTURE, + device::input, }; -use self::codeset::{CODE_SET_1_00, CODE_SET_1_00_SHIFT}; - mod codeset; struct Inner { command: IoPort, data: IoPort, - - shift: bool, - ctrl: bool, - - consumer: Option<&'static dyn KeyboardConsumer>, // tty: Option<&'static dyn TtyDevice<16>>, } /// PS/2 controller driver @@ -38,15 +34,13 @@ pub struct PS2Controller { inner: IrqSafeSpinlock, } -fn translate(codeset: &[u8], key: u8) -> Option { - if key as usize > codeset.len() { - return None; - } - let value = codeset[key as usize]; - if value != 0 { - Some(value) +fn translate(e0: bool, key: u8) -> KeyboardKey { + debug_assert!(key < 0x80); + + if e0 { + CODE_SET_1_E0[key as usize] } else { - None + CODE_SET_1_00[key as usize] } } @@ -82,85 +76,33 @@ impl Inner { } } -impl KeyboardProducer for PS2Controller { - fn connect(&self, to: &'static dyn KeyboardConsumer) { - self.inner.lock().consumer.replace(to); - } -} - impl InterruptHandler for PS2Controller { fn handle_irq(&self) -> bool { let mut count = 0; let mut inner = self.inner.lock(); + loop { let Some(mut scancode) = inner.try_recv() else { break; }; + count += 1; + let e0 = scancode == 0xE0; + let release = scancode >= 0x80; - if e0 { - scancode = inner.recv_timeout(100000000).unwrap(); + if release { + scancode -= 0x80; } - if !e0 { - match scancode { - // LCtrl pressed - 0x1D => { - inner.ctrl = true; - continue; - } - // LCtrl released - 0x9D => { - inner.ctrl = false; - continue; - } - // LShift pressed - 0x2A => { - inner.shift = true; - continue; - } - // LShift released - 0xAA => { - inner.shift = false; - continue; - } - _ => {} - } - } - - if scancode > 128 { - continue; - } - - let key = match (inner.shift, inner.ctrl, e0) { - // No shift, no ctrl - (false, false, false) => translate(CODE_SET_1_00, scancode).unwrap_or(0), - // Shift, no ctrl - (true, false, false) => translate(CODE_SET_1_00_SHIFT, scancode).unwrap_or(0), - // No shift, ctrl - (false, true, false) => { - let key = translate(CODE_SET_1_00, scancode).unwrap_or(0); - - if key == b'c' { - 0x3 - } else { - 0 - } - } - // Other keys - _ => 0, + let key = translate(e0, scancode); + let event = if release { + KeyboardKeyEvent::Released(key) + } else { + KeyboardKeyEvent::Pressed(key) }; - if key == 0 { - continue; - } - - if let Some(consumer) = inner.consumer { - consumer.handle_character(key).ok(); - } - - count += 1; + input::send_event(event); } count != 0 @@ -212,9 +154,6 @@ impl PS2Controller { let inner = Inner { command: IoPort::new(cmd_port), data: IoPort::new(data_port), - shift: false, - ctrl: false, - consumer: None, }; Self { @@ -224,3 +163,7 @@ impl PS2Controller { } } } + +async fn input_worker() { + loop {} +} diff --git a/src/debug.rs b/src/debug.rs index 6ceb6113..a692daa5 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -165,14 +165,14 @@ impl RingLoggerSink { const fn new() -> Self { Self { inner: IrqSafeSpinlock::new(RingLoggerInner { - data: RingBuffer::new(0), + data: RingBuffer::new(), }), } } /// Reads data from the sink without blocking and waiting for more to arrive pub fn read(&self, pos: usize, buffer: &mut [u8]) -> usize { - self.inner.lock().data.read_all_static(pos, buffer) + unsafe { self.inner.lock().data.read_all_static(pos, buffer) } } fn write_fmt(&self, args: fmt::Arguments<'_>) -> fmt::Result { diff --git a/src/device/input.rs b/src/device/input.rs new file mode 100644 index 00000000..d1b772d6 --- /dev/null +++ b/src/device/input.rs @@ -0,0 +1,150 @@ +// TODO +#![allow(missing_docs)] + +use core::{ + marker::PhantomData, + pin::Pin, + task::{Context, Poll}, +}; + +use abi::{ + error::Error, + io::{DeviceRequest, KeyboardKeyEvent}, +}; +use futures_util::{task::AtomicWaker, Future}; +use kernel_util::{ + block, + sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, +}; +use vfs::{CharDevice, FileReadiness}; + +use crate::util::ring::RingBuffer; + +// #[derive(Default)] +// pub struct InputState { +// lshift: bool, +// rshift: bool, +// lalt: bool, +// ralt: bool, +// lcontrol: bool, +// rcontrol: bool, +// } + +pub struct MpscChannel { + queue: IrqSafeSpinlock>, + notify: AtomicWaker, +} + +pub struct KeyboardDevice; + +// impl InputState { +// pub fn shift(&self) -> bool { +// self.lshift || self.rshift +// } +// +// pub fn alt(&self) -> bool { +// self.lalt || self.ralt +// } +// +// pub fn control(&self) -> bool { +// self.lcontrol || self.rcontrol +// } +// } + +impl MpscChannel { + pub const fn new() -> Self { + Self { + queue: IrqSafeSpinlock::new(RingBuffer::new()), + notify: AtomicWaker::new(), + } + } + + pub fn send(&self, value: T) { + self.queue.lock().write(value); + self.notify.wake(); + } + + pub fn poll_read( + &self, + cx: &mut Context<'_>, + ) -> Poll>> { + let lock = self.queue.lock(); + if lock.is_readable() { + return Poll::Ready(lock); + } + drop(lock); + + self.notify.register(cx.waker()); + + let lock = self.queue.lock(); + if lock.is_readable() { + Poll::Ready(lock) + } else { + Poll::Pending + } + } + + pub fn receive(&self) -> impl Future + '_ + where + T: Copy, + { + struct F<'f, T: Copy> { + channel: &'f MpscChannel, + _pd: PhantomData, + } + + impl<'f, T: Copy> Future for F<'f, T> { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.channel + .poll_read(cx) + .map(|mut lock| unsafe { lock.read_single_unchecked() }) + } + } + + F { + channel: self, + _pd: PhantomData, + } + } +} + +impl FileReadiness for KeyboardDevice { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + INPUT_QUEUE.poll_read(cx).map(|_| Ok(())) + } +} + +impl CharDevice for KeyboardDevice { + fn read(&'static self, buf: &mut [u8]) -> Result { + if buf.len() < 4 { + return Ok(0); + } + + let ev = block!(INPUT_QUEUE.receive().await)?; + + buf[..4].copy_from_slice(&ev.as_bytes()); + + Ok(4) + } + + fn is_writable(&self) -> bool { + false + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + todo!() + } + + fn is_terminal(&self) -> bool { + false + } +} + +static INPUT_QUEUE: MpscChannel = MpscChannel::new(); +pub static KEYBOARD_DEVICE: KeyboardDevice = KeyboardDevice; + +pub fn send_event(ev: KeyboardKeyEvent) { + INPUT_QUEUE.send(ev); +} diff --git a/src/device/mod.rs b/src/device/mod.rs index 7fa4d9e5..79bccf41 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -11,6 +11,7 @@ pub mod devtree; pub mod bus; pub mod display; +pub mod input; pub mod power; pub mod serial; pub mod timer; diff --git a/src/device/tty.rs b/src/device/tty.rs index e064556b..eb154f16 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -30,7 +30,7 @@ impl TerminalRing { const fn new() -> Self { Self { - buffer: IrqSafeSpinlock::new(RingBuffer::new(0)), + buffer: IrqSafeSpinlock::new(RingBuffer::new()), eof: AtomicBool::new(false), notify: QueueWaker::new(), } @@ -78,7 +78,7 @@ impl TerminalRing { if lock.is_readable() { self.ring.notify.remove(cx.waker()); - Poll::Ready(Some(lock.read_single_unchecked())) + Poll::Ready(Some(unsafe { lock.read_single_unchecked() })) } else { Poll::Pending } @@ -91,133 +91,13 @@ impl TerminalRing { fn can_read_now(&self, canonical: bool) -> bool { let lock = self.buffer.lock(); if canonical { - lock.contains(b'\n') + lock.contains(&b'\n') } else { lock.is_readable() } } } -#[cfg(feature = "fb_console")] -pub mod combined { - //! Combined console + keyboard terminal device - use core::task::{Context, Poll}; - - use crate::task::process::ProcessId; - use abi::{ - error::Error, - io::{DeviceRequest, TerminalLineOptions, TerminalSize}, - }; - use device_api::{input::KeyboardConsumer, serial::SerialDevice}; - use kernel_util::block; - use vfs::{CharDevice, FileReadiness}; - // use vfs::CharDevice; - - use crate::device::{ - display::{console::DisplayConsole, fb_console::FramebufferConsole}, - Device, - }; - - use super::{TtyContext, TtyDevice}; - - // TODO rewrite this - /// Helper device to combine a display and a keyboard input into a single terminal - pub struct CombinedTerminal { - output: &'static (dyn DisplayConsole + Sync + 'static), - context: TtyContext, - } - - impl CombinedTerminal { - /// Create a combined terminal device from a keyboard and console output devices - pub fn new(output: &'static FramebufferConsole) -> Self { - Self { - output, - context: TtyContext::new(), - } - } - } - - impl TtyDevice for CombinedTerminal { - fn context(&self) -> &TtyContext { - &self.context - } - } - - impl KeyboardConsumer for CombinedTerminal { - fn handle_character(&self, data: u8) -> Result<(), Error> { - self.recv_byte(data); - Ok(()) - } - } - - impl SerialDevice for CombinedTerminal { - fn send(&self, byte: u8) -> Result<(), Error> { - self.output.write_char(byte); - Ok(()) - } - } - - impl Device for CombinedTerminal { - fn display_name(&self) -> &'static str { - "Combined terminal device" - } - } - - impl CharDevice for CombinedTerminal { - fn read(&'static self, data: &mut [u8]) -> Result { - match block! { - self.line_read(data).await - } { - Ok(res) => res, - Err(err) => Err(err), - } - } - - fn write(&self, data: &[u8]) -> Result { - self.line_write(data) - } - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - match req { - &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(ProcessId::from(id)); - Ok(()) - } - DeviceRequest::SetTerminalOptions(config) => self.context.set_config(config), - DeviceRequest::GetTerminalOptions(config) => { - config.write(self.context.config()); - Ok(()) - } - DeviceRequest::GetTerminalSize(out) => { - let (rows, columns) = self.output.text_dimensions(); - out.write(TerminalSize { rows, columns }); - Ok(()) - } - _ => Err(Error::InvalidArgument), - } - } - } - - impl FileReadiness for CombinedTerminal { - fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { - let inner = self.context.inner.lock(); - let canonical = inner.config.line.contains(TerminalLineOptions::CANONICAL); - - self.context.ring.notify.register(cx.waker()); - - if self.context.ring.can_read_now(canonical) { - self.context.ring.notify.remove(cx.waker()); - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - } - } -} - -#[cfg(feature = "fb_console")] -pub use combined::CombinedTerminal; - struct TtyContextInner { config: TerminalOptions, process_group: Option, diff --git a/src/init.rs b/src/init.rs index 60dc6826..289ee5e8 100644 --- a/src/init.rs +++ b/src/init.rs @@ -1,14 +1,14 @@ //! Kernel main process implementation: filesystem initialization and userspace init start -use abi::{ - error::Error, - io::{OpenOptions, RawFd}, -}; +use abi::error::Error; +use alloc::borrow::ToOwned; use kernel_fs::devfs; +use kernel_util::runtime; use memfs::MemoryFilesystem; -use vfs::{Action, IoContext, NodeRef}; +use vfs::{IoContext, NodeRef}; use crate::{ + device::input, fs::{FileBlockAllocator, INITRD_DATA}, proc::{self, random}, }; @@ -31,13 +31,15 @@ pub fn kinit() -> Result<(), Error> { #[cfg(feature = "fb_console")] { use crate::device::display::console::update_consoles_task; - use kernel_util::runtime; runtime::spawn(async move { update_consoles_task().await; })?; } + // Add keyboard device + devfs::add_named_char_device(&input::KEYBOARD_DEVICE, "kbd".to_owned())?; + random::init(); let root = setup_root()?; @@ -46,30 +48,14 @@ pub fn kinit() -> Result<(), Error> { let devfs = devfs::root(); - #[cfg(target_arch = "x86_64")] - let console = ioctx.find(Some(devfs.clone()), "tty0", true, true)?; - #[cfg(target_arch = "aarch64")] - let console = ioctx.find(Some(devfs.clone()), "ttyS0", true, true)?; - - let access = ioctx.check_access(Action::Read, &console)?; - let stdin = console.open(OpenOptions::READ, access)?; - let access = ioctx.check_access(Action::Write, &console)?; - let stdout = console.open(OpenOptions::WRITE, access)?; - let stderr = stdout.clone(); - { let (user_init, user_init_main) = proc::exec::load(&mut ioctx, "/init", &["/init", "xxx"], &[])?; let mut io = user_init.io.lock(); io.set_ioctx(ioctx); - io.files.set_file(RawFd::STDIN, stdin)?; - io.files.set_file(RawFd::STDOUT, stdout)?; - io.files.set_file(RawFd::STDERR, stderr)?; drop(io); - user_init.set_session_terminal(console); - user_init_main.enqueue_somewhere(); } diff --git a/src/main.rs b/src/main.rs index b20e03fd..e1ec8010 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,10 @@ slice_split_once, iter_collect_into, iter_next_chunk, - exact_size_is_empty + exact_size_is_empty, + inline_const, + maybe_uninit_uninit_array, + const_maybe_uninit_uninit_array )] #![allow( clippy::new_without_default, diff --git a/src/proc/exec.rs b/src/proc/exec.rs index dacc0ca3..e9444000 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -153,12 +153,11 @@ fn setup_binary>( tls_address, )?; - Ok(Process::new_with_main( - name, - Arc::new(space), - context, - Some(image), - )) + let (process, main) = Process::new_with_main(name, Arc::new(space), context, Some(image)); + + infoln!("exec::setup_binary -> {:?}", process.id()); + + Ok((process, main)) } fn load_binary( @@ -184,6 +183,8 @@ pub fn load>( let path = path.as_ref(); let file = ioctx.open_exec(None, path)?; + infoln!("exec::load({:?}, {:?})", path, args); + file.seek(SeekFrom::Start(0))?; let count = file.read(&mut head)?; let head = &head[..count]; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 96715792..912ebd12 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -166,7 +166,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result process.set_session_terminal(node.clone()); } - let fd = io.files.place_file(file)?; + let fd = io.files.place_file(file, true)?; Ok(fd.0 as usize) }) } @@ -178,7 +178,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let node = io.ioctx().find(at, path, true, true)?; let access = io.ioctx().check_access(vfs::Action::Read, &node)?; let file = node.open_directory(access)?; - let fd = io.files.place_file(file)?; + let fd = io.files.place_file(file, true)?; Ok(fd.0 as usize) }) @@ -303,8 +303,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io(process, |mut io| { let (read, write) = File::new_pipe_pair(256); - let read_fd = io.files.place_file(read)?; - let write_fd = io.files.place_file(write)?; + let read_fd = io.files.place_file(read, true)?; + let write_fd = io.files.place_file(write, true)?; infoln!("Read end: {:?}", read_fd); infoln!("Write end: {:?}", write_fd); @@ -317,7 +317,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } SyscallFunction::CreatePoll => run_with_io(process, |mut io| { let poll = File::new_poll_channel(); - let fd = io.files.place_file(poll)?; + let fd = io.files.place_file(poll, true)?; Ok(fd.0 as usize) }), @@ -367,7 +367,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io(process, |mut io| { let file = File::new_message_channel(name, with_sub); - let fd = io.files.place_file(file)?; + let fd = io.files.place_file(file, true)?; Ok(fd.0 as usize) }) @@ -426,8 +426,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result for opt in options.optional { match opt { &SpawnOption::InheritFile { source, child } => { - let src_file = io.files.file(source)?; - child_io.files.set_file(child, src_file.clone())?; + if let Ok(src_file) = io.files.file(source) { + child_io.files.set_file(child, src_file.clone())?; + } } &SpawnOption::SetProcessGroup(pgroup) => { child_process.set_group_id(pgroup.into()); diff --git a/src/util/ring.rs b/src/util/ring.rs index 93471639..b8d9e1dc 100644 --- a/src/util/ring.rs +++ b/src/util/ring.rs @@ -1,19 +1,21 @@ //! Ring buffer implementation +use core::mem::MaybeUninit; + /// Ring buffer base pub struct RingBuffer { rd: usize, wr: usize, - data: [T; N], + data: [MaybeUninit; N], } -impl RingBuffer { - /// Constructs a new [RingBuffer] and fills it with `value` - pub const fn new(value: T) -> Self { +impl RingBuffer { + /// Constructs an empty [RingBuffer] + pub const fn new() -> Self { Self { rd: 0, wr: 0, - data: [value; N], + data: MaybeUninit::uninit_array(), } } @@ -46,14 +48,14 @@ impl RingBuffer { } /// Returns `true` if the ring contains given element - pub fn contains(&self, t: T) -> bool + pub fn contains(&self, t: &T) -> bool where T: PartialEq, { let count = self.readable_count_at(self.rd); for i in 0..count { - if self.data[(self.rd + i) % N] == t { + if unsafe { self.data[(self.rd + i) % N].assume_init_ref() == t } { return true; } } @@ -63,18 +65,24 @@ impl RingBuffer { /// Reads a single value from the ring without checking if it's empty #[inline] - pub fn read_single_unchecked(&mut self) -> T { - let res = self.data[self.rd]; + pub unsafe fn read_single_unchecked(&mut self) -> T + where + T: Copy, + { + let res = self.data[self.rd].assume_init(); self.rd = (self.rd + 1) % N; res } /// Reads all entries available from `pos` to the write head - pub fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize { + pub unsafe fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize + where + T: Copy, + { let mut pos = (self.rd + pos) % N; let mut off = 0; while off < buffer.len() && self.is_readable_at(pos) { - buffer[off] = self.data[pos]; + buffer[off] = self.data[pos].assume_init(); pos += 1; off += 1; } @@ -84,7 +92,7 @@ impl RingBuffer { /// Writes a single entry to the buffer #[inline] pub fn write(&mut self, ch: T) { - self.data[self.wr] = ch; + self.data[self.wr].write(ch); self.wr = (self.wr + 1) % N; } } From 21ff2616f94fc3828e783e859787bd4064089926 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 28 Dec 2023 22:32:33 +0200 Subject: [PATCH 138/211] channel: add sender identity to ReceiveMessage --- lib/vfs/src/channel.rs | 48 ++++++++++++++++++++++++++++++------------ src/syscall/mod.rs | 19 ++++++++--------- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index 322f595e..4a40b6ac 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -1,5 +1,6 @@ use core::{ pin::Pin, + sync::atomic::{AtomicU32, Ordering}, task::{Context, Poll}, }; @@ -8,22 +9,23 @@ use alloc::{ collections::{BTreeMap, VecDeque}, string::String, sync::Arc, - vec::Vec, }; use futures_util::{task::AtomicWaker, Future}; use kernel_util::{ block, sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, }; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{error::Error, io::MessageDestination}; use crate::FileReadiness; pub struct Channel { - subscriptions: Mutex>>, + last_id: AtomicU32, + subscriptions: Mutex>>, } pub struct Message { + source: u32, data: Box<[u8]>, } @@ -33,6 +35,7 @@ pub struct Subscription { } pub struct ChannelDescriptor { + id: u32, tx: Arc, rx: Option>, } @@ -40,16 +43,18 @@ pub struct ChannelDescriptor { impl ChannelDescriptor { pub fn open(name: &str, subscribe: bool) -> ChannelDescriptor { let tx = Channel::get_or_create(name.into()); + // NOTE The first one to open the channel is guaranteed to get an ID of 0 + let id = tx.last_id.fetch_add(1, Ordering::SeqCst); let rx = if subscribe { - Some(tx.subscribe()) + Some(tx.subscribe(id)) } else { None }; - Self { tx, rx } + Self { tx, rx, id } } - pub fn receive_message(&self, buf: &mut [u8]) -> Result { + pub fn receive_message(&self, buf: &mut [u8]) -> Result<(u32, usize), Error> { let Some(rx) = self.rx.as_ref() else { return Err(Error::InvalidOperation); }; @@ -63,17 +68,33 @@ impl ChannelDescriptor { buf[..len].copy_from_slice(&message.data); - Ok(len) + Ok((message.source, len)) } - pub fn send_message(&self, msg: &[u8]) -> Result<(), Error> { + pub fn send_message(&self, msg: &[u8], dst: MessageDestination) -> Result<(), Error> { let message = Arc::new(Message { + source: self.id, data: Box::from(msg), }); let lock = self.tx.subscriptions.lock()?; - for sub in lock.iter() { - sub.push_message(message.clone())?; + + match dst { + MessageDestination::Specific(id) => { + if let Some(sub) = lock.get(&id) { + sub.push_message(message)?; + } + } + MessageDestination::AllExceptSelf => { + for (&id, sub) in lock.iter() { + if id == self.id { + continue; + } + + sub.push_message(message.clone())?; + } + } + MessageDestination::All => todo!(), } Ok(()) @@ -83,7 +104,8 @@ impl ChannelDescriptor { impl Channel { fn new() -> Arc { Arc::new(Self { - subscriptions: Mutex::new(Vec::new()), + last_id: AtomicU32::new(0), + subscriptions: Mutex::new(BTreeMap::new()), }) } @@ -93,7 +115,7 @@ impl Channel { channels.entry(name).or_insert_with(Self::new).clone() } - pub fn subscribe(&self) -> Arc { + fn subscribe(&self, id: u32) -> Arc { let mut lock = self.subscriptions.lock().unwrap(); let sub = Arc::new(Subscription { @@ -101,7 +123,7 @@ impl Channel { notify: AtomicWaker::new(), }); - lock.push(sub.clone()); + lock.insert(id, sub.clone()); sub } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 912ebd12..1f1730cd 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,8 +4,8 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{ - DeviceRequest, DirectoryEntry, FileAttr, FileMode, OpenOptions, PollControl, RawFd, - SeekFrom, + DeviceRequest, DirectoryEntry, FileAttr, FileMode, MessageDestination, OpenOptions, + PollControl, RawFd, SeekFrom, }, mem::MappingSource, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, @@ -346,8 +346,6 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let timeout = arg_user_ref::>(args[1] as usize)?; let output = arg_user_mut::)>>(args[2] as usize)?; - debugln!("PollWait {:?}", timeout); - run_with_io(process, |io| { let poll_file = io.files.file(poll_fd)?; let poll = poll_file.as_poll_channel()?; @@ -375,14 +373,13 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::SendMessage => { let fd = RawFd::from(args[0] as u32); let buf = arg_buffer_ref(args[1] as usize, args[2] as usize)?; - - debugln!("SendMessage {:?}, len={}", fd, buf.len()); + let destination = MessageDestination::from(args[3]); run_with_io(process, |io| { let file = io.files.file(fd)?; let channel = file.as_message_channel()?; - channel.send_message(buf)?; + channel.send_message(buf, destination)?; Ok(0) }) @@ -390,14 +387,16 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::ReceiveMessage => { let fd = RawFd::from(args[0] as u32); let buf = arg_buffer_mut(args[1] as usize, args[2] as usize)?; - - debugln!("ReceiveMessage {:?}, len = {}", fd, buf.len()); + let from = arg_user_mut::>(args[3] as usize)?; run_with_io(process, |io| { let file = io.files.file(fd)?; let channel = file.as_message_channel()?; - channel.receive_message(buf) + let (id, len) = channel.receive_message(buf)?; + from.write(id); + + Ok(len) }) } // Process management From 293dcfea6a94032303aba04e742679dbe2a46edf Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 31 Dec 2023 01:53:43 +0200 Subject: [PATCH 139/211] channel: send files over channels --- lib/kernel-util/src/runtime/executor.rs | 12 +++++- lib/vfs/src/channel.rs | 40 ++++++++++-------- lib/vfs/src/file/mod.rs | 10 ++++- lib/vfs/src/file/regular.rs | 1 + lib/vfs/src/lib.rs | 1 + lib/vfs/src/node/impls.rs | 6 +-- src/arch/mod.rs | 8 ++-- src/syscall/mod.rs | 56 +++++++++++++++++++------ 8 files changed, 96 insertions(+), 38 deletions(-) diff --git a/lib/kernel-util/src/runtime/executor.rs b/lib/kernel-util/src/runtime/executor.rs index e0b053cd..864f2147 100644 --- a/lib/kernel-util/src/runtime/executor.rs +++ b/lib/kernel-util/src/runtime/executor.rs @@ -1,10 +1,17 @@ use core::task::{Context, Poll}; use alloc::{boxed::Box, format, sync::Arc}; -use futures_util::{task::waker_ref, Future}; +use futures_util::{ + task::{waker_ref, ArcWake, WakerRef}, + Future, +}; use yggdrasil_abi::error::Error; -use crate::thread::Thread; +use crate::{ + api::__cpu_index, + cpu_index, + thread::{CurrentThread, Thread}, +}; use super::{ task::{Task, Termination}, @@ -48,6 +55,7 @@ pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> loop { let thread = Thread::current(); + let waker = waker_ref(&thread); let context = &mut Context::from_waker(&waker); diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index 4a40b6ac..e86604fc 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -15,18 +15,29 @@ use kernel_util::{ block, sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, }; -use yggdrasil_abi::{error::Error, io::MessageDestination}; +use yggdrasil_abi::{ + error::Error, + io::{MessageDestination, ReceivedMessageMetadata}, +}; -use crate::FileReadiness; +use crate::{FileReadiness, FileRef}; pub struct Channel { last_id: AtomicU32, subscriptions: Mutex>>, } +/// Describes message payload +pub enum MessagePayload { + /// Payload contains a file + File(FileRef), + /// Payload contains byte data + Data(Box<[u8]>), +} + pub struct Message { - source: u32, - data: Box<[u8]>, + pub source: u32, + pub payload: MessagePayload, } pub struct Subscription { @@ -54,27 +65,22 @@ impl ChannelDescriptor { Self { tx, rx, id } } - pub fn receive_message(&self, buf: &mut [u8]) -> Result<(u32, usize), Error> { + pub fn receive_message(&self) -> Result, Error> { let Some(rx) = self.rx.as_ref() else { return Err(Error::InvalidOperation); }; - let message = rx.receive_message_inner()?; - let len = message.data.len(); - - if buf.len() < len { - return Err(Error::MissingData); - } - - buf[..len].copy_from_slice(&message.data); - - Ok((message.source, len)) + rx.receive_message_inner() } - pub fn send_message(&self, msg: &[u8], dst: MessageDestination) -> Result<(), Error> { + pub fn send_message( + &self, + payload: MessagePayload, + dst: MessageDestination, + ) -> Result<(), Error> { let message = Arc::new(Message { source: self.id, - data: Box::from(msg), + payload, }); let lock = self.tx.subscriptions.lock()?; diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 049a2019..4fac85d5 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -37,7 +37,7 @@ mod pipe; mod regular; /// Per-file optional instance data created when a regular file is opened -pub type InstanceData = Box; +pub type InstanceData = Arc; /// Describes the starting position of the directory pub enum DirectoryOpenPosition { @@ -159,6 +159,14 @@ impl File { }))) } + /// Clones an open file for sending it to another process + pub fn send(&self) -> Result, Error> { + match self { + Self::Regular(file) => Ok(Arc::new(Self::Regular(file.clone()))), + _ => Err(Error::InvalidOperation), + } + } + /// Reads entries from the directory pub fn read_dir(&self, entries: &mut [MaybeUninit]) -> Result { match self { diff --git a/lib/vfs/src/file/regular.rs b/lib/vfs/src/file/regular.rs index 8d56df44..f9638dbe 100644 --- a/lib/vfs/src/file/regular.rs +++ b/lib/vfs/src/file/regular.rs @@ -4,6 +4,7 @@ use yggdrasil_abi::{error::Error, io::SeekFrom}; use super::InstanceData; use crate::node::NodeRef; +#[derive(Clone)] pub struct RegularFile { pub(super) node: NodeRef, pub(super) read: bool, diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 88ee5468..44665a0a 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -19,6 +19,7 @@ pub(crate) mod path; pub(crate) mod poll; pub(crate) mod traits; +pub use channel::MessagePayload; pub use device::CharDevice; pub use file::{DirectoryOpenPosition, File, FileRef, FileSet, InstanceData}; pub use ioctx::{Action, IoContext}; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index 08a52f9a..ae53bab8 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -141,7 +141,7 @@ where return Err(Error::ReadOnly); } let t = (self.read)()?; - Ok((0, Some(Box::new(t.as_instance_data())))) + Ok((0, Some(Arc::new(t.as_instance_data())))) } fn read( @@ -240,10 +240,10 @@ where if opts.contains(OpenOptions::READ | OpenOptions::WRITE) { Err(Error::InvalidOperation) } else if opts.contains(OpenOptions::WRITE) { - Ok((0, Some(Box::new(FnNodeData::write())))) + Ok((0, Some(Arc::new(FnNodeData::write())))) } else if opts.contains(OpenOptions::READ) { let t = (self.read)()?; - Ok((0, Some(Box::new(FnNodeData::read(t))))) + Ok((0, Some(Arc::new(FnNodeData::read(t))))) } else { Err(Error::InvalidOperation) } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index d92dc897..8f21ee51 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -35,7 +35,10 @@ use kernel_util::{ sync::IrqGuard, }; -use crate::{mem::phys::PhysicalMemoryRegion, task::sched::CpuQueue}; +use crate::{ + mem::phys::PhysicalMemoryRegion, + task::{sched::CpuQueue, Cpu}, +}; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -278,8 +281,7 @@ fn __release_irq_guard(mask: bool) { #[no_mangle] fn __cpu_index() -> usize { - todo!() - // Cpu::local_id() as _ + Cpu::local().id() as _ } #[no_mangle] diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 1f1730cd..b1fc9ba6 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -5,15 +5,15 @@ use abi::{ error::Error, io::{ DeviceRequest, DirectoryEntry, FileAttr, FileMode, MessageDestination, OpenOptions, - PollControl, RawFd, SeekFrom, + PollControl, RawFd, ReceivedMessageMetadata, SeekFrom, SentMessage, }, mem::MappingSource, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, syscall::SyscallFunction, }; -use alloc::sync::Arc; +use alloc::{boxed::Box, sync::Arc}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; -use vfs::{File, IoContext, NodeRef, Read, Seek, Write}; +use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ @@ -372,31 +372,63 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } SyscallFunction::SendMessage => { let fd = RawFd::from(args[0] as u32); - let buf = arg_buffer_ref(args[1] as usize, args[2] as usize)?; - let destination = MessageDestination::from(args[3]); + let message = arg_user_ref::(args[1] as usize)?; + let destination = MessageDestination::from(args[2]); run_with_io(process, |io| { let file = io.files.file(fd)?; let channel = file.as_message_channel()?; - channel.send_message(buf, destination)?; + match message { + &SentMessage::File(fd) => { + let sent_file = io.files.file(fd)?; + + channel + .send_message(MessagePayload::File(sent_file.clone()), destination)?; + } + &SentMessage::Data(data) => { + channel.send_message(MessagePayload::Data(Box::from(data)), destination)?; + } + } Ok(0) }) } SyscallFunction::ReceiveMessage => { let fd = RawFd::from(args[0] as u32); - let buf = arg_buffer_mut(args[1] as usize, args[2] as usize)?; - let from = arg_user_mut::>(args[3] as usize)?; + let metadata = arg_user_mut::>(args[1] as usize)?; + let buf = arg_buffer_mut(args[2] as usize, args[3] as usize)?; + let from = arg_user_mut::>(args[4] as usize)?; - run_with_io(process, |io| { + run_with_io(process, |mut io| { let file = io.files.file(fd)?; let channel = file.as_message_channel()?; - let (id, len) = channel.receive_message(buf)?; - from.write(id); + let message = channel.receive_message()?; - Ok(len) + from.write(message.source); + + match &message.payload { + MessagePayload::Data(data) => { + // TODO allow truncated messages? + let len = data.len(); + if buf.len() < len { + return Err(Error::MissingData); + } + + metadata.write(ReceivedMessageMetadata::Data(len)); + buf[..len].copy_from_slice(&data); + + Ok(len) + } + MessagePayload::File(file) => { + let fd = io.files.place_file(file.clone(), true)?; + + metadata.write(ReceivedMessageMetadata::File(fd)); + + Ok(0) + } + } }) } // Process management From ae7ba554d4ebc43be3222177668060f5c3aa302e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 31 Dec 2023 12:50:16 +0200 Subject: [PATCH 140/211] mem: implement a better vmalloc --- driver/block/core/src/device.rs | 15 ++- driver/block/core/src/lib.rs | 8 +- lib/kernel-util/src/mem/mod.rs | 5 + lib/vfs/src/file/mod.rs | 29 +++-- lib/vmalloc/Cargo.toml | 2 + lib/vmalloc/src/lib.rs | 115 ++++++++++++------- src/device/display/linear_fb.rs | 13 ++- src/mem/process.rs | 192 ++++++++++++++++++++++++-------- src/proc/elf.rs | 12 +- src/proc/exec.rs | 14 ++- src/syscall/mod.rs | 37 +++--- 11 files changed, 303 insertions(+), 139 deletions(-) diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index c0dd5579..7dab9f99 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -8,7 +8,10 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{mem::PageBox, runtime::QueueWaker}; +use kernel_util::{ + mem::{address::PhysicalAddress, PageBox, PageProvider}, + runtime::QueueWaker, +}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::{ @@ -237,6 +240,16 @@ impl<'a, D: NgBlockDevice + 'a> NgBlockDeviceWrapper<'a, D> { } } +impl<'a, D: NgBlockDevice + 'a> PageProvider for NgBlockDeviceWrapper<'a, D> { + fn get_page(&self, _offset: u64) -> Result { + todo!() + } + + fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { + todo!() + } +} + impl<'a, D: NgBlockDevice + 'a> BlockDevice for NgBlockDeviceWrapper<'a, D> { fn poll_read( &self, diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index 1af33678..0ddaf8d6 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::task::{Context, Poll}; -use kernel_util::mem::address::PhysicalAddress; +use kernel_util::mem::{address::PhysicalAddress, PageProvider}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; @@ -57,7 +57,7 @@ pub fn probe_partitions< /// Block device interface #[allow(unused)] -pub trait BlockDevice: Sync { +pub trait BlockDevice: PageProvider + Sync { fn poll_read( &self, cx: &mut Context<'_>, @@ -99,10 +99,6 @@ pub trait BlockDevice: Sync { Err(Error::NotImplemented) } - fn get_page(&self, offset: u64) -> Result { - Err(Error::NotImplemented) - } - // fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> { // let count = self.read(pos, buf)?; // if count == buf.len() { diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 7392a516..972d56bc 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -16,6 +16,11 @@ pub mod device; pub mod pointer; pub mod table; +pub trait PageProvider { + fn get_page(&self, offset: u64) -> Result; + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>; +} + pub struct PageBox { value: *mut T, page_count: usize, diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 4fac85d5..4f0ad75b 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -10,7 +10,10 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use kernel_util::{mem::address::PhysicalAddress, sync::IrqSafeSpinlock}; +use kernel_util::{ + mem::{address::PhysicalAddress, PageProvider}, + sync::IrqSafeSpinlock, +}; use yggdrasil_abi::{ error::Error, io::{DirectoryEntry, OpenOptions, RawFd, SeekFrom}, @@ -196,14 +199,6 @@ impl File { } } - /// Gets a page from a file to be mapped into virtual memory - pub fn get_page(&self, offset: u64) -> Result { - match self { - Self::Block(f) => f.device.0.get_page(offset), - _ => Err(Error::NotImplemented), - } - } - /// Interprets the file as a poll channel pub fn as_poll_channel(&self) -> Result<&FdPoll, Error> { if let Self::Poll(poll) = self { @@ -223,6 +218,22 @@ impl File { } } +impl PageProvider for File { + fn get_page(&self, offset: u64) -> Result { + match self { + Self::Block(f) => f.device.0.get_page(offset), + _ => Err(Error::InvalidOperation), + } + } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + match self { + Self::Block(f) => f.device.0.release_page(offset, phys), + _ => Err(Error::InvalidOperation), + } + } +} + impl Read for File { fn read(&self, buf: &mut [u8]) -> Result { match self { diff --git a/lib/vmalloc/Cargo.toml b/lib/vmalloc/Cargo.toml index 24ba6152..b4ec8bc5 100644 --- a/lib/vmalloc/Cargo.toml +++ b/lib/vmalloc/Cargo.toml @@ -9,6 +9,8 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +discrete_range_map = { git = "https://git.alnyan.me/yggdrasil/discrete_range_map.git" } + [dev-dependencies] itertools = "0.11.0" proptest = "1.2.0" diff --git a/lib/vmalloc/src/lib.rs b/lib/vmalloc/src/lib.rs index b7b6b04f..70b918ab 100644 --- a/lib/vmalloc/src/lib.rs +++ b/lib/vmalloc/src/lib.rs @@ -1,64 +1,97 @@ +//! Virtual memory allocator for the Yggdrasil kernel. +//! +//! The allocator uses a [DiscreteRangeMap] to track the memory regions and allows attaching +//! metadata values to each region. "Touching" region coalescing is enabled through the [Eq] trait +//! implemented for [RangeData]. + +#![deny(missing_docs)] #![no_std] #![feature(linked_list_cursors, let_chains, btree_extract_if)] extern crate alloc; -use allocator::TreeAllocator; +use core::ops::Range; + +use discrete_range_map::{DiscreteRangeMap, InclusiveInterval, InclusiveRange}; use yggdrasil_abi::error::Error; -pub(crate) mod allocator; +#[cfg(target_pointer_width = "64")] +type PfnIndex = u64; -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct VirtualMemoryRange { - start_pfn: usize, - end_pfn: usize, +/// Metadata associated with an allocated memory region. The [Eq] trait is used to coalesce "equal" +/// regions if they "touch". +pub trait RangeData: Eq {} + +fn ie(from: PfnIndex, to: PfnIndex) -> InclusiveInterval { + InclusiveInterval::from(from..to) } -pub struct VirtualMemoryAllocator { - inner: TreeAllocator, +/// Main virtual memory allocator +pub struct VirtualMemoryAllocator { + map: DiscreteRangeMap, D>, + outer_range: InclusiveInterval, } -impl VirtualMemoryRange { - pub fn start_pfn(&self) -> usize { - self.start_pfn - } - - pub fn end_pfn(&self) -> usize { - self.end_pfn - } - - pub fn pfn_count(&self) -> usize { - self.end_pfn - self.start_pfn - } - - pub fn contains(&self, other: &Self) -> bool { - other.start_pfn >= self.start_pfn && other.end_pfn <= self.end_pfn - } -} - -impl VirtualMemoryAllocator { - pub fn new(start_pfn: usize, end_pfn: usize) -> Self { +impl VirtualMemoryAllocator { + /// Creates a new virtual memory allocator, bounded by `lower_limit_pfn..upper_limit_pfn` + pub fn new(lower_limit_pfn: usize, upper_limit_pfn: usize) -> Self { Self { - inner: TreeAllocator::new(start_pfn, end_pfn), + map: DiscreteRangeMap::new(), + outer_range: ie(lower_limit_pfn as _, upper_limit_pfn as _), } } - pub fn allocate(&mut self, pfn_count: usize) -> Result { - self.inner.allocate(pfn_count).ok_or(Error::OutOfMemory) + /// Allocates a contiguous range of virtual address space and associates metadata with it + pub fn allocate(&mut self, page_count: usize, data: D) -> Result { + let start_pfn = self + .map + .gaps_trimmed(self.outer_range) + .find_map(|range| { + if range.size() >= page_count as _ { + Some(range.start() as usize) + } else { + None + } + }) + .ok_or(Error::OutOfMemory)?; + + // Should not fail + self.insert(start_pfn, page_count, data)?; + + Ok(start_pfn) } - pub fn free(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { - self.inner.free(start_pfn, pfn_count) + /// Tries to insert given PF# range with its associated metadata as allocated memory, + /// returning [Error] if requested range overlaps any existing allocated ranges + pub fn insert(&mut self, start_pfn: usize, page_count: usize, data: D) -> Result<(), Error> { + let end_pfn = (start_pfn + page_count) as PfnIndex; + let start_pfn = start_pfn as PfnIndex; + + self.map + .insert_merge_touching_if_values_equal(ie(start_pfn, end_pfn), data) + .map_err(|_| Error::AlreadyExists)?; + + Ok(()) } - pub fn insert(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { - self.inner.insert(start_pfn, pfn_count) - } + /// Releases any pages overlapping the requested range, calling `release` on the ranges + pub fn free, D) -> Result<(), Error>>( + &mut self, + start_pfn: usize, + page_count: usize, + mut release: F, + ) -> Result<(), Error> + where + D: Clone, + { + let end_pfn = (start_pfn + page_count) as PfnIndex; + let start_pfn = start_pfn as PfnIndex; - pub fn ranges(&self) -> impl Iterator + '_ { - self.inner.ranges() + self.map + .cut_with_origin(ie(start_pfn, end_pfn)) + .try_for_each(|(origin, range, data)| { + let range = range.start() as usize..range.end() as usize + 1; + release(origin.start() as _, range, data) + }) } } - -#[cfg(test)] -mod tests {} diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 6db637bc..8f13b060 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -8,7 +8,9 @@ use core::{ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; use kernel_util::{ - mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel}, + mem::{ + address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, PageProvider, + }, sync::IrqSafeSpinlock, }; use ygg_driver_block::BlockDevice; @@ -137,7 +139,9 @@ impl BlockDevice for LinearFramebuffer { _ => todo!(), } } +} +impl PageProvider for LinearFramebuffer { fn get_page(&self, offset: u64) -> Result { let offset = offset as usize; if offset + L3::SIZE > self.size { @@ -148,9 +152,14 @@ impl BlockDevice for LinearFramebuffer { ); Err(Error::InvalidMemoryOperation) } else { - Ok(self.phys_base.add(offset)) + let page = self.phys_base.add(offset); + Ok(page) } } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + Ok(()) + } } impl Device for LinearFramebuffer { diff --git a/src/mem/process.rs b/src/mem/process.rs index 6f1afffa..d7b02a2b 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,11 +1,17 @@ //! Process address space structures and management functions +use core::ops::Range; + use abi::error::Error; use cfg_if::cfg_if; -use kernel_util::sync::IrqSafeSpinlock; -use vmalloc::VirtualMemoryAllocator; +use kernel_util::{ + mem::{table::EntryLevelExt, PageProvider}, + sync::IrqSafeSpinlock, +}; +use vfs::FileRef; +use vmalloc::{RangeData, VirtualMemoryAllocator}; -use crate::mem::phys; +use crate::{arch::L3, mem::phys}; use super::{table::MapAttributes, PhysicalAddress}; @@ -57,8 +63,82 @@ pub trait ProcessAddressSpaceManager: Sized { fn as_address_with_asid(&self) -> u64; } +/// Describes how the physical memory is provided for the mapping +#[derive(Clone)] +pub enum VirtualRangeBacking { + /// Memory is taken from regular "anonymous" physical memory + Anonymous, + /// Mapping is backed by file blocks/device memory + File(FileBacking), +} + +/// Describes a file-backed memory range provider +#[derive(Clone)] +pub struct FileBacking { + offset: u64, + file: FileRef, +} + +impl VirtualRangeBacking { + /// Creates a file-backed memory range provider + pub fn file(offset: u64, file: FileRef) -> Result { + if !(offset as usize).is_page_aligned_for::() { + todo!(); + } + + Ok(Self::File(FileBacking { offset, file })) + } + + /// Creates a range of anonymous memory + pub fn anonymous() -> Self { + Self::Anonymous + } +} + +impl PageProvider for VirtualRangeBacking { + fn get_page(&self, offset: u64) -> Result { + match self { + Self::Anonymous => phys::alloc_page(), + Self::File(f) => f.file.get_page(f.offset + offset), + } + } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + match self { + Self::Anonymous => unsafe { + phys::free_page(phys); + Ok(()) + }, + Self::File(f) => f.file.release_page(f.offset + offset, phys), + } + } +} + +impl PartialEq for VirtualRangeBacking { + fn eq(&self, other: &Self) -> bool { + return matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous); + } +} + +impl Eq for VirtualRangeBacking {} + +impl RangeData for VirtualRangeBacking {} + +impl core::fmt::Debug for VirtualRangeBacking { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Anonymous => f.debug_struct("VirtualRangeBacking::Anonymous").finish(), + Self::File(fb) => f + .debug_struct("VirtualRangeBacking::File") + .field("offset", &fb.offset) + .field("file", fb.file.as_ref()) + .finish(), + } + } +} + struct Inner { - allocator: VirtualMemoryAllocator, + allocator: VirtualMemoryAllocator, table: ProcessAddressSpaceImpl, } @@ -68,16 +148,17 @@ pub struct ProcessAddressSpace { } impl Inner { - fn try_map_pages Result>( + fn try_map_pages( &mut self, address: usize, page_count: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), (usize, Error)> { for i in 0..page_count { + let offset = (i * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; let virt = address + i * ProcessAddressSpaceImpl::PAGE_SIZE; - let phys = match get_page(i * ProcessAddressSpaceImpl::PAGE_SIZE) { + let phys = match backing.get_page(offset) { Ok(page) => page, Err(err) => { return Err((i, err)); @@ -92,18 +173,19 @@ impl Inner { Ok(()) } - fn map_range Result>( + fn map_range( &mut self, address: usize, page_count: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), Error> { // If inserting fails, the range cannot be mapped let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; - self.allocator.insert(start_pfn, page_count)?; + self.allocator + .insert(start_pfn, page_count, backing.clone())?; - if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) { + if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { // TODO rollback & remove the range todo!(); }; @@ -111,16 +193,41 @@ impl Inner { Ok(()) } - fn alloc_range Result>( + fn map_single( + &mut self, + address: usize, + backing: VirtualRangeBacking, + attributes: MapAttributes, + ) -> Result { + let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; + self.allocator.insert(start_pfn, 1, backing.clone())?; + + let phys = match backing.get_page(0) { + Ok(page) => page, + Err(_err) => { + // TODO rollback + todo!(); + } + }; + + if let Err(_err) = unsafe { self.table.map_page(address, phys, attributes) } { + // TODO rollback + todo!(); + } + + Ok(phys) + } + + fn alloc_range( &mut self, page_count: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { - let start_pfn = self.allocator.allocate(page_count)?; + let start_pfn = self.allocator.allocate(page_count, backing.clone())?; let address = start_pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) { + if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { // TODO rollback todo!(); }; @@ -128,27 +235,22 @@ impl Inner { Ok(address) } - unsafe fn unmap_range( - &mut self, - start_address: usize, - page_count: usize, - free_page: F, - ) -> Result<(), Error> { + unsafe fn unmap_range(&mut self, start_address: usize, page_count: usize) -> Result<(), Error> { let start_pfn = start_address / ProcessAddressSpaceImpl::PAGE_SIZE; - // Deallocate the range first - self.allocator.free(start_pfn, page_count)?; + self.allocator + .free(start_pfn, page_count, |origin_pfn, pfn_range, backing| { + for pfn in pfn_range { + let offset = ((pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; - // Then unmap it from the table - for i in 0..page_count { - let virt = start_address + i * ProcessAddressSpaceImpl::PAGE_SIZE; - // This should not fail under normal circumstances - // TODO handle failures here? - let phys = self.table.unmap_page(virt).unwrap(); + let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; + let phys = self.table.unmap_page(virt)?; - // TODO this will fail spectacularly for memory-mapped files and devices - free_page(virt, phys); - } + backing.release_page(offset, phys)?; + } + + Ok(()) + })?; Ok(()) } @@ -169,11 +271,11 @@ impl ProcessAddressSpace { /// Allocates a region of virtual memory within the address space and maps the pages to the /// ones returned from `get_page` function - pub fn allocate Result>( + pub fn allocate( &self, _hint: Option, size: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); @@ -182,17 +284,17 @@ impl ProcessAddressSpace { lock.alloc_range( size / ProcessAddressSpaceImpl::PAGE_SIZE, - get_page, + backing, attributes, ) } /// Maps a region of memory in the address space - pub fn map Result>( + pub fn map( &self, address: usize, size: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), Error> { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); @@ -203,7 +305,7 @@ impl ProcessAddressSpace { lock.map_range( address, size / ProcessAddressSpaceImpl::PAGE_SIZE, - get_page, + backing, attributes, ) } @@ -212,16 +314,14 @@ impl ProcessAddressSpace { pub fn map_single( &self, address: usize, - physical: PhysicalAddress, + backing: VirtualRangeBacking, attributes: MapAttributes, - ) -> Result<(), Error> { + ) -> Result { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); // XXX // assert!(physical.is_aligned_for::()); - self.inner - .lock() - .map_range(address, 1, |_| Ok(physical), attributes) + self.inner.lock().map_single(address, backing, attributes) } /// Returns the [PhysicalAddress] associated with given virtual `address`, @@ -245,11 +345,7 @@ impl ProcessAddressSpace { let mut lock = self.inner.lock(); - lock.unmap_range( - address, - size / ProcessAddressSpaceImpl::PAGE_SIZE, - |_, paddr| phys::free_page(paddr), - ) + lock.unmap_range(address, size / ProcessAddressSpaceImpl::PAGE_SIZE) } /// Returns the physical address of this table, with ASID applied diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 50193b1b..227b7e7d 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -12,7 +12,11 @@ use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ - mem::{phys, process::ProcessAddressSpace, table::MapAttributes}, + mem::{ + phys, + process::{ProcessAddressSpace, VirtualRangeBacking}, + table::MapAttributes, + }, task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; @@ -68,7 +72,7 @@ pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result Result { // TODO growing buffer - let phys_page = phys::alloc_page()?; - space.map_single( + let phys_page = space.map_single( virt, - phys_page, + VirtualRangeBacking::anonymous(), MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; @@ -117,7 +121,7 @@ fn setup_binary>( space.map( virt_stack_base, USER_STACK_PAGES * 0x1000, - |_| phys::alloc_page(), + VirtualRangeBacking::anonymous(), MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index b1fc9ba6..6a206946 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -20,7 +20,7 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::{phys, table::MapAttributes}, + mem::{phys, process::VirtualRangeBacking, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, @@ -94,34 +94,25 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let space = thread.address_space(); len = len.page_align_up::(); - // if len & 0xFFF != 0 { - // todo!(); - // } - match source { - MappingSource::Anonymous => space.allocate( + run_with_io(process, |io| { + let backing = match source { + MappingSource::Anonymous => VirtualRangeBacking::anonymous(), + &MappingSource::File(fd, offset) => { + let file = io.files.file(fd)?; + VirtualRangeBacking::file(offset, file.clone())? + } + }; + + space.allocate( None, len, - |_| phys::alloc_page(), + backing, MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - ), - &MappingSource::File(fd, offset) => run_with_io(process, |io| { - let file = io.files.file(fd)?; - log::info!("len = {:#x}", len); - - space.allocate( - None, - len, - |off| file.get_page(offset + off as u64), - // TODO make get_page return attributes for each page itself - MapAttributes::USER_WRITE - | MapAttributes::USER_READ - | MapAttributes::NON_GLOBAL, - ) - }), - } + ) + }) } SyscallFunction::UnmapMemory => { let addr = args[0] as usize; From 69c73454c182682d5ec132d7f0e0d3bfcb4652f9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 2 Jan 2024 14:01:33 +0200 Subject: [PATCH 141/211] proc: shared memory + scheduler rework --- lib/kernel-util/src/mem/mod.rs | 6 +- lib/kernel-util/src/sync/mod.rs | 1 + lib/kernel-util/src/sync/spin_rwlock.rs | 162 ++++++++ lib/vfs/src/file/mod.rs | 25 +- lib/vfs/src/lib.rs | 2 + lib/vfs/src/shared_memory.rs | 38 ++ src/arch/mod.rs | 28 +- src/arch/x86_64/apic/mod.rs | 5 +- src/arch/x86_64/boot/mod.rs | 2 - src/arch/x86_64/cpu.rs | 37 +- src/arch/x86_64/exception.rs | 2 +- src/arch/x86_64/syscall.rs | 3 +- src/debug.rs | 2 +- src/init.rs | 4 +- src/mem/phys/mod.rs | 5 + src/syscall/mod.rs | 17 +- src/task/mod.rs | 2 +- src/task/process.rs | 12 +- src/task/sched.rs | 399 +++++++++---------- src/task/thread.rs | 504 ++++++++++++------------ 20 files changed, 732 insertions(+), 524 deletions(-) create mode 100644 lib/kernel-util/src/sync/spin_rwlock.rs create mode 100644 lib/vfs/src/shared_memory.rs diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 972d56bc..bb96ffef 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -7,7 +7,7 @@ use core::{ use yggdrasil_abi::error::Error; -use crate::api::{__allocate_contiguous_pages, __free_page, __physicalize}; +use crate::api::{self, __allocate_contiguous_pages, __free_page, __physicalize}; use self::address::{AsPhysicalAddress, PhysicalAddress}; @@ -254,3 +254,7 @@ impl fmt::Display for PageBox { unsafe impl Send for PageBox {} unsafe impl Sync for PageBox {} + +pub fn allocate_page() -> Result { + unsafe { api::__allocate_page() } +} diff --git a/lib/kernel-util/src/sync/mod.rs b/lib/kernel-util/src/sync/mod.rs index b9df129b..3b8870eb 100644 --- a/lib/kernel-util/src/sync/mod.rs +++ b/lib/kernel-util/src/sync/mod.rs @@ -6,6 +6,7 @@ use yggdrasil_abi::error::Error; pub mod fence; pub mod guard; pub mod mutex; +pub mod spin_rwlock; pub mod spinlock; pub use fence::SpinFence; diff --git a/lib/kernel-util/src/sync/spin_rwlock.rs b/lib/kernel-util/src/sync/spin_rwlock.rs new file mode 100644 index 00000000..64e6f368 --- /dev/null +++ b/lib/kernel-util/src/sync/spin_rwlock.rs @@ -0,0 +1,162 @@ +use core::{ + cell::UnsafeCell, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use yggdrasil_abi::error::Error; + +use super::{IrqGuard, LockMethod}; + +struct RwLockInner { + value: AtomicUsize, +} + +pub struct IrqSafeRwLock { + value: UnsafeCell, + inner: RwLockInner, +} + +pub struct IrqSafeRwLockReadGuard<'a, T> { + lock: &'a IrqSafeRwLock, + guard: IrqGuard, +} + +pub struct IrqSafeRwLockWriteGuard<'a, T> { + lock: &'a IrqSafeRwLock, + guard: IrqGuard, +} + +impl RwLockInner { + const LOCKED_READ: usize = 1 << 2; + const LOCKED_WRITE: usize = 1; + + const fn new() -> Self { + Self { + value: AtomicUsize::new(0), + } + } + + #[inline] + fn acquire_read_raw(&self) -> usize { + let value = self.value.fetch_add(Self::LOCKED_READ, Ordering::Acquire); + + if value > usize::MAX / 2 { + self.value.fetch_sub(Self::LOCKED_READ, Ordering::Relaxed); + panic!("Too many read locks acquired"); + } + + value + } + + #[inline] + fn try_acquire_read(&self) -> bool { + let value = self.acquire_read_raw(); + let acquired = value & Self::LOCKED_WRITE != Self::LOCKED_WRITE; + + if !acquired { + unsafe { + self.release_read(); + } + } + + acquired + } + + #[inline] + fn try_acquire_write(&self) -> bool { + self.value + .compare_exchange(0, Self::LOCKED_WRITE, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } + + #[inline] + fn acquire_read(&self) { + while !self.try_acquire_read() { + core::hint::spin_loop(); + } + } + + #[inline] + fn acquire_write(&self) { + while !self.try_acquire_write() { + core::hint::spin_loop(); + } + } + + #[inline] + unsafe fn release_read(&self) { + self.value.fetch_sub(Self::LOCKED_READ, Ordering::Release); + } + + #[inline] + unsafe fn release_write(&self) { + self.value.fetch_and(!Self::LOCKED_WRITE, Ordering::Release); + } +} + +impl IrqSafeRwLock { + pub const fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + inner: RwLockInner::new(), + } + } + + pub fn read(&self) -> IrqSafeRwLockReadGuard { + let guard = IrqGuard::acquire(); + self.inner.acquire_read(); + IrqSafeRwLockReadGuard { lock: self, guard } + } + + pub fn write(&self) -> IrqSafeRwLockWriteGuard { + let guard = IrqGuard::acquire(); + self.inner.acquire_write(); + IrqSafeRwLockWriteGuard { lock: self, guard } + } + + unsafe fn release_read(&self) { + self.inner.release_read(); + } + + unsafe fn release_write(&self) { + self.inner.release_write(); + } +} + +unsafe impl Sync for IrqSafeRwLock {} +unsafe impl Send for IrqSafeRwLock {} + +impl<'a, T> Deref for IrqSafeRwLockReadGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.lock.value.get() } + } +} + +impl<'a, T> Drop for IrqSafeRwLockReadGuard<'a, T> { + fn drop(&mut self) { + unsafe { self.lock.release_read() } + } +} + +impl<'a, T> Deref for IrqSafeRwLockWriteGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.lock.value.get() } + } +} + +impl<'a, T> DerefMut for IrqSafeRwLockWriteGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.lock.value.get() } + } +} + +impl<'a, T> Drop for IrqSafeRwLockWriteGuard<'a, T> { + fn drop(&mut self) { + unsafe { self.lock.release_write() } + } +} diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 4f0ad75b..44b0efc0 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -24,7 +24,7 @@ use crate::{ device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, traits::{Read, Seek, Write}, - FdPoll, FileReadiness, + FdPoll, FileReadiness, SharedMemory, }; use self::{ @@ -64,6 +64,7 @@ pub enum File { AnonymousPipe(PipeEnd), Poll(FdPoll), Channel(ChannelDescriptor), + SharedMemory(Arc), } /// Contains a per-process fd -> FileRef map @@ -92,6 +93,12 @@ impl File { Arc::new(Self::Channel(channel)) } + /// Creates a buffer of shared memory and associates a [File] with it + pub fn new_shared_memory(size: usize) -> Result, Error> { + let shm = SharedMemory::new(size)?; + Ok(Arc::new(Self::SharedMemory(Arc::new(shm)))) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -166,6 +173,7 @@ impl File { pub fn send(&self) -> Result, Error> { match self { Self::Regular(file) => Ok(Arc::new(Self::Regular(file.clone()))), + Self::SharedMemory(shm) => Ok(Arc::new(Self::SharedMemory(shm.clone()))), _ => Err(Error::InvalidOperation), } } @@ -185,7 +193,7 @@ impl File { Self::Regular(file) => Some(&file.node), Self::Block(file) => Some(&file.node), Self::Char(file) => Some(&file.node), - Self::AnonymousPipe(_) | Self::Poll(_) | Self::Channel(_) => None, + _ => None, } } @@ -222,6 +230,7 @@ impl PageProvider for File { fn get_page(&self, offset: u64) -> Result { match self { Self::Block(f) => f.device.0.get_page(offset), + Self::SharedMemory(f) => f.get_page(offset), _ => Err(Error::InvalidOperation), } } @@ -229,6 +238,7 @@ impl PageProvider for File { fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { match self { Self::Block(f) => f.device.0.release_page(offset, phys), + Self::SharedMemory(f) => f.release_page(offset, phys), _ => Err(Error::InvalidOperation), } } @@ -245,6 +255,7 @@ impl Read for File { Self::Poll(_) => Err(Error::InvalidOperation), // TODO maybe allow reading messages from Channels? Self::Channel(_) => Err(Error::InvalidOperation), + Self::SharedMemory(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -261,6 +272,7 @@ impl Write for File { Self::Poll(_) => Err(Error::InvalidOperation), // TODO maybe allow writing messages to Channels? Self::Channel(_) => Err(Error::InvalidOperation), + Self::SharedMemory(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -271,10 +283,8 @@ impl Seek for File { match self { Self::Regular(file) => Ok(*file.position.lock()), Self::Block(file) => Ok(*file.position.lock()), - Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) | Self::Channel(_) => { - Err(Error::InvalidOperation) - } Self::Directory(_) => Err(Error::IsADirectory), + _ => Err(Error::InvalidOperation), } } @@ -282,10 +292,8 @@ impl Seek for File { match self { Self::Regular(file) => file.seek(from), Self::Block(file) => file.seek(from), - Self::Char(_) | Self::AnonymousPipe(_) | Self::Poll(_) | Self::Channel(_) => { - Err(Error::InvalidOperation) - } Self::Directory(_) => Err(Error::IsADirectory), + _ => Err(Error::InvalidOperation), } } } @@ -314,6 +322,7 @@ impl fmt::Debug for File { Self::AnonymousPipe(_) => f.debug_struct("AnonymousPipe").finish_non_exhaustive(), Self::Poll(_) => f.debug_struct("Poll").finish_non_exhaustive(), Self::Channel(_) => f.debug_struct("Channel").finish_non_exhaustive(), + Self::SharedMemory(_) => f.debug_struct("SharedMemory").finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 44665a0a..3e3ca8e1 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -17,6 +17,7 @@ pub(crate) mod ioctx; pub(crate) mod node; pub(crate) mod path; pub(crate) mod poll; +pub(crate) mod shared_memory; pub(crate) mod traits; pub use channel::MessagePayload; @@ -28,4 +29,5 @@ pub use node::{ RegularImpl, SymlinkImpl, }; pub use poll::FdPoll; +pub use shared_memory::SharedMemory; pub use traits::{FileReadiness, Read, Seek, Write}; diff --git a/lib/vfs/src/shared_memory.rs b/lib/vfs/src/shared_memory.rs new file mode 100644 index 00000000..5ab1d387 --- /dev/null +++ b/lib/vfs/src/shared_memory.rs @@ -0,0 +1,38 @@ +use alloc::vec::Vec; +use kernel_util::mem::{address::PhysicalAddress, allocate_page, PageProvider}; +use yggdrasil_abi::error::Error; + +/// Shared memory VFS object +pub struct SharedMemory { + pages: Vec, +} + +impl SharedMemory { + /// Creates a new buffer of shared memory + pub fn new(size: usize) -> Result { + assert_eq!(size & 0xFFF, 0); + let page_count = size / 0x1000; + + let pages = (0..page_count) + .map(|_| allocate_page()) + .collect::>()?; + + Ok(Self { pages }) + } +} + +impl PageProvider for SharedMemory { + fn get_page(&self, offset: u64) -> Result { + // TODO: magic numbers + let index = (offset / 0x1000) as usize; + self.pages + .get(index) + .copied() + .ok_or(Error::InvalidMemoryOperation) + } + + fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { + // TODO track get/release? + Ok(()) + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 8f21ee51..9641c84c 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -37,7 +37,7 @@ use kernel_util::{ use crate::{ mem::phys::PhysicalMemoryRegion, - task::{sched::CpuQueue, Cpu}, + task::{sched::CpuQueue, thread::ThreadId, Cpu}, }; cfg_if! { @@ -242,11 +242,17 @@ pub trait CpuAccess: Sized { /// Safe wrapper for accessing local CPU type Local: LocalCpuAccess; - /// Returns the local CPU wrapper - fn local() -> Self::Local; /// Returns the local CPU wrapper or None, if not yet initialized: fn try_local() -> Option; + /// Returns the local CPU wrapper + fn local() -> Self::Local { + Self::try_local().expect("Local CPU has not yet been initialized") + } + + /// Returns the ID of the local processor or 0 if the processor has not yet been initialized + fn local_id() -> u32; + /// Initializes the CPU's scheduling queue. /// /// # Panics @@ -254,11 +260,23 @@ pub trait CpuAccess: Sized { /// Will panic, if the initialization has already been performed. fn init_queue(&mut self, queue: &'static CpuQueue); - /// Returns the CPU's scheduling queue - fn queue(&self) -> &'static CpuQueue; + /// Returns the CPU's scheduling queue or None if it has not yet been initialized + fn get_queue(&self) -> Option<&'static CpuQueue>; + + /// Returns the CPU's scheduling queue or panics if it has not yet been initialized + fn queue(&self) -> &'static CpuQueue { + self.get_queue() + .expect("CPU's queue has not yet been initialized") + } /// Returns the CPU index fn id(&self) -> u32; + + /// Returns current thread ID or none if idle + fn current_thread_id(&self) -> Option; + + /// Update the current thread ID + unsafe fn set_current_thread_id(&mut self, id: Option); } // External API for architecture specifics diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 982422e0..d816d388 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -100,7 +100,10 @@ unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { let cpu = Cpu::local(); // Clear interrupt before switching, because otherwise we won't receive the next one cpu.local_apic().clear_interrupt(); - cpu.queue().yield_cpu(); + + if let Some(queue) = cpu.get_queue() { + queue.yield_cpu(); + } if let Some(thread) = Thread::get_current() { thread.handle_pending_signals(frame); diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index f7b891ca..4667b1ae 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -120,8 +120,6 @@ pub extern "C" fn __x86_64_ap_entry() -> ! { // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base - infoln!("cpu{} initializing", cpu_id); - unsafe { // Cpu::init_local(LocalApic::new(), cpu_id as u32); // syscall::init_syscall(); diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index f8689381..13bbecb5 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -17,7 +17,7 @@ use crate::{ x86_64::{cpuid, gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, CpuAccess, CpuMessage, LocalCpuAccess, }, - task::sched::CpuQueue, + task::{sched::CpuQueue, thread::ThreadId}, }; use super::{apic::local::LocalApic, smp::CPU_COUNT}; @@ -35,6 +35,7 @@ pub struct Cpu { id: u32, local_apic: LocalApic, + current_thread_id: Option, queue: OneTimeInit<&'static CpuQueue>, } @@ -79,8 +80,6 @@ impl Cpu { /// /// Only meant to be called once per each CPU during their init. pub(super) unsafe fn init_local(local_apic: LocalApic, id: u32) { - infoln!("Initialize CPU with id {}", id); - // Initialize CPU features cpuid::enable_features(); @@ -93,6 +92,7 @@ impl Cpu { id, local_apic, + current_thread_id: None, queue: OneTimeInit::new(), }); let this = Box::into_raw(this); @@ -101,6 +101,8 @@ impl Cpu { MSR_IA32_KERNEL_GS_BASE.set(this as u64); core::arch::asm!("wbinvd; swapgs"); + infoln!("cpu{} initialized", id); + syscall::init_syscall(); } @@ -140,10 +142,7 @@ impl LocalCpu { impl CpuAccess for Cpu { type Local = LocalCpu; - fn local() -> Self::Local { - Self::try_local().unwrap() - } - + #[inline] fn try_local() -> Option { let mut addr: usize; let guard = IrqGuard::acquire(); @@ -154,16 +153,36 @@ impl CpuAccess for Cpu { } } + #[inline] fn id(&self) -> u32 { self.id } + #[inline] + fn current_thread_id(&self) -> Option { + self.current_thread_id + } + + #[inline] + unsafe fn set_current_thread_id(&mut self, id: Option) { + self.current_thread_id = id; + } + + #[inline] + fn local_id() -> u32 { + if let Some(local) = Self::try_local() { + local.id() + } else { + 0 + } + } + fn init_queue(&mut self, queue: &'static CpuQueue) { self.queue.init(queue); } - fn queue(&self) -> &'static CpuQueue { - self.queue.get() + fn get_queue(&self) -> Option<&'static CpuQueue> { + self.queue.try_get().copied() } } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 7ebff9d7..71f3e4df 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -286,7 +286,7 @@ static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { let thread = Thread::current(); - warnln!("{:?} in {} {:?}", kind, thread.id(), thread.name()); + warnln!("{:?} in {} {:?}", kind, thread.id, thread.name); warnln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); warnln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index fc46a106..7675b6b4 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -125,8 +125,9 @@ fn syscall_inner(frame: &mut SyscallFrame) { extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { let frame = unsafe { &mut *frame }; - let thread = Thread::current(); syscall_inner(frame); + + let thread = Thread::current(); unsafe { thread.handle_pending_signals(frame); } diff --git a/src/debug.rs b/src/debug.rs index a692daa5..57b3b14d 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -64,7 +64,7 @@ macro_rules! log_print_raw { macro_rules! log_print { ($level:expr, $($args:tt)+) => { - log_print_raw!($level, "cpu{}:{}:{}: {}", /* $crate::task::Cpu::local_id() */ 0, file!(), line!(), format_args!($($args)+)) + log_print_raw!($level, "cpu{}:{}:{}: {}", <$crate::task::Cpu as $crate::arch::CpuAccess>::local_id(), file!(), line!(), format_args!($($args)+)) }; } diff --git a/src/init.rs b/src/init.rs index 289ee5e8..c0f6a092 100644 --- a/src/init.rs +++ b/src/init.rs @@ -46,8 +46,6 @@ pub fn kinit() -> Result<(), Error> { let mut ioctx = IoContext::new(root); - let devfs = devfs::root(); - { let (user_init, user_init_main) = proc::exec::load(&mut ioctx, "/init", &["/init", "xxx"], &[])?; @@ -56,7 +54,7 @@ pub fn kinit() -> Result<(), Error> { io.set_ioctx(ioctx); drop(io); - user_init_main.enqueue_somewhere(); + user_init_main.enqueue(); } Ok(()) diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 283e829a..3f760516 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -209,6 +209,11 @@ fn kernel_physical_memory_region() -> PhysicalMemoryRegion { PhysicalMemoryRegion { base, size } } +#[no_mangle] +fn __allocate_page() -> Result { + alloc_page() +} + #[no_mangle] fn __allocate_contiguous_pages(count: usize) -> Result { alloc_pages_contiguous(count) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 6a206946..688355ad 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -63,7 +63,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result match func { SyscallFunction::DebugTrace => { let pid = process.id(); - let tid = thread.id(); + let tid = thread.id; let arg = arg_user_str(args[0] as usize, args[1] as usize)?; @@ -422,6 +422,16 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } }) } + SyscallFunction::CreateSharedMemory => { + let size = args[0] as usize; + let size = size.page_align_up::(); + + run_with_io(process, |mut io| { + let file = File::new_shared_memory(size)?; + let fd = io.files.place_file(file, true)?; + Ok(fd.0 as usize) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; @@ -475,7 +485,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } drop(child_io); - child_main.enqueue_somewhere(); + child_main.enqueue(); Ok(pid as _) }) @@ -503,8 +513,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } SyscallFunction::ExitThread => { let code = ExitCode::from(args[0] as i32); - thread.exit(code); - unreachable!() + thread.exit(code) } SyscallFunction::ExitProcess => { // A bit different from thread exit: wait for other threads to finish and exit only diff --git a/src/task/mod.rs b/src/task/mod.rs index fa1491d4..a81006a9 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -27,7 +27,7 @@ pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send f: F, ) -> Result<(), Error> { let thread = Thread::new_kthread(name, TaskContext::kernel_closure(f)?); - thread.enqueue_somewhere(); + thread.enqueue(); Ok(()) } diff --git a/src/task/process.rs b/src/task/process.rs index 446c4284..39ae143c 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -224,11 +224,11 @@ impl Process { tls_address, )?; let thread = Thread::new_uthread(self.clone(), space, context); - let id = thread.id(); + let id = thread.id; self.inner.lock().threads.push(thread.clone()); - thread.enqueue_somewhere(); + thread.enqueue(); Ok(id) } @@ -335,7 +335,7 @@ impl Process { // TODO make this cleaner let old_len = inner.threads.len(); - inner.threads.retain(|t| t.id() != thread); + inner.threads.retain(|t| t.id != thread); assert_ne!(inner.threads.len(), old_len); let last_thread = inner.threads.is_empty(); @@ -396,15 +396,15 @@ impl Process { let mut inner = self.inner.lock(); for thread in inner.threads.iter() { - if thread.id() == except { + if thread.id == except { continue; } - infoln!("Terminate thread {}", thread.id()); + infoln!("Terminate thread {}", thread.id); thread.terminate().await; } - inner.threads.retain(|t| t.id() == except); + inner.threads.retain(|t| t.id == except); } } diff --git a/src/task/sched.rs b/src/task/sched.rs index 3c839fcb..b9eba858 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -4,7 +4,11 @@ use core::sync::atomic::Ordering; use alloc::{collections::VecDeque, sync::Arc, vec::Vec}; use cfg_if::cfg_if; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use crossbeam_queue::SegQueue; +use kernel_util::{ + sync::{IrqGuard, IrqSafeSpinlock}, + util::OneTimeInit, +}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess}, @@ -17,32 +21,184 @@ use super::{ Cpu, TaskContext, }; -/// Per-CPU statistics -#[derive(Default)] -pub struct CpuQueueStats { - /// Ticks spent idling - pub idle_time: u64, - /// Ticks spent running CPU tasks - pub cpu_time: u64, - - /// Time since last measurement - measure_time: u64, -} - -struct CpuQueueInner { - current: Option, - queue: VecDeque, - stats: CpuQueueStats, -} +// /// Per-CPU statistics +// #[derive(Default)] +// pub struct CpuQueueStats { +// /// Ticks spent idling +// pub idle_time: u64, +// /// Ticks spent running CPU tasks +// pub cpu_time: u64, +// +// /// Time since last measurement +// measure_time: u64, +// } /// Per-CPU queue pub struct CpuQueue { - inner: IrqSafeSpinlock, - idle: TaskContext, + queue: SegQueue, index: usize, + idle: TaskContext, } -static QUEUES: OneTimeInit> = OneTimeInit::new(); +impl CpuQueue { + /// Creates a new [CpuQueue] for CPU with given `index` + pub fn new(index: usize) -> Self { + let idle = TaskContext::kernel(__idle, Cpu::local().id() as usize) + .expect("Could not construct an idle task"); + + Self { + queue: SegQueue::new(), + index, + idle, + } + } + + /// Returns the queue's associated CPU index + #[inline] + pub fn index(&self) -> usize { + self.index + } + + /// "Enters" the scheduler by selecting a first task to execute + pub unsafe fn enter(&self) -> ! { + let _guard = IrqGuard::acquire(); + + self.idle.enter() + } + + /// "Pushes" a thread to the queue for execution + pub fn push(&self, tid: ThreadId) { + self.queue.push(tid); + } + + fn pop(&self) -> (Option>, Option) { + while let Some(id) = self.queue.pop() { + let Some(thread) = Thread::get(id) else { + continue; + }; + + let mut sched = thread.sched.lock(); + + assert!(sched.in_queue); + assert!(core::ptr::eq(self, sched.queue.unwrap())); + + match sched.state { + ThreadState::Ready => { + sched.state = ThreadState::Running; + drop(sched); + return (Some(thread), Some(id)); + } + ThreadState::Running => { + panic!("Unexpected state: Running ({:?})", id); + } + ThreadState::Terminated => { + sched.in_queue = false; + sched.queue = None; + thread.set_terminated(); + } + ThreadState::Suspended => { + sched.queue = None; + sched.in_queue = false; + } + } + } + + (None, None) + } + + /// Selects a new thread from the queue and performs a context switch if necessary + pub unsafe fn yield_cpu(&self) -> bool { + assert!(ArchitectureImpl::interrupt_mask()); + + let mut cpu = Cpu::local(); + + let current_id = cpu.current_thread_id(); + let current = current_id.and_then(Thread::get); + + if let Some(current) = current.as_ref() { + let mut sched = current.sched.lock(); + + let q = sched.queue.unwrap(); + assert!(core::ptr::eq(q, self)); + assert!(sched.in_queue); + + match sched.state { + ThreadState::Ready => { + panic!("Thread {:?} was running, but is marked Ready", current.id); + } + ThreadState::Running => { + sched.state = ThreadState::Ready; + + self.queue.push(current.id); + } + ThreadState::Terminated => { + sched.in_queue = false; + sched.queue = None; + + current.set_terminated(); + } + ThreadState::Suspended => { + sched.queue = None; + sched.in_queue = false; + } + } + } + + let (next, next_id) = self.pop(); + + let current_ctx = if let Some(current) = current.as_ref() { + ¤t.context + } else { + &self.idle + }; + + let next_ctx = if let Some(next) = next.as_ref() { + &next.context + } else { + &self.idle + }; + + cpu.set_current_thread_id(next_id); + + if !core::ptr::eq(current_ctx, next_ctx) { + // Perform the switch + next_ctx.switch(current_ctx); + + true + } else { + false + } + } + + /// Returns a queue for given CPU `index` + pub fn for_cpu(index: usize) -> &'static CpuQueue { + &QUEUES.get()[index] + } + + /// Returns a queue for the local CPU + pub fn local() -> &'static CpuQueue { + Cpu::local().queue() + } + + /// Returns the preferred queue for a thread to be scheduled into. Takes affinity mask into + /// account when picking a queue. + pub fn for_affinity_mask(mask: u64) -> &'static CpuQueue { + debug_assert_ne!(mask, 0); + + QUEUES + .get() + .iter() + .filter(|c| mask & (1 << c.index) != 0) + .min_by_key(|c| c.queue.len()) + .unwrap() + } + + /// Returns `true` if the queue is local to the current CPU + pub fn is_local(&self) -> bool { + assert!(ArchitectureImpl::interrupt_mask()); + core::ptr::eq(Self::local(), self) + } +} #[naked] extern "C" fn __idle(_x: usize) -> ! { @@ -63,206 +219,7 @@ extern "C" fn __idle(_x: usize) -> ! { } } -impl CpuQueueStats { - /// Reset the stats to zero values - pub fn reset(&mut self) { - self.cpu_time = 0; - self.idle_time = 0; - } -} - -impl CpuQueueInner { - /// Picks a next task for execution, skipping (dropping) those that were suspended. May return - /// None if the queue is empty or no valid task was found, in which case the scheduler should - /// go idle. - pub fn next_ready_task(&mut self) -> Option> { - #[allow(clippy::never_loop)] - while !self.queue.is_empty() { - let task = self.queue.pop_front().unwrap(); - let task_t = Thread::get(task).unwrap(); - - match task_t.state.load(Ordering::Acquire) { - ThreadState::Ready => { - return Some(task_t); - } - e => panic!( - "Unexpected thread state in CpuQueue: {:?} ({} {:?}, cpu_id={})", - e, - task_t.id(), - task_t.name(), - 1234 // task_t.cpu_id() - ), - } - } - - None - } - - /// Returns an iterator over all the threads in the queue plus the currently running thread, - /// if there is one. - pub fn iter(&self) -> impl Iterator { - Iterator::chain(self.queue.iter(), self.current.iter()) - } -} - -impl CpuQueue { - /// Constructs an empty queue with its own idle task - pub fn new(index: usize) -> Self { - let idle = TaskContext::kernel(__idle, Cpu::local().id() as usize) - .expect("Could not construct an idle task"); - - Self { - inner: { - IrqSafeSpinlock::new(CpuQueueInner { - current: None, - queue: VecDeque::new(), - stats: CpuQueueStats::default(), - }) - }, - idle, - index, - } - } - - /// Starts queue execution on current CPU. - /// - /// # Safety - /// - /// Only meant to be called from [crate::task::enter()] function. - pub unsafe fn enter(&self) -> ! { - assert!(ArchitectureImpl::interrupt_mask()); - // Start from idle thread to avoid having a Arc stuck here without getting dropped - // let t = CNTPCT_EL0.get(); - // self.lock().stats.measure_time = t; - self.idle.enter() - } - - /// Yields CPU execution to the next task in queue (or idle task if there aren't any). - /// - /// # Safety - /// - /// The function is only meant to be called from kernel threads (e.g. if they want to yield - /// CPU execution to wait for something) or interrupt handlers. - pub unsafe fn yield_cpu(&self) { - // Will also disable interrupts in this section - let cpu = Cpu::local(); - - let mut inner = self.inner.lock(); - - let current = inner.current; - let current_t = current.and_then(Thread::get); - - if let Some(current_t) = current_t.as_ref() { - if current_t.state.load(Ordering::Acquire) == ThreadState::Terminated { - current_t.exit_notify.wake_all(); - } - - if current_t - .state - .compare_exchange( - ThreadState::Running, - ThreadState::Ready, - Ordering::SeqCst, - Ordering::Relaxed, - ) - .is_ok() - { - inner.queue.push_back(current_t.id()); - } - } - - let next_t = inner.next_ready_task(); - - inner.current = next_t.as_deref().map(Thread::id); - - // Can drop the lock, we hold current and next Arc's - drop(inner); - - let (from, _from_rc) = if let Some(current_t) = current_t.as_ref() { - (current_t.context(), Arc::strong_count(current_t)) - } else { - (&self.idle, 0) - }; - - let (to, _to_rc) = if let Some(next_t) = next_t.as_ref() { - next_t.set_running(cpu.id()); - (next_t.context(), Arc::strong_count(next_t)) - } else { - (&self.idle, 0) - }; - - assert!(ArchitectureImpl::interrupt_mask()); - to.switch(from) - } - - /// Pushes the process to the back of the execution queue. - /// - /// # Safety - /// - /// Only meant to be called from Process impl. The function does not set any process accounting - /// information, which may lead to invalid states. - pub unsafe fn enqueue(&self, tid: ThreadId) { - let mut inner = self.inner.lock(); - assert!(ArchitectureImpl::interrupt_mask()); - // assert_eq!(p.state(), ProcessState::Ready); - - inner.queue.push_back(tid); - } - - /// Removes thread with given TID from the exeuction queue. - pub fn dequeue(&self, tid: ThreadId) { - assert!(ArchitectureImpl::interrupt_mask()); - - let mut inner = self.inner.lock(); - inner.queue.retain(|&p| p != tid) - } - - /// Returns the queue length at this moment. - /// - /// # Note - /// - /// This value may immediately change. - pub fn len(&self) -> usize { - self.inner.lock().queue.len() - } - - /// Returns `true` if the queue is empty at the moment. - /// - /// # Note - /// - /// This may immediately change. - pub fn is_empty(&self) -> bool { - self.inner.lock().queue.is_empty() - } - - /// Returns the current [ThreadId], or [None] if idle - pub fn current_id(&self) -> Option { - self.inner.lock().current - } - - /// Returns a queue for given CPU index - pub fn for_cpu(id: usize) -> &'static CpuQueue { - &QUEUES.get()[id] - } - - /// Returns an iterator over all queues of the system - #[inline] - pub fn all() -> impl Iterator { - QUEUES.get().iter() - } - - /// Picks a queue with the least amount of tasks in it - pub fn least_loaded() -> Option<(usize, &'static CpuQueue)> { - let queues = QUEUES.get(); - - queues.iter().enumerate().min_by_key(|(_, q)| q.len()) - } - - /// Returns the CPU index of the queue - pub fn index(&self) -> usize { - self.index - } -} +static QUEUES: OneTimeInit> = OneTimeInit::new(); /// Initializes the global queue list pub fn init_queues(queues: Vec) { diff --git a/src/task/thread.rs b/src/task/thread.rs index 5beb802f..fe12c438 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -1,3 +1,5 @@ +// TODO XXX TODO XXX +#![allow(missing_docs)] //! Thread data structures and management use core::{ @@ -5,7 +7,7 @@ use core::{ mem::size_of, ops::Deref, pin::Pin, - sync::atomic::{AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, task::{Context, Poll}, }; @@ -13,17 +15,14 @@ use abi::{ error::Error, process::{ExitCode, Signal, SignalEntryData}, }; -use alloc::{ - collections::{BTreeMap, VecDeque}, - string::String, - sync::Arc, -}; +use alloc::{collections::BTreeMap, string::String, sync::Arc}; use atomic_enum::atomic_enum; -use futures_util::{task::ArcWake, Future}; +use crossbeam_queue::SegQueue; +use futures_util::Future; use kernel_util::{ block, runtime::QueueWaker, - sync::{IrqGuard, IrqSafeSpinlock}, + sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, }; use crate::{ @@ -61,35 +60,90 @@ pub enum ThreadId { #[repr(C)] pub struct CurrentThread(Arc, IrqGuard); +#[derive(Debug)] +#[repr(transparent)] +pub struct ThreadAffinity(AtomicU64); + struct SignalEntry { entry: usize, stack: usize, } struct ThreadInner { - queue: Option<&'static CpuQueue>, - signal_entry: Option, - signal_stack: VecDeque, } +struct GlobalThreadList { + data: BTreeMap>, +} + +pub struct ThreadSchedulingInfo { + pub state: ThreadState, + + pub in_queue: bool, + pub queue: Option<&'static CpuQueue>, +} + +static THREADS: IrqSafeRwLock = IrqSafeRwLock::new(GlobalThreadList::new()); + /// Describes a single thread within the system pub struct Thread { - context: TaskContext, - name: Option, - id: ThreadId, - - pub(super) state: AtomicThreadState, + pub id: ThreadId, + pub name: Option, + pub sched: IrqSafeSpinlock, + pub context: TaskContext, process: Option>, space: Option>, inner: IrqSafeSpinlock, + signal_queue: SegQueue, + pub(super) terminated: AtomicBool, pub(super) exit_notify: Arc, + pub affinity: ThreadAffinity, } -static THREADS: IrqSafeSpinlock>> = - IrqSafeSpinlock::new(BTreeMap::new()); +impl GlobalThreadList { + pub const fn new() -> Self { + Self { + data: BTreeMap::new(), + } + } + + #[inline] + pub fn get(&self, id: ThreadId) -> Option<&Arc> { + self.data.get(&id) + } + + pub fn insert(&mut self, thread: Arc) { + let id = thread.id; + debug_assert!(!self.data.contains_key(&id)); + self.data.insert(id, thread); + } +} + +impl ThreadAffinity { + pub const ANY_CPU: u64 = u64::MAX; + + pub const fn any_cpu() -> Self { + Self(AtomicU64::new(Self::ANY_CPU)) + } + + pub const fn only_cpu(index: usize) -> Self { + Self(AtomicU64::new(1 << index)) + } + + #[inline] + pub fn get(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } + + #[inline] + pub fn set(&self, value: u64) { + debug_assert_ne!(value, 0); + self.0.store(value, Ordering::Relaxed); + } +} impl Thread { fn new( @@ -100,30 +154,40 @@ impl Thread { context: TaskContext, ) -> Arc { let thread = Arc::new(Self { - context, - name, id, - - state: AtomicThreadState::new(ThreadState::Suspended), - + name, + sched: IrqSafeSpinlock::new(ThreadSchedulingInfo { + state: ThreadState::Suspended, + in_queue: false, + queue: None, + }), + context, process, space, - inner: IrqSafeSpinlock::new(ThreadInner { - queue: None, - - signal_stack: VecDeque::new(), - signal_entry: None, - }), - + inner: IrqSafeSpinlock::new(ThreadInner { signal_entry: None }), + signal_queue: SegQueue::new(), exit_notify: Arc::new(QueueWaker::new()), + terminated: AtomicBool::new(false), + affinity: ThreadAffinity::any_cpu(), }); - THREADS.lock().insert(id, thread.clone()); + THREADS.write().insert(thread.clone()); thread } + /// Constructs a new kernel-space thread + pub fn new_kthread>(name: S, context: TaskContext) -> Arc { + Self::new( + ThreadId::next_kernel(), + Some(name.into()), + None, + None, + context, + ) + } + /// Constructs a new user-space thread pub fn new_uthread( parent: Arc, @@ -139,53 +203,151 @@ impl Thread { ) } - /// Constructs a new kernel-space thread - pub fn new_kthread>(name: S, context: TaskContext) -> Arc { - Self::new( - ThreadId::next_kernel(), - Some(name.into()), - None, - None, - context, - ) + // Get/Set + + /// Updates the thread affinity to run on a specific set of CPUs + pub fn set_affinity(&self, affinity: u64) { + self.affinity.set(affinity); } - // Info - - /// Returns the thread's ID - pub fn id(&self) -> ThreadId { - self.id + /// Updates the thread signal entry/stack information + pub fn set_signal_entry(&self, entry: usize, stack: usize) { + let mut inner = self.inner.lock(); + inner.signal_entry.replace(SignalEntry { entry, stack }); } - /// Returns the thread's name, if set - pub fn name(&self) -> Option<&String> { - self.name.as_ref() - } - - /// Returns the thread's [TaskContext] - pub fn context(&self) -> &TaskContext { - &self.context - } - - /// Returns the thread's [ProcessAddressSpace] reference. - /// - /// # Panics - /// - /// Will panic if the thread has no associated address space. - pub fn address_space(&self) -> &Arc { + /// Returns the thread address space (usually provided by its parent process). If none exists, + /// panics. + pub fn address_space(&self) -> &ProcessAddressSpace { self.space.as_ref().unwrap() } - /// Returns the thread's parent [Process] reference. - /// - /// # Panics - /// - /// Will panic if the thread has no process associated (i.e. it's a kernel thread). + /// Returns the thread's parent process, panics if there's none pub fn process(&self) -> &Arc { self.process.as_ref().unwrap() } - // Queue operation + /// Updates the thread's terminated status and wakes up any other threads waiting for it to + /// exit + pub fn set_terminated(&self) { + if !self.terminated.swap(true, Ordering::Release) { + self.exit_notify.wake_all(); + } + } + + // Signals + + /// Pushes a signal to the thread's signal queue + pub fn raise_signal(&self, signal: Signal) { + self.signal_queue.push(signal); + } + + // Scheduling + + /// Changes thread state to "Ready" and inserts it into given `queue`, if it's not yet in one + pub fn enqueue_to(&self, queue: &'static CpuQueue) { + let mut sched = self.sched.lock(); + + assert_ne!(sched.state, ThreadState::Terminated); + + match sched.state { + ThreadState::Running | ThreadState::Ready => { + assert!(sched.in_queue); + } + ThreadState::Suspended => { + sched.state = ThreadState::Ready; + + if !sched.in_queue { + assert!(sched.queue.is_none()); + + sched.in_queue = true; + sched.queue = Some(queue); + + queue.push(self.id); + } + } + ThreadState::Terminated => panic!("Cannot enqueue a terminated thread"), + } + } + + /// Changes thread state to `state` and removes it from its queue. If the thread is currently + /// running on local CPU, will yield control to the next thread + pub fn dequeue(&self, state: ThreadState) { + let mut sched = self.sched.lock(); + + debug_assert_ne!(state, ThreadState::Running); + debug_assert_ne!(state, ThreadState::Ready); + + let old_state = sched.state; + sched.state = state; + + if let Some(queue) = sched.queue { + match (queue.is_local(), old_state) { + (true, ThreadState::Running) => unsafe { + debug_assert!(sched.in_queue); + drop(sched); + queue.yield_cpu(); + }, + (false, ThreadState::Running) => { + debugln!("deq remote {:?}", self.id); + debug_assert!(sched.in_queue); + } + (_, ThreadState::Ready) => { + debug_assert!(sched.in_queue); + } + (_, ThreadState::Suspended | ThreadState::Terminated) => { + todo!() + } + } + } else { + assert!(!sched.in_queue); + assert_ne!(old_state, ThreadState::Running); + assert_ne!(old_state, ThreadState::Ready); + } + } + + /// Inserts the thread as "Ready" into the best queue (based on affinity and queue load) + pub fn enqueue(&self) { + let queue = CpuQueue::for_affinity_mask(self.affinity.get()); + self.enqueue_to(queue); + } + + fn is_terminated(&self) -> bool { + self.terminated.load(Ordering::Acquire) + } + + /// Waits until thread is terminated and removed from scheduling queues + pub fn wait_for_exit(self: &Arc) -> impl Future { + struct F(Arc); + + impl Future for F { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let F(thread) = self.deref(); + + thread.exit_notify.register(cx.waker()); + + if thread.is_terminated() { + thread.exit_notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + } + + infoln!("wait_for_exit: pending {:?}", self.id); + F(self.clone()) + } + + /// Requests thread termination and blocks until said thread finishes fully + pub async fn terminate(self: &Arc) { + // Will not abort the execution: called from another thread + self.dequeue(ThreadState::Terminated); + + self.wait_for_exit().await + } /// Returns the current thread on the CPU. /// @@ -201,204 +363,30 @@ impl Thread { pub fn get_current() -> Option { // IrqGuard is held throughout let cpu = Cpu::local(); - let thread = cpu.queue().current_id().and_then(Self::get); + let thread = cpu.current_thread_id().and_then(Self::get); thread.map(|t| CurrentThread(t, cpu.into_guard())) } - /// Enqueues the thread onto any (usually the least loaded) CPU queue and returns its index. - /// See [Thread::enqueue_to]. - pub fn enqueue_somewhere(&self) -> usize { - // Doesn't have to be precise, so even if something changes, we can still be rebalanced - // to another CPU - let (index, queue) = CpuQueue::least_loaded().unwrap(); - - self.enqueue_to(queue); - - index - } - - /// Enqueues the thread onto the specific CPU's queue. - /// - /// # Panics - /// - /// Will panic if the process is in some weird state while being queued. - pub fn enqueue_to(&self, queue: &'static CpuQueue) { - let _irq = IrqGuard::acquire(); - - { - let mut inner = self.inner.lock(); - let old_queue = inner.queue.replace(queue); - if old_queue.is_some() { - // Already in some queue - return; - } - } - - match self.state.compare_exchange( - ThreadState::Suspended, - ThreadState::Ready, - Ordering::SeqCst, - Ordering::Relaxed, - ) { - Err(ThreadState::Terminated) => { - // Process might've been killed while `await`ing in a `block!` - debugln!( - "Thread {} {:?} already terminated, dropping", - self.id(), - self.name() - ); - } - Err(state) => { - todo!("Unexpected process state when enqueueing: {:?}", state) - } - Ok(_) => unsafe { - queue.enqueue(self.id); - }, - } - } - - // TODO maybe separate dequeue for current and "other" threads - fn dequeue(&self, new_state: ThreadState) { - let _irq = IrqGuard::acquire(); - assert_ne!(new_state, ThreadState::Ready); - assert_ne!(new_state, ThreadState::Running); - - let current_state = self.state.swap(new_state, Ordering::SeqCst); - let mut inner = self.inner.lock(); - let proc_queue = inner.queue.take().unwrap(); - - proc_queue.dequeue(self.id()); - - match current_state { - // NOTE: I'm not sure if the process could've been queued between the store and this - // but most likely not (if I'm not that bad with atomics) - // Do nothing, its queue will just drop the process - ThreadState::Ready => (), - // Do nothing, not in a queue already - ThreadState::Suspended => (), - ThreadState::Terminated => panic!("Thread is terminated"), - ThreadState::Running => { - let queue = Cpu::local().queue(); - - if core::ptr::eq(queue, proc_queue) { - drop(inner); - // Suspending a process running on local CPU - unsafe { queue.yield_cpu() } - } else { - todo!(); - } - } - } - } - - /// Marks the thread as running and sets its "current" CPU index. - /// - /// # Safety - /// - /// Only meant to be called from within the scheduler. - pub unsafe fn set_running(&self, _cpu: u32) { - self.state.store(ThreadState::Running, Ordering::Release); - } - - // Accounting - - /// Returns the thread with given [ThreadId], if it exists pub fn get(id: ThreadId) -> Option> { - THREADS.lock().get(&id).cloned() - } - - // Thread inner - - /// Changes the thread's signal entry point information - pub fn set_signal_entry(&self, entry: usize, stack: usize) { - let mut inner = self.inner.lock(); - inner.signal_entry.replace(SignalEntry { entry, stack }); - } - - /// Pushes a [Signal] onto the thread's signal stack. - /// - /// When executed on a current thread, the signal is guaranteed to be handled exactly before - /// returning to user context (i.e. from syscall). Otherwise, the signal handling order and - /// whether it will be delivered at all is not guaranteed. - pub fn raise_signal(self: &Arc, signal: Signal) { - self.inner.lock().signal_stack.push_back(signal); - - if self.state.load(Ordering::Acquire) == ThreadState::Suspended { - self.clone().enqueue_somewhere(); - } - } - - /// Requests thread termination and blocks until said thread finishes fully: - pub fn terminate(self: &Arc) -> impl Future { - struct F(Arc); - - impl Future for F { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let F(thread) = self.deref(); - - thread.exit_notify.register(cx.waker()); - - if thread.state.load(Ordering::Acquire) == ThreadState::Terminated { - thread.exit_notify.remove(cx.waker()); - Poll::Ready(()) - } else { - Poll::Pending - } - } - } - - // Will not abort the execution: called from another thread - self.dequeue(ThreadState::Terminated); - - F(self.clone()) - } -} - -impl ArcWake for Thread { - fn wake_by_ref(arc_self: &Arc) { - arc_self.clone().enqueue_somewhere(); + THREADS.read().get(id).cloned() } } impl CurrentThread { - fn dequeue_terminate(&self, code: ExitCode) { - self.state - .compare_exchange( - ThreadState::Running, - ThreadState::Terminated, - Ordering::AcqRel, - Ordering::Relaxed, - ) - .unwrap(); - + /// Terminate the current thread + pub fn exit(&self, code: ExitCode) -> ! { if let Some(process) = self.process.as_ref() { - process.handle_thread_exit(self.id(), code); + process.handle_thread_exit(self.id, code); } - - let queue = Cpu::local().queue(); - - queue.dequeue(self.id()); - unsafe { queue.yield_cpu() } - + self.dequeue(ThreadState::Terminated); unreachable!() } - /// Terminate the current thread - pub fn exit(&self, code: ExitCode) { - // May already have been terminated by process exit - if self.state.load(Ordering::Acquire) == ThreadState::Terminated { - return; - } - - self.dequeue_terminate(code) - } - + // TODO: test multithreaded process exit /// Terminate the parent process of the thread, including all other threads and the current /// thread itself - pub fn exit_process(&self, code: ExitCode) { + pub fn exit_process(&self, code: ExitCode) -> ! { let _guard = IrqGuard::acquire(); let process = self @@ -409,20 +397,17 @@ impl CurrentThread { let p = process.clone(); block! { - p.terminate_others(self.id()).await; + p.terminate_others(self.id).await; } .unwrap(); - self.exit(code); - unreachable!(); + self.exit(code) } - /// Suspends the current thread until it is waken up again. Guaranteed to happen immediately. pub fn suspend(&self) -> Result<(), Error> { self.dequeue(ThreadState::Suspended); - let inner = self.inner.lock(); - if !inner.signal_stack.is_empty() { + if !self.signal_queue.is_empty() { return Err(Error::Interrupted); } @@ -440,9 +425,9 @@ impl CurrentThread { return; } - let mut inner = self.inner.lock(); + if let Some(signal) = self.signal_queue.pop() { + let inner = self.inner.lock(); - if let Some(signal) = inner.signal_stack.pop_front() { let Some(entry) = inner.signal_entry.as_ref() else { todo!(); }; @@ -546,7 +531,7 @@ fn __create_kthread( #[no_mangle] fn __enqueue(t: &Arc) { - t.enqueue_somewhere(); + t.enqueue(); } #[no_mangle] @@ -562,11 +547,10 @@ fn __suspend_current(t: &CurrentThread) -> Result<(), Error> { #[no_mangle] fn __exit_current(t: &CurrentThread, code: ExitCode) -> ! { t.exit(code); - unreachable!(); } #[no_mangle] unsafe fn __yield() { let _irq = IrqGuard::acquire(); - Cpu::local().queue().yield_cpu() + Cpu::local().queue().yield_cpu(); } From 2444e147c41874ef1ab66677cd5a5837b9f61e2a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 4 Jan 2024 21:22:18 +0200 Subject: [PATCH 142/211] vfs: implement PTY devices --- driver/block/core/src/lib.rs | 2 +- lib/device-api/src/input.rs | 3 - lib/device-api/src/lib.rs | 1 - lib/kernel-util/src/api.rs | 7 +- lib/kernel-util/src/lib.rs | 10 +- lib/kernel-util/src/runtime/executor.rs | 11 +- lib/kernel-util/src/sync/spin_rwlock.rs | 18 +- lib/kernel-util/src/util.rs | 2 + {src => lib/kernel-util/src}/util/ring.rs | 12 +- lib/vfs/src/channel.rs | 5 +- lib/vfs/src/file/mod.rs | 42 ++- lib/vfs/src/lib.rs | 2 + lib/vfs/src/node/impls.rs | 1 - lib/vfs/src/poll.rs | 14 +- lib/vfs/src/pty.rs | 417 ++++++++++++++++++++++ src/arch/mod.rs | 6 +- src/arch/x86_64/mod.rs | 2 +- src/arch/x86_64/peripherals/ps2/mod.rs | 26 +- src/debug.rs | 7 +- src/device/display/linear_fb.rs | 2 +- src/device/input.rs | 5 +- src/device/tty.rs | 22 +- src/mem/process.rs | 4 +- src/proc/elf.rs | 1 - src/proc/exec.rs | 1 - src/syscall/mod.rs | 34 +- src/task/context.rs | 3 +- src/task/process.rs | 5 + src/task/sched.rs | 35 +- src/task/thread.rs | 5 +- src/util/mod.rs | 1 - 31 files changed, 589 insertions(+), 117 deletions(-) delete mode 100644 lib/device-api/src/input.rs rename {src => lib/kernel-util/src}/util/ring.rs (89%) create mode 100644 lib/vfs/src/pty.rs diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index 0ddaf8d6..5bed61ad 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::task::{Context, Poll}; -use kernel_util::mem::{address::PhysicalAddress, PageProvider}; +use kernel_util::mem::PageProvider; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; diff --git a/lib/device-api/src/input.rs b/lib/device-api/src/input.rs deleted file mode 100644 index 907357ac..00000000 --- a/lib/device-api/src/input.rs +++ /dev/null @@ -1,3 +0,0 @@ -use yggdrasil_abi::error::Error; - -use crate::Device; diff --git a/lib/device-api/src/lib.rs b/lib/device-api/src/lib.rs index 30357982..7225486b 100644 --- a/lib/device-api/src/lib.rs +++ b/lib/device-api/src/lib.rs @@ -5,7 +5,6 @@ extern crate alloc; pub mod bus; pub mod device; -pub mod input; pub mod interrupt; pub mod manager; pub mod serial; diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index 2b5acb2b..fae48fe4 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -2,7 +2,10 @@ use core::time::Duration; use alloc::{string::String, sync::Arc}; use device_api::interrupt::MessageInterruptController; -use yggdrasil_abi::{error::Error, process::ExitCode}; +use yggdrasil_abi::{ + error::Error, + process::{ExitCode, Signal}, +}; use crate::{ mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}, @@ -46,4 +49,6 @@ extern "Rust" { pub fn __monotonic_timestamp() -> Result; pub fn __message_interrupt_controller() -> &'static dyn MessageInterruptController; + + pub fn __signal_process_group(group_id: u32, signal: Signal); } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 73f29421..c3627a67 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -8,10 +8,13 @@ strict_provenance, never_type, let_chains, - allocator_api + allocator_api, + maybe_uninit_uninit_array, + const_maybe_uninit_uninit_array )] use device_api::interrupt::MessageInterruptController; +use yggdrasil_abi::process::Signal; extern crate alloc; @@ -38,6 +41,11 @@ pub fn cpu_count() -> usize { unsafe { api::__cpu_count() } } +#[inline] +pub fn signal_process_group(group_id: u32, signal: Signal) { + unsafe { api::__signal_process_group(group_id, signal) } +} + #[repr(C)] pub struct AlignedTo { pub align: [Align; 0], diff --git a/lib/kernel-util/src/runtime/executor.rs b/lib/kernel-util/src/runtime/executor.rs index 864f2147..f4ff9ac2 100644 --- a/lib/kernel-util/src/runtime/executor.rs +++ b/lib/kernel-util/src/runtime/executor.rs @@ -1,17 +1,10 @@ use core::task::{Context, Poll}; use alloc::{boxed::Box, format, sync::Arc}; -use futures_util::{ - task::{waker_ref, ArcWake, WakerRef}, - Future, -}; +use futures_util::{task::waker_ref, Future}; use yggdrasil_abi::error::Error; -use crate::{ - api::__cpu_index, - cpu_index, - thread::{CurrentThread, Thread}, -}; +use crate::thread::Thread; use super::{ task::{Task, Termination}, diff --git a/lib/kernel-util/src/sync/spin_rwlock.rs b/lib/kernel-util/src/sync/spin_rwlock.rs index 64e6f368..5ed6feda 100644 --- a/lib/kernel-util/src/sync/spin_rwlock.rs +++ b/lib/kernel-util/src/sync/spin_rwlock.rs @@ -4,9 +4,7 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; -use yggdrasil_abi::error::Error; - -use super::{IrqGuard, LockMethod}; +use super::IrqGuard; struct RwLockInner { value: AtomicUsize, @@ -19,12 +17,12 @@ pub struct IrqSafeRwLock { pub struct IrqSafeRwLockReadGuard<'a, T> { lock: &'a IrqSafeRwLock, - guard: IrqGuard, + _guard: IrqGuard, } pub struct IrqSafeRwLockWriteGuard<'a, T> { lock: &'a IrqSafeRwLock, - guard: IrqGuard, + _guard: IrqGuard, } impl RwLockInner { @@ -106,13 +104,19 @@ impl IrqSafeRwLock { pub fn read(&self) -> IrqSafeRwLockReadGuard { let guard = IrqGuard::acquire(); self.inner.acquire_read(); - IrqSafeRwLockReadGuard { lock: self, guard } + IrqSafeRwLockReadGuard { + lock: self, + _guard: guard, + } } pub fn write(&self) -> IrqSafeRwLockWriteGuard { let guard = IrqGuard::acquire(); self.inner.acquire_write(); - IrqSafeRwLockWriteGuard { lock: self, guard } + IrqSafeRwLockWriteGuard { + lock: self, + _guard: guard, + } } unsafe fn release_read(&self) { diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util.rs index c5abb833..252ac535 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util.rs @@ -7,6 +7,8 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; +pub mod ring; + pub enum ConstAssert {} pub trait IsTrue {} diff --git a/src/util/ring.rs b/lib/kernel-util/src/util/ring.rs similarity index 89% rename from src/util/ring.rs rename to lib/kernel-util/src/util/ring.rs index b8d9e1dc..dd371e53 100644 --- a/src/util/ring.rs +++ b/lib/kernel-util/src/util/ring.rs @@ -63,7 +63,11 @@ impl RingBuffer { false } - /// Reads a single value from the ring without checking if it's empty + /// Reads a single value from the ring without checking if it's empty. + /// + /// # Safety + /// + /// The caller must perform the necessary checks to avoid reading beyond the write head. #[inline] pub unsafe fn read_single_unchecked(&mut self) -> T where @@ -74,7 +78,11 @@ impl RingBuffer { res } - /// Reads all entries available from `pos` to the write head + /// Reads all entries available from `pos` to the write head. + /// + /// # Safety + /// + /// The caller must perform the necessary checks to avoid reading beyond the write head. pub unsafe fn read_all_static(&mut self, pos: usize, buffer: &mut [T]) -> usize where T: Copy, diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index e86604fc..51f4ff95 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -15,10 +15,7 @@ use kernel_util::{ block, sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, }; -use yggdrasil_abi::{ - error::Error, - io::{MessageDestination, ReceivedMessageMetadata}, -}; +use yggdrasil_abi::{error::Error, io::MessageDestination}; use crate::{FileReadiness, FileRef}; diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 44b0efc0..d79aced8 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -6,7 +6,6 @@ use core::{ }; use alloc::{ - boxed::Box, collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; @@ -16,7 +15,9 @@ use kernel_util::{ }; use yggdrasil_abi::{ error::Error, - io::{DirectoryEntry, OpenOptions, RawFd, SeekFrom}, + io::{ + DeviceRequest, DirectoryEntry, OpenOptions, RawFd, SeekFrom, TerminalOptions, TerminalSize, + }, }; use crate::{ @@ -24,7 +25,7 @@ use crate::{ device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, traits::{Read, Seek, Write}, - FdPoll, FileReadiness, SharedMemory, + FdPoll, FileReadiness, PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave, SharedMemory, }; use self::{ @@ -61,10 +62,13 @@ pub enum File { Regular(RegularFile), Block(BlockFile), Char(CharFile), + AnonymousPipe(PipeEnd), Poll(FdPoll), Channel(ChannelDescriptor), SharedMemory(Arc), + PtySlave(PseudoTerminalSlave), + PtyMaster(PseudoTerminalMaster), } /// Contains a per-process fd -> FileRef map @@ -99,6 +103,18 @@ impl File { Ok(Arc::new(Self::SharedMemory(Arc::new(shm)))) } + /// Creates a pair of PTY master/slave + pub fn new_pseudo_terminal( + config: TerminalOptions, + size: TerminalSize, + ) -> (Arc, Arc) { + let (master, slave) = PseudoTerminal::new(config, size); + ( + Arc::new(Self::PtyMaster(master)), + Arc::new(Self::PtySlave(slave)), + ) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -202,11 +218,25 @@ impl File { match self { Self::Char(f) => f.device.0.poll_read(cx), Self::Channel(ch) => ch.poll_read(cx), + Self::Poll(ch) => ch.poll_read(cx), + Self::PtyMaster(f) => f.poll_read(cx), + Self::PtySlave(f) => f.poll_read(cx), // Polling not implemented, return ready immediately (XXX ?) _ => Poll::Ready(Ok(())), } } + /// Performs a device-specific request + pub fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match self { + Self::Char(f) => f.device.0.device_request(req), + Self::Block(f) => f.device.0.device_request(req), + Self::PtySlave(f) => f.device_request(req), + Self::PtyMaster(f) => f.device_request(req), + _ => Err(Error::InvalidOperation), + } + } + /// Interprets the file as a poll channel pub fn as_poll_channel(&self) -> Result<&FdPoll, Error> { if let Self::Poll(poll) = self { @@ -251,6 +281,8 @@ impl Read for File { Self::Block(file) => file.read(buf), Self::Char(file) => file.read(buf), Self::AnonymousPipe(pipe) => pipe.read(buf), + Self::PtySlave(pt) => pt.read(buf), + Self::PtyMaster(pt) => pt.read(buf), // TODO maybe allow reading FDs from poll channels as if they were regular streams? Self::Poll(_) => Err(Error::InvalidOperation), // TODO maybe allow reading messages from Channels? @@ -268,6 +300,8 @@ impl Write for File { Self::Block(file) => file.write(buf), Self::Char(file) => file.write(buf), Self::AnonymousPipe(pipe) => pipe.write(buf), + Self::PtySlave(pt) => pt.write(buf), + Self::PtyMaster(pt) => pt.write(buf), // TODO maybe allow adding FDs to poll channels this way Self::Poll(_) => Err(Error::InvalidOperation), // TODO maybe allow writing messages to Channels? @@ -323,6 +357,8 @@ impl fmt::Debug for File { Self::Poll(_) => f.debug_struct("Poll").finish_non_exhaustive(), Self::Channel(_) => f.debug_struct("Channel").finish_non_exhaustive(), Self::SharedMemory(_) => f.debug_struct("SharedMemory").finish_non_exhaustive(), + Self::PtySlave(_) => f.debug_struct("PtySlave").finish_non_exhaustive(), + Self::PtyMaster(_) => f.debug_struct("PtyMaster").finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 3e3ca8e1..186f7c70 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -17,6 +17,7 @@ pub(crate) mod ioctx; pub(crate) mod node; pub(crate) mod path; pub(crate) mod poll; +pub(crate) mod pty; pub(crate) mod shared_memory; pub(crate) mod traits; @@ -29,5 +30,6 @@ pub use node::{ RegularImpl, SymlinkImpl, }; pub use poll::FdPoll; +pub use pty::{PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave}; pub use shared_memory::SharedMemory; pub use traits::{FileReadiness, Read, Seek, Write}; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index ae53bab8..0f752b61 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -2,7 +2,6 @@ use core::{marker::PhantomData, str::FromStr}; use alloc::{ - boxed::Box, string::{String, ToString}, sync::Arc, vec::Vec, diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index f96d9a83..8668509f 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -13,7 +13,7 @@ use kernel_util::{ }; use yggdrasil_abi::{error::Error, io::RawFd}; -use crate::FileRef; +use crate::{FileReadiness, FileRef}; /// Poll channel implementation. Allows blocking until a file descriptor signals an event or a /// timeout is reached. @@ -75,3 +75,15 @@ impl<'a> Future for FdPollFuture<'a> { Poll::Pending } } + +impl FileReadiness for FdPoll { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + for (_, file) in self.fds.lock().unwrap().iter() { + if file.poll_read(cx).is_ready() { + return Poll::Ready(Ok(())); + } + } + + Poll::Pending + } +} diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs new file mode 100644 index 00000000..c5a9c1a7 --- /dev/null +++ b/lib/vfs/src/pty.rs @@ -0,0 +1,417 @@ +//! Pseudo-terminal devices + +// TODO handle erase key +// TODO handle werase key +use core::{ + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, + task::{Context, Poll}, +}; + +use alloc::sync::Arc; +use futures_util::Future; +use kernel_util::{ + block, + runtime::QueueWaker, + signal_process_group, + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, + util::ring::RingBuffer, +}; +use yggdrasil_abi::{ + error::Error, + io::{ + DeviceRequest, TerminalInputOptions, TerminalLineOptions, TerminalOptions, + TerminalOutputOptions, TerminalSize, + }, + process::Signal, +}; + +const CAPACITY: usize = 1024; + +struct PtyHalf { + ring: IrqSafeSpinlock>, + notify: QueueWaker, +} + +/// Pseudo-terminal shared device +pub struct PseudoTerminal { + config: IrqSafeRwLock, + signal_pgroup: IrqSafeRwLock>, + slave_to_master: PtyHalf, + master_to_slave: PtyHalf, + size: IrqSafeRwLock, + eof: AtomicBool, +} +/// Slave part of a PTY device +pub struct PseudoTerminalSlave { + pty: Arc, +} +/// Master part of a PTY device +pub struct PseudoTerminalMaster { + pty: Arc, +} + +fn input_discipline( + mut lock: IrqSafeSpinlockGuard>, + config: &TerminalOptions, + eof: &AtomicBool, + buffer: &mut [u8], +) -> Result { + let mut pos = 0; + + if !config.is_canonical() { + while !eof.load(Ordering::Acquire) && pos < buffer.len() && lock.is_readable() { + buffer[pos] = unsafe { lock.read_single_unchecked() }; + pos += 1; + } + Ok(pos) + } else { + while !eof.load(Ordering::Acquire) && pos < buffer.len() && lock.is_readable() { + let ch = unsafe { lock.read_single_unchecked() }; + + if ch == config.chars.erase { + pos = pos.saturating_sub(1); + continue; + } + + buffer[pos] = ch; + pos += 1; + + if ch == b'\n' { + break; + } + } + Ok(pos) + } +} + +impl PtyHalf { + pub fn new() -> Self { + Self { + ring: IrqSafeSpinlock::new(RingBuffer::new()), + notify: QueueWaker::new(), + } + } + + fn try_begin_read<'a>( + &'a self, + config: &TerminalOptions, + ) -> Option>> { + let lock = self.ring.lock(); + + let ready = if config.is_canonical() { + lock.is_readable() && lock.contains(&b'\n') + } else { + lock.is_readable() + }; + + if ready { + Some(lock) + } else { + None + } + } + + fn read( + &self, + config: &TerminalOptions, + eof: &AtomicBool, + buffer: &mut [u8], + ) -> Result { + if buffer.is_empty() || eof.load(Ordering::Acquire) { + return Ok(0); + } + + if let Some(lock) = self.try_begin_read(config) { + input_discipline(lock, config, eof, buffer) + } else { + block!(self.read_async(config, eof, buffer).await)? + } + } + + fn read_async<'a>( + &'a self, + config: &'a TerminalOptions, + eof: &'a AtomicBool, + buffer: &'a mut [u8], + ) -> impl Future> + 'a { + struct F<'f> { + config: &'f TerminalOptions, + eof: &'f AtomicBool, + half: &'f PtyHalf, + buffer: &'f mut [u8], + } + + impl<'f> Future for F<'f> { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.half.notify.register(cx.waker()); + + if self.eof.load(Ordering::Acquire) { + self.half.notify.remove(cx.waker()); + Poll::Ready(Ok(0)) + } else if let Some(lock) = self.half.try_begin_read(self.config) { + self.half.notify.remove(cx.waker()); + Poll::Ready(input_discipline(lock, self.config, self.eof, self.buffer)) + } else { + Poll::Pending + } + } + } + + F { + half: self, + eof, + buffer, + config, + } + } + + fn poll_read( + &self, + config: &TerminalOptions, + eof: &AtomicBool, + cx: &mut Context<'_>, + ) -> Poll> { + self.notify.register(cx.waker()); + + if eof.load(Ordering::Acquire) || self.try_begin_read(config).is_some() { + self.notify.remove(cx.waker()); + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + + fn read_raw(&self, eof: &AtomicBool, buffer: &mut [u8]) -> Result { + if buffer.is_empty() || eof.load(Ordering::Acquire) { + return Ok(0); + } + + let mut lock = self.ring.lock(); + if lock.is_readable() { + let mut pos = 0; + while !eof.load(Ordering::Acquire) && lock.is_readable() { + buffer[pos] = unsafe { lock.read_single_unchecked() }; + pos += 1; + } + + Ok(pos) + } else { + todo!() + } + } + + fn poll_raw(&self, eof: &AtomicBool, cx: &mut Context<'_>) -> Poll> { + self.notify.register(cx.waker()); + + if eof.load(Ordering::Acquire) || self.ring.lock().is_readable() { + self.notify.remove(cx.waker()); + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + + fn putc(&self, byte: u8, signal: bool) { + self.ring.lock().write(byte); + if signal { + self.notify.wake_all(); + } + } +} + +impl PseudoTerminal { + /// Creates a pair of PTY slave/master devices + pub fn new( + config: TerminalOptions, + size: TerminalSize, + ) -> (PseudoTerminalMaster, PseudoTerminalSlave) { + let pty = Arc::new(Self { + config: IrqSafeRwLock::new(config), + signal_pgroup: IrqSafeRwLock::new(None), + master_to_slave: PtyHalf::new(), + slave_to_master: PtyHalf::new(), + size: IrqSafeRwLock::new(size), + eof: AtomicBool::new(false), + }); + + let master = PseudoTerminalMaster { pty: pty.clone() }; + let slave = PseudoTerminalSlave { pty }; + + (master, slave) + } + + fn putc_from_slave(&self, byte: u8) { + let config = self.config.read(); + + if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + self.slave_to_master.putc(b'\r', true); + } + + self.slave_to_master.putc(byte, true) + } + + fn putc_from_master(&self, mut byte: u8) { + let config = self.config.read(); + + if byte == b'\r' && config.input.contains(TerminalInputOptions::CR_TO_NL) { + byte = b'\n'; + } + + if byte == b'\n' { + if config.is_echo_newline() { + if config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + self.slave_to_master.putc(b'\r', true); + } + self.slave_to_master.putc(byte, true); + } + } else if config.line.contains(TerminalLineOptions::ECHO) { + if byte.is_ascii_control() { + if byte != config.chars.erase && byte != config.chars.werase { + self.slave_to_master.putc(b'^', false); + self.slave_to_master.putc(byte + 0x40, true); + } + } else { + self.slave_to_master.putc(byte, true); + } + } + + if byte == config.chars.interrupt && config.line.contains(TerminalLineOptions::SIGNAL) { + self.slave_to_master.notify.wake_all(); + + if let Some(group_id) = *self.signal_pgroup.read() { + signal_process_group(group_id, Signal::Interrupted); + return; + } + } + + if byte == config.chars.eof { + self.slave_to_master.notify.wake_all(); + self.master_to_slave.notify.wake_all(); + self.eof.store(true, Ordering::Release); + + return; + } + + self.master_to_slave.putc( + byte, + !config.is_canonical() || byte == b'\n' || byte == b'\r', + ); + } + + fn write_from_slave(&self, buf: &[u8]) -> Result { + for &ch in buf { + self.putc_from_slave(ch); + } + Ok(buf.len()) + } + + fn write_from_master(&self, buf: &[u8]) -> Result { + for &ch in buf { + self.putc_from_master(ch); + } + Ok(buf.len()) + } + + fn read_from_slave(&self, buf: &mut [u8]) -> Result { + self.slave_to_master.read_raw(&self.eof, buf) + } + + fn read_from_master(&self, buf: &mut [u8]) -> Result { + let config = self.config.read(); + self.master_to_slave.read(&config, &self.eof, buf) + } + + fn poll_from_slave(&self, cx: &mut Context<'_>) -> Poll> { + self.slave_to_master.poll_raw(&self.eof, cx) + } + + fn poll_from_master(&self, cx: &mut Context<'_>) -> Poll> { + let config = self.config.read(); + self.master_to_slave.poll_read(&config, &self.eof, cx) + } + + fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + match req { + DeviceRequest::SetTerminalGroup(group_id) => { + self.signal_pgroup.write().replace(*group_id); + Ok(()) + } + DeviceRequest::GetTerminalOptions(options) => { + options.write(*self.config.read()); + Ok(()) + } + DeviceRequest::SetTerminalOptions(options) => { + *self.config.write() = *options; + Ok(()) + } + DeviceRequest::GetTerminalSize(size) => { + size.write(*self.size.read()); + Ok(()) + } + _ => Err(Error::InvalidOperation), + } + } +} + +impl PseudoTerminalSlave { + /// Reads from the master-to-slave half of the PTY + pub fn read(&self, buf: &mut [u8]) -> Result { + self.pty.read_from_master(buf) + } + + /// Writes to the slave-to-master half of the PTY + pub fn write(&self, buf: &[u8]) -> Result { + self.pty.write_from_slave(buf) + } + + /// Polls PTY read readiness + pub fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + self.pty.poll_from_master(cx) + } + + /// Performs a device-specific request to the PTY + pub fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + self.pty.device_request(req) + } +} + +impl PseudoTerminalMaster { + /// Reads from the slave-to-master half of the PTY + pub fn read(&self, buf: &mut [u8]) -> Result { + self.pty.read_from_slave(buf) + } + + /// Writes to the master-to-slave half of the PTY + pub fn write(&self, buf: &[u8]) -> Result { + self.pty.write_from_master(buf) + } + + /// Polls PTY read readiness + pub fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + self.pty.poll_from_slave(cx) + } + + /// Performs a device-specific request to the PTY + pub fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + self.pty.device_request(req) + } +} + +impl Drop for PseudoTerminalMaster { + fn drop(&mut self) { + self.pty.eof.store(true, Ordering::Release); + self.pty.master_to_slave.notify.wake_all(); + self.pty.slave_to_master.notify.wake_all(); + } +} + +impl Drop for PseudoTerminalSlave { + fn drop(&mut self) { + self.pty.eof.store(true, Ordering::Release); + self.pty.master_to_slave.notify.wake_all(); + self.pty.slave_to_master.notify.wake_all(); + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 9641c84c..58b7c2c3 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -275,7 +275,11 @@ pub trait CpuAccess: Sized { /// Returns current thread ID or none if idle fn current_thread_id(&self) -> Option; - /// Update the current thread ID + /// Update the current thread ID. + /// + /// # Safety + /// + /// Only meant to be called from within the scheduler. unsafe fn set_current_thread_id(&mut self, id: Option); } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index f4666f7d..689a0597 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -10,7 +10,7 @@ use device_api::{ Device, }; use git_version::git_version; -use kernel_fs::devfs::{self, CharDeviceType}; +use kernel_fs::devfs; use kernel_util::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index 193fbd87..bb650446 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -55,17 +55,17 @@ impl Inner { self.command.write(cmd); } - fn recv_timeout(&mut self, timeout: u64) -> Option { - let mut counter = 0; - while self.command.read() & Self::STATUS_OUTPUT_FULL == 0 { - counter += 1; - if counter > timeout { - return None; - } - core::hint::spin_loop(); - } - Some(self.data.read()) - } + // fn recv_timeout(&mut self, timeout: u64) -> Option { + // let mut counter = 0; + // while self.command.read() & Self::STATUS_OUTPUT_FULL == 0 { + // counter += 1; + // if counter > timeout { + // return None; + // } + // core::hint::spin_loop(); + // } + // Some(self.data.read()) + // } fn try_recv(&mut self) -> Option { if self.command.read() & Inner::STATUS_OUTPUT_FULL != 0 { @@ -163,7 +163,3 @@ impl PS2Controller { } } } - -async fn input_worker() { - loop {} -} diff --git a/src/debug.rs b/src/debug.rs index 57b3b14d..c717a66a 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,9 +2,12 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use kernel_util::{sync::IrqSafeSpinlock, util::StaticVector}; +use kernel_util::{ + sync::IrqSafeSpinlock, + util::{ring::RingBuffer, StaticVector}, +}; -use crate::{task::process::Process, util::ring::RingBuffer}; +use crate::task::process::Process; const MAX_DEBUG_SINKS: usize = 4; const RING_LOGGER_CAPACITY: usize = 65536; diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 8f13b060..0dfe5f41 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -157,7 +157,7 @@ impl PageProvider for LinearFramebuffer { } } - fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { Ok(()) } } diff --git a/src/device/input.rs b/src/device/input.rs index d1b772d6..dc6cc515 100644 --- a/src/device/input.rs +++ b/src/device/input.rs @@ -15,11 +15,10 @@ use futures_util::{task::AtomicWaker, Future}; use kernel_util::{ block, sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, + util::ring::RingBuffer, }; use vfs::{CharDevice, FileReadiness}; -use crate::util::ring::RingBuffer; - // #[derive(Default)] // pub struct InputState { // lshift: bool, @@ -133,7 +132,7 @@ impl CharDevice for KeyboardDevice { false } - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { + fn device_request(&self, _req: &mut DeviceRequest) -> Result<(), Error> { todo!() } diff --git a/src/device/tty.rs b/src/device/tty.rs index eb154f16..cf55e8d8 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -12,12 +12,9 @@ use abi::{ }; use device_api::serial::SerialDevice; use futures_util::Future; -use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; +use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBuffer}; -use crate::{ - task::process::{Process, ProcessId}, - util::ring::RingBuffer, -}; +use crate::task::process::{Process, ProcessId}; struct TerminalRing { buffer: IrqSafeSpinlock>, @@ -47,12 +44,6 @@ impl TerminalRing { self.notify.wake_all(); } - // TODO EOF not yet implemented - // fn signal_eof(&self) { - // self.eof.store(true, Ordering::Release); - // self.notify.wake_all(); - // } - fn read_blocking(&self) -> impl Future> + '_ { struct F<'f> { ring: &'f TerminalRing, @@ -87,15 +78,6 @@ impl TerminalRing { F { ring: self } } - - fn can_read_now(&self, canonical: bool) -> bool { - let lock = self.buffer.lock(); - if canonical { - lock.contains(&b'\n') - } else { - lock.is_readable() - } - } } struct TtyContextInner { diff --git a/src/mem/process.rs b/src/mem/process.rs index d7b02a2b..bab29089 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,7 +1,5 @@ //! Process address space structures and management functions -use core::ops::Range; - use abi::error::Error; use cfg_if::cfg_if; use kernel_util::{ @@ -116,7 +114,7 @@ impl PageProvider for VirtualRangeBacking { impl PartialEq for VirtualRangeBacking { fn eq(&self, other: &Self) -> bool { - return matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous); + matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous) } } diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 227b7e7d..8d5c0fb1 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -13,7 +13,6 @@ use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ mem::{ - phys, process::{ProcessAddressSpace, VirtualRangeBacking}, table::MapAttributes, }, diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 2d803408..44e91ec3 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -14,7 +14,6 @@ use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ mem::{ - phys, process::{ProcessAddressSpace, VirtualRangeBacking}, table::MapAttributes, ForeignPointer, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 688355ad..e35981b4 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -5,7 +5,8 @@ use abi::{ error::Error, io::{ DeviceRequest, DirectoryEntry, FileAttr, FileMode, MessageDestination, OpenOptions, - PollControl, RawFd, ReceivedMessageMetadata, SeekFrom, SentMessage, + PollControl, RawFd, ReceivedMessageMetadata, SeekFrom, SentMessage, TerminalOptions, + TerminalSize, }, mem::MappingSource, process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, @@ -20,7 +21,7 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::{phys, process::VirtualRangeBacking, table::MapAttributes}, + mem::{process::VirtualRangeBacking, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, @@ -236,8 +237,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io(process, |io| { let file = io.files.file(fd)?; - let node = file.node().ok_or(Error::InvalidFile)?; - node.device_request(req)?; + // let node = file.node().ok_or(Error::InvalidFile)?; + file.device_request(req)?; Ok(0) }) } @@ -408,7 +409,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } metadata.write(ReceivedMessageMetadata::Data(len)); - buf[..len].copy_from_slice(&data); + buf[..len].copy_from_slice(data); Ok(len) } @@ -432,6 +433,21 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(fd.0 as usize) }) } + SyscallFunction::CreatePty => { + let options = arg_user_ref::(args[0] as usize)?; + let size = arg_user_ref::(args[1] as usize)?; + let output = arg_user_mut::>(args[2] as usize)?; + + run_with_io(process, |mut io| { + let (master, slave) = File::new_pseudo_terminal(*options, *size); + let master_fd = io.files.place_file(master, true)?; + let slave_fd = io.files.place_file(slave, true)?; + + output.write((master_fd, slave_fd)); + + Ok(0) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; @@ -478,10 +494,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result }) { debugln!("{} requested terminal {:?}", pid, fd); let file = child_io.files.file(fd)?; - let node = file.node().ok_or(Error::InvalidFile)?; + // let node = file.node().ok_or(Error::InvalidFile)?; let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); + file.device_request(&mut req)?; - node.device_request(&mut req)?; + // node.device_request(&mut req)?; } drop(child_io); @@ -520,8 +537,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // after that let code = ExitCode::from(args[0] as i32); - thread.exit_process(code); - unreachable!() + thread.exit_process(code) } SyscallFunction::SendSignal => { let pid = ProcessId::from(args[0] as u32); diff --git a/src/task/context.rs b/src/task/context.rs index e3a7e23f..79268e1f 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -90,8 +90,7 @@ pub trait TaskContextImpl: Sized { ) -> ! { let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; let result = closure(); - Thread::current().exit(result.into_exit_code()); - unreachable!(); + Thread::current().exit(result.into_exit_code()) } let closure = Box::new(f); diff --git a/src/task/process.rs b/src/task/process.rs index 39ae143c..b55d81b9 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -435,3 +435,8 @@ impl ProcessId { Self(id) } } + +#[no_mangle] +fn __signal_process_group(group_id: u32, signal: Signal) { + Process::signal_group(ProcessId(group_id as _), signal) +} diff --git a/src/task/sched.rs b/src/task/sched.rs index b9eba858..1398f96a 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -1,14 +1,9 @@ //! Per-CPU queue implementation -use core::sync::atomic::Ordering; - -use alloc::{collections::VecDeque, sync::Arc, vec::Vec}; +use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; -use kernel_util::{ - sync::{IrqGuard, IrqSafeSpinlock}, - util::OneTimeInit, -}; +use kernel_util::{sync::IrqGuard, util::OneTimeInit}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess}, @@ -21,18 +16,6 @@ use super::{ Cpu, TaskContext, }; -// /// Per-CPU statistics -// #[derive(Default)] -// pub struct CpuQueueStats { -// /// Ticks spent idling -// pub idle_time: u64, -// /// Ticks spent running CPU tasks -// pub cpu_time: u64, -// -// /// Time since last measurement -// measure_time: u64, -// } - /// Per-CPU queue pub struct CpuQueue { queue: SegQueue, @@ -59,7 +42,11 @@ impl CpuQueue { self.index } - /// "Enters" the scheduler by selecting a first task to execute + /// "Enters" the scheduler by selecting a first task to execute. + /// + /// # Safety + /// + /// Only meant to be called once per each CPU when everything's ready. pub unsafe fn enter(&self) -> ! { let _guard = IrqGuard::acquire(); @@ -106,7 +93,11 @@ impl CpuQueue { (None, None) } - /// Selects a new thread from the queue and performs a context switch if necessary + /// Selects a new thread from the queue and performs a context switch if necessary. + /// + /// # Safety + /// + /// Only meant to be called from within the timer handler or the thread impl. pub unsafe fn yield_cpu(&self) -> bool { assert!(ArchitectureImpl::interrupt_mask()); @@ -124,7 +115,7 @@ impl CpuQueue { match sched.state { ThreadState::Ready => { - panic!("Thread {:?} was running, but is marked Ready", current.id); + self.queue.push(current.id); } ThreadState::Running => { sched.state = ThreadState::Ready; diff --git a/src/task/thread.rs b/src/task/thread.rs index fe12c438..4e0f5e81 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -240,6 +240,7 @@ impl Thread { /// Pushes a signal to the thread's signal queue pub fn raise_signal(&self, signal: Signal) { self.signal_queue.push(signal); + self.enqueue(); } // Scheduling @@ -248,7 +249,9 @@ impl Thread { pub fn enqueue_to(&self, queue: &'static CpuQueue) { let mut sched = self.sched.lock(); - assert_ne!(sched.state, ThreadState::Terminated); + if sched.state == ThreadState::Terminated { + return; + } match sched.state { ThreadState::Running | ThreadState::Ready => { diff --git a/src/util/mod.rs b/src/util/mod.rs index 44184486..26f1d001 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,7 +1,6 @@ //! Various kernel utility functions pub mod queue; -pub mod ring; /// Extension trait for [Iterator]s of [Result]s pub trait ResultIterator { From b760a5bad9cc57f4046e28cc568ff31108ce237e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 4 Jan 2024 21:30:36 +0200 Subject: [PATCH 143/211] block/ahci: add authors --- driver/block/ahci/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/driver/block/ahci/Cargo.toml b/driver/block/ahci/Cargo.toml index 34366bdb..a922f145 100644 --- a/driver/block/ahci/Cargo.toml +++ b/driver/block/ahci/Cargo.toml @@ -2,6 +2,7 @@ name = "ygg_driver_ahci" version = "0.1.0" edition = "2021" +authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 0e8860c719e96c211bbb1292ffcec9b8e34f6232 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 4 Jan 2024 23:04:34 +0200 Subject: [PATCH 144/211] arch/aarch64: fix aarch64 build --- Cargo.toml | 6 +- src/arch/aarch64/boot/mod.rs | 12 ++- src/arch/aarch64/cpu.rs | 138 ++++++++++++++++++++------------ src/arch/aarch64/exception.rs | 10 +-- src/arch/aarch64/gic/gicc.rs | 3 +- src/arch/aarch64/gic/gicd.rs | 3 +- src/arch/aarch64/gic/mod.rs | 18 +++-- src/arch/aarch64/mem/mod.rs | 34 ++++---- src/arch/aarch64/mem/process.rs | 28 ++++--- src/arch/aarch64/mem/table.rs | 28 ++++--- src/arch/aarch64/mod.rs | 56 ++++++------- src/arch/aarch64/timer.rs | 4 +- src/arch/mod.rs | 16 +++- src/device/devtree.rs | 6 +- src/device/serial/pl011.rs | 23 ++++-- src/task/sched.rs | 1 + 16 files changed, 228 insertions(+), 158 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b51c072b..cf505ff5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,6 @@ vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } # Drivers -ygg_driver_pci = { path = "driver/bus/pci" } -ygg_driver_nvme = { path = "driver/block/nvme" } -ygg_driver_ahci = { path = "driver/block/ahci" } ygg_driver_block = { path = "driver/block/core" } kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } @@ -57,6 +54,9 @@ acpi_lib = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branc acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } # TODO currently only supported here xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } +ygg_driver_pci = { path = "driver/bus/pci" } +ygg_driver_nvme = { path = "driver/block/nvme" } +ygg_driver_ahci = { path = "driver/block/ahci" } [features] default = ["fb_console"] diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 65adfc33..734e41b8 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -5,6 +5,14 @@ use aarch64_cpu::{ asm::barrier, registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; +use kernel_fs::devfs; +use kernel_util::{ + mem::{ + address::{IntoRaw, PhysicalAddress}, + table::EntryLevel, + }, + runtime, +}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use super::{ @@ -13,10 +21,8 @@ use super::{ }; use crate::{ arch::{aarch64::mem::table::L3, Architecture}, - fs::devfs, kernel_main, kernel_secondary_main, - mem::{address::IntoRaw, phys, table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET}, - task::runtime, + mem::{phys, KERNEL_VIRT_OFFSET}, }; unsafe fn pre_init_mmu() { diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 177927d2..70c46a8c 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -1,12 +1,22 @@ //! Per-CPU data structures -use core::sync::atomic::Ordering; +use core::{ + ops::{Deref, DerefMut}, + sync::atomic::Ordering, +}; use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + sync::{IrqGuard, IrqSafeSpinlock}, + util::OneTimeInit, +}; use tock_registers::interfaces::{Readable, Writeable}; -use crate::{arch::CpuMessage, panic, task::sched::CpuQueue}; +use crate::{ + arch::{CpuAccess, CpuMessage, LocalCpuAccess}, + panic, + task::{sched::CpuQueue, thread::ThreadId}, +}; use super::smp::CPU_COUNT; @@ -18,8 +28,17 @@ pub struct Cpu { id: u32, queue: OneTimeInit<&'static CpuQueue>, + thread_id: Option, } +/// Handle allowing safe access to the local CPU. +/// +/// # Note +/// +/// Obtaining this handle prevents IRQs from being delivered to ensure the context having this +/// struct cannot be interrupted by another (which could obtain it as well). +pub struct LocalCpu(&'static mut Cpu, IrqGuard); + struct IpiQueue { data: IrqSafeSpinlock>, } @@ -47,17 +66,11 @@ impl IpiQueue { } impl Cpu { - /// Returns a safe reference to the local CPU's private data structure - #[inline(always)] - pub fn local<'a>() -> &'a Self { - Self::get_local().unwrap() - } - - /// Returns the local CPU data structure reference, if it was set up - #[inline(always)] - pub fn get_local<'a>() -> Option<&'a Self> { - let tpidr = TPIDR_EL1.get() as *mut Cpu; - unsafe { tpidr.as_ref() } + /// Sets up global list of interprocessor message queues + pub fn init_ipi_queues() { + IPI_QUEUES.init(Vec::from_iter( + (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), + )); } /// Sets up the local CPU's private data structure. @@ -69,60 +82,81 @@ impl Cpu { let this = Box::new(Cpu { id: Self::local_id(), queue: OneTimeInit::new(), + thread_id: None, }); TPIDR_EL1.set(Box::into_raw(this) as _); } +} - /// Sets up the local CPU's execution queue. - pub fn init_queue(&self, queue: &'static CpuQueue) { - self.queue.init(queue); +impl CpuAccess for Cpu { + type Local = LocalCpu; + + fn id(&self) -> u32 { + self.id } - /// Returns the local CPU's execution queue. - pub fn queue(&self) -> &'static CpuQueue { - self.queue.get() - } - - /// Returns the index of the local CPU - #[inline(always)] - pub fn local_id() -> u32 { + fn local_id() -> u32 { (MPIDR_EL1.get() & 0xFF) as _ } - /// Inserts an IPI message to the back of the target CPU's message queue - pub fn push_ipi_queue(cpu_id: u32, msg: CpuMessage) { - let ipi_queue = &IPI_QUEUES.get()[cpu_id as usize]; - ipi_queue.push(msg); + fn try_local() -> Option { + let guard = IrqGuard::acquire(); + let tpidr = TPIDR_EL1.get() as *mut Cpu; + unsafe { tpidr.as_mut() }.map(|cpu| LocalCpu(cpu, guard)) } - /// Pops the first IPI message received for this CPU. - /// - /// # Note - /// - /// Currently the queue consists of only one entry, so the CPU will only receive the last one. - pub fn get_ipi(&self) -> Option { - let ipi_queue = &IPI_QUEUES.get()[self.id as usize]; - ipi_queue.pop() + fn get_queue(&self) -> Option<&'static CpuQueue> { + self.queue.try_get().copied() } - /// Sets up global list of interprocessor message queues - pub fn init_ipi_queues() { - IPI_QUEUES.init(Vec::from_iter( - (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), - )); + fn init_queue(&mut self, queue: &'static CpuQueue) { + self.queue.init(queue); } - /// Gets an IPI message from the processor's queue and takes corresponding actions. If there is - /// none, this is treated as a spurious IPI and ignored. See [CpuMessage]. - pub fn handle_ipi(&self) { - let Some(msg) = self.get_ipi() else { - warnln!("Spurious IPI in cpu{}", self.id); - return; - }; + unsafe fn set_current_thread_id(&mut self, id: Option) { + self.thread_id = id; + } - match msg { - CpuMessage::Panic => panic::panic_secondary(), - CpuMessage::Shutdown => todo!(), + fn current_thread_id(&self) -> Option { + self.thread_id + } + + fn push_ipi(id: u32, msg: CpuMessage) { + if let Some(q) = IPI_QUEUES.try_get().and_then(|q| q.get(id as usize)) { + q.push(msg); + } + } + + fn handle_ipi(&mut self) { + if let Some(ipi) = IPI_QUEUES + .try_get() + .and_then(|q| q.get(self.id() as usize)) + .and_then(|q| q.pop()) + { + match ipi { + CpuMessage::Panic => panic::panic_secondary(), + CpuMessage::Shutdown => todo!(), + } } } } + +impl LocalCpuAccess for LocalCpu { + fn into_guard(self) -> IrqGuard { + self.1 + } +} + +impl Deref for LocalCpu { + type Target = Cpu; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl DerefMut for LocalCpu { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + } +} diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 6984c228..50152751 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -323,11 +323,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { // BRK in AArch64 0b111100 => { let thread = Thread::current(); - warnln!( - "Thread {} {:?} hit a breakpoint", - thread.id(), - thread.name() - ); + warnln!("Thread {} {:?} hit a breakpoint", thread.id, thread.name); thread.raise_signal(Signal::Aborted); } _ => { @@ -337,8 +333,8 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { let thread = Thread::current(); warnln!( "Data abort in {} {:?} at {:#x} with address {:#x}", - thread.id(), - thread.name(), + thread.id, + thread.name, ELR_EL1.get(), FAR_EL1.get() ); diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs index 1f11d485..745c236c 100644 --- a/src/arch/aarch64/gic/gicc.rs +++ b/src/arch/aarch64/gic/gicc.rs @@ -1,12 +1,11 @@ //! ARM GICv2 CPU registers +use kernel_util::mem::device::DeviceMemoryIo; use tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, register_structs, registers::ReadWrite, }; -use crate::mem::device::DeviceMemoryIo; - register_bitfields! { u32, CTLR [ diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index 60314cc8..dc1aaf2c 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -1,5 +1,6 @@ //! ARM GICv2 Distributor registers use device_api::interrupt::IpiDeliveryTarget; +use kernel_util::mem::device::DeviceMemoryIo; use spinning_top::Spinlock; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, @@ -7,8 +8,6 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use crate::mem::device::DeviceMemoryIo; - register_bitfields! { u32, CTLR [ diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index f912f143..bad6b6b9 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -12,17 +12,19 @@ use device_api::{ }, Device, }; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + mem::{ + address::{FromRaw, PhysicalAddress}, + device::{DeviceMemoryIo, RawDeviceMemoryMapping}, + }, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use crate::{ - arch::{aarch64::IrqNumber, Architecture, CpuMessage}, + arch::{aarch64::IrqNumber, Architecture, CpuAccess, CpuMessage}, device::devtree::{self, DevTreeIndexPropExt}, device_tree_driver, - mem::{ - address::FromRaw, - device::{DeviceMemoryIo, RawDeviceMemoryMapping}, - PhysicalAddress, - }, }; use self::{gicc::Gicc, gicd::Gicd}; @@ -141,7 +143,7 @@ impl LocalInterruptController for Gic { let local = Cpu::local_id(); for i in 0..CPU_COUNT.load(Ordering::Acquire) { if i != local as usize { - Cpu::push_ipi_queue(i as u32, msg); + Cpu::push_ipi(i as u32, msg); } } } diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index e607f048..b596b426 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -6,19 +6,21 @@ use core::{ use abi::error::Error; use cfg_if::cfg_if; -use kernel_util::util::OneTimeInit; +use kernel_util::{ + mem::{ + address::{FromRaw, PhysicalAddress}, + device::RawDeviceMemoryMapping, + table::{EntryLevel, EntryLevelExt}, + }, + util::OneTimeInit, +}; use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1}; use static_assertions::const_assert_eq; use tock_registers::interfaces::Writeable; -use crate::mem::{ - address::{FromRaw, IntoRaw, KernelImageObject}, - device::RawDeviceMemoryMapping, - table::EntryLevel, - PhysicalAddress, KERNEL_VIRT_OFFSET, -}; +use crate::mem::{address::KernelImageObject, KERNEL_VIRT_OFFSET}; use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}; @@ -36,8 +38,8 @@ cfg_if! { } // Precomputed mappings -const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); -const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_L1_INDEX: usize = (KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE).page_index::(); +const KERNEL_START_L2_INDEX: usize = (KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE).page_index::(); const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT; // Must not be zero, should be at 4MiB @@ -177,7 +179,7 @@ unsafe fn unmap_early_page(address: usize) { panic!("Tried to unmap invalid early mapping: {:#x}", address); } - let l3i = L3::index(address - EARLY_MAPPING_OFFSET); + let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); assert!(EARLY_MAPPING_L3[l3i].is_present()); EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; @@ -262,14 +264,14 @@ pub(super) unsafe fn map_device_memory( ) -> Result { // debugln!("Map {}B @ {:#x}", size, base); let l3_aligned = base.page_align_down::(); - let l3_offset = L3::page_offset(base.into_raw()); - let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE; + let l3_offset = base.page_offset::(); + let page_count = (l3_offset + size).page_count::(); if page_count > 256 { // Large mapping, use L2 mapping instead let l2_aligned = base.page_align_down::(); - let l2_offset = L2::page_offset(base.into_raw()); - let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE; + let l2_offset = base.page_offset::(); + let page_count = (l2_offset + size).page_count::(); let base_address = map_device_memory_l2(l2_aligned, page_count)?; let address = base_address + l2_offset; @@ -304,8 +306,8 @@ pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { L3::SIZE => { for i in 0..map.page_count { let page = map.base_address + i * L3::SIZE; - let l2i = L2::index(page); - let l3i = L3::index(page); + let l2i = page.page_index::(); + let l3i = page.page_index::(); assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index 51f740c6..bedc8b8e 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -2,14 +2,16 @@ use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; +use kernel_util::mem::{ + address::{AsPhysicalAddress, PhysicalAddress}, + pointer::PhysicalRefMut, + table::{EntryLevel, EntryLevelExt}, +}; use crate::mem::{ - address::AsPhysicalAddress, phys, - pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, - table::{EntryLevel, MapAttributes, NextPageTable}, - PhysicalAddress, + table::{MapAttributes, NextPageTable}, }; use super::table::{PageEntry, PageTable, L1, L2, L3}; @@ -75,9 +77,9 @@ impl ProcessAddressSpaceImpl { entry: PageEntry, overwrite: bool, ) -> Result<(), Error> { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); let mut l2 = self.l1.get_mut_or_alloc(l1i)?; let mut l3 = l2.get_mut_or_alloc(l2i)?; @@ -93,9 +95,9 @@ impl ProcessAddressSpaceImpl { } fn pop_l3_entry(&mut self, virt: usize) -> Result { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); // TODO somehow drop tables if they're known to be empty? let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; @@ -110,9 +112,9 @@ impl ProcessAddressSpaceImpl { } fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); let l2 = self.l1.get(l1i)?; let l3 = l2.get(l2i)?; diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs index 36daff94..f61f2c32 100644 --- a/src/arch/aarch64/mem/table.rs +++ b/src/arch/aarch64/mem/table.rs @@ -5,13 +5,15 @@ use core::{ use abi::error::Error; use bitflags::bitflags; +use kernel_util::mem::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + pointer::{PhysicalRef, PhysicalRefMut}, + table::EntryLevel, +}; use crate::mem::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw}, phys, - pointer::{PhysicalRef, PhysicalRefMut}, - table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, - PhysicalAddress, + table::{MapAttributes, NextPageTable, NonTerminalEntryLevel}, }; bitflags! { @@ -60,23 +62,23 @@ pub struct L2; #[derive(Clone, Copy)] pub struct L3; -impl const EntryLevel for L1 { - const SHIFT: usize = 30; -} - impl NonTerminalEntryLevel for L1 { type NextLevel = L2; } -impl const EntryLevel for L2 { - const SHIFT: usize = 21; -} - impl NonTerminalEntryLevel for L2 { type NextLevel = L3; } -impl const EntryLevel for L3 { +impl EntryLevel for L1 { + const SHIFT: usize = 30; +} + +impl EntryLevel for L2 { + const SHIFT: usize = 21; +} + +impl EntryLevel for L3 { const SHIFT: usize = 12; } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 5f3b3ddf..d76beadd 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -13,7 +13,15 @@ use device_api::{ }; use fdt_rs::base::DevTree; use git_version::git_version; -use kernel_util::util::OneTimeInit; +use kernel_util::{ + mem::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::RawDeviceMemoryMapping, + pointer::PhysicalRef, + table::{EntryLevel, EntryLevelExt}, + }, + util::OneTimeInit, +}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ @@ -29,13 +37,8 @@ use crate::{ }, fs::{Initrd, INITRD_DATA}, mem::{ - address::{FromRaw, IntoRaw}, - device::RawDeviceMemoryMapping, heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, - pointer::PhysicalRef, - table::EntryLevel, - PhysicalAddress, }, }; @@ -80,6 +83,8 @@ impl Architecture for AArch64 { type IrqNumber = IrqNumber; + type L3 = L3; + unsafe fn start_application_processors(&self) { let dt = self.dt.get(); if let Err(error) = smp::start_ap_cores(dt) { @@ -125,7 +130,7 @@ impl Architecture for AArch64 { _memory_start: PhysicalAddress, memory_end: PhysicalAddress, ) -> Result<(), Error> { - let end_l1i = L1::index(memory_end.page_align_up::().into_raw()); + let end_l1i = memory_end.page_align_up::().page_index::(); if end_l1i > mem::RAM_MAPPING_L1_COUNT { todo!() } @@ -142,25 +147,23 @@ impl Architecture for AArch64 { Ok(()) } - fn virtualize(address: PhysicalAddress) -> Result { - let raw: usize = address.into_raw(); - if raw < *mem::MEMORY_LIMIT.get() { - Ok(raw + mem::RAM_MAPPING_OFFSET) + fn virtualize(address: u64) -> usize { + let address = address as usize; + if address < *mem::MEMORY_LIMIT.get() { + address + mem::RAM_MAPPING_OFFSET } else { - errorln!("Invalid physical address: {:#x}", address); - Err(Error::InvalidMemoryOperation) + panic!("Invalid physical address: {:#x}", address); } } - fn physicalize(address: usize) -> Result { + fn physicalize(address: usize) -> u64 { if address < mem::RAM_MAPPING_OFFSET || address - mem::RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() { - errorln!("Not a virtualized physical address: {:#x}", address); - return Err(Error::InvalidMemoryOperation); + panic!("Not a virtualized physical address: {:#x}", address); } - Ok(PhysicalAddress::from_raw(address - mem::RAM_MAPPING_OFFSET)) + (address - mem::RAM_MAPPING_OFFSET) as _ } fn local_interrupt_controller( @@ -362,23 +365,22 @@ impl AArch64 { infoln!("Initializing aarch64 platform"); let nodes = dt.root().children(); - if let Err(error) = devtree::enumerate_dt( - address_cells, - size_cells, - nodes, - |_, probe| { + if let Err(error) = + devtree::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { // Skip chosen-stdout, already initialized - if let Some(ref chosen_stdout) = chosen_stdout && chosen_stdout.name() == probe.node.name() { - return Ok(()); - } + if let Some(ref chosen_stdout) = chosen_stdout + && chosen_stdout.name() == probe.node.name() + { + return Ok(()); + } if let Some((device, _)) = devtree::probe_dt_node(&probe) { device.init()?; } Ok(()) - }, - ) { + }) + { warnln!( "{} errors encountered when initializing platform devices", error diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 9a7fe43e..6c463b53 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -6,12 +6,12 @@ use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0 use abi::error::Error; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; +use kernel_util::runtime; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, + arch::{aarch64::IrqNumber, Architecture, CpuAccess, ARCHITECTURE}, device_tree_driver, - task::runtime, }; use super::cpu::Cpu; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 58b7c2c3..2332174f 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -251,7 +251,9 @@ pub trait CpuAccess: Sized { } /// Returns the ID of the local processor or 0 if the processor has not yet been initialized - fn local_id() -> u32; + fn local_id() -> u32 { + Self::try_local().map(|cpu| cpu.id()).unwrap_or(0) + } /// Initializes the CPU's scheduling queue. /// @@ -281,6 +283,18 @@ pub trait CpuAccess: Sized { /// /// Only meant to be called from within the scheduler. unsafe fn set_current_thread_id(&mut self, id: Option); + + /// Push an IPI message to the CPU's queue + fn push_ipi(id: u32, msg: CpuMessage) { + let _ = id; + let _ = msg; + // Not implemented by default + } + + /// Pops an IPI message (if any is present) from the CPU's queue and handles it + fn handle_ipi(&mut self) { + // Not implemented by default + } } // External API for architecture specifics diff --git a/src/device/devtree.rs b/src/device/devtree.rs index 3009ceaa..19733857 100644 --- a/src/device/devtree.rs +++ b/src/device/devtree.rs @@ -9,11 +9,9 @@ use fdt_rs::{ index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, prelude::PropReader, }; +use kernel_util::mem::address::{FromRaw, PhysicalAddress}; -use crate::{ - debug::LogLevel, - mem::{address::FromRaw, phys::PhysicalMemoryRegion, PhysicalAddress}, -}; +use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; use super::register_device; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 58910108..4114bb20 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -2,25 +2,32 @@ use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use futures_util::task::{Context, Poll}; +use kernel_fs::devfs::{self, CharDeviceType}; +use kernel_util::{ + block, + mem::{ + address::{FromRaw, PhysicalAddress}, + device::DeviceMemoryIo, + }, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use vfs::CharDevice; +use vfs::{CharDevice, FileReadiness}; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, - block, debug::{self, DebugSink, LogLevel}, device::{ devtree::{self, DevTreeIndexPropExt}, tty::{TtyContext, TtyDevice}, }, device_tree_driver, - fs::devfs::{self, CharDeviceType}, - mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, task::process::ProcessId, }; @@ -108,6 +115,12 @@ impl TtyDevice for Pl011 { } } +impl FileReadiness for Pl011 { + fn poll_read(&self, _cx: &mut Context<'_>) -> Poll> { + todo!() + } +} + // impl CharDevice for Pl011 { // fn write(&self, blocking: bool, data: &[u8]) -> Result { // assert!(blocking); diff --git a/src/task/sched.rs b/src/task/sched.rs index 1398f96a..c19eb529 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -214,5 +214,6 @@ static QUEUES: OneTimeInit> = OneTimeInit::new(); /// Initializes the global queue list pub fn init_queues(queues: Vec) { + debugln!("init_queues!!!"); QUEUES.init(queues); } From e7a6243cb3b1bb2dd47a610baf91837df66cfa07 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 8 Jan 2024 18:44:55 +0200 Subject: [PATCH 145/211] WIP: proc: fork()/execve() implementation for c compat --- lib/vfs/src/file/mod.rs | 18 ++++- lib/vfs/src/pty.rs | 77 +++++++------------ lib/vmalloc/src/lib.rs | 27 +++++-- src/arch/x86_64/context.S | 18 +++++ src/arch/x86_64/context.rs | 60 ++++++++++++++- src/arch/x86_64/exception.rs | 8 +- src/arch/x86_64/syscall.rs | 54 ++++++++++---- src/main.rs | 3 +- src/mem/process.rs | 111 ++++++++++++++++++++++++++- src/proc/exec.rs | 95 ++++++++++++++++------- src/syscall/mod.rs | 57 +++++++++++++- src/task/context.rs | 11 ++- src/task/process.rs | 141 +++++++++++++++++++++++++++-------- src/task/sched.rs | 13 +++- src/task/thread.rs | 14 +++- 15 files changed, 559 insertions(+), 148 deletions(-) diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index d79aced8..5634dd39 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -186,11 +186,18 @@ impl File { } /// Clones an open file for sending it to another process - pub fn send(&self) -> Result, Error> { - match self { + pub fn send(self: &Arc) -> Result, Error> { + match self.as_ref() { + Self::Char(_) => Ok(self.clone()), + Self::Block(_) => todo!(), Self::Regular(file) => Ok(Arc::new(Self::Regular(file.clone()))), Self::SharedMemory(shm) => Ok(Arc::new(Self::SharedMemory(shm.clone()))), - _ => Err(Error::InvalidOperation), + Self::PtySlave(pt) => Ok(Arc::new(Self::PtySlave(pt.clone()))), + Self::PtyMaster(pt) => Ok(Arc::new(Self::PtyMaster(pt.clone()))), + _ => { + log::info!("Invalid file send(): {:?}", self); + Err(Error::InvalidOperation) + } } } @@ -416,6 +423,11 @@ impl FileSet { self.map.retain(predicate); } + /// XXX + pub fn iter(&self) -> impl Iterator { + self.map.iter() + } + /// Closes all of the files pub fn close_all(&mut self) { self.map.clear(); diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index c5a9c1a7..fa172abe 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -40,13 +40,14 @@ pub struct PseudoTerminal { slave_to_master: PtyHalf, master_to_slave: PtyHalf, size: IrqSafeRwLock, - eof: AtomicBool, } /// Slave part of a PTY device +#[derive(Clone)] pub struct PseudoTerminalSlave { pty: Arc, } /// Master part of a PTY device +#[derive(Clone)] pub struct PseudoTerminalMaster { pty: Arc, } @@ -54,21 +55,23 @@ pub struct PseudoTerminalMaster { fn input_discipline( mut lock: IrqSafeSpinlockGuard>, config: &TerminalOptions, - eof: &AtomicBool, buffer: &mut [u8], ) -> Result { let mut pos = 0; if !config.is_canonical() { - while !eof.load(Ordering::Acquire) && pos < buffer.len() && lock.is_readable() { + while pos < buffer.len() && lock.is_readable() { buffer[pos] = unsafe { lock.read_single_unchecked() }; pos += 1; } Ok(pos) } else { - while !eof.load(Ordering::Acquire) && pos < buffer.len() && lock.is_readable() { + while pos < buffer.len() && lock.is_readable() { let ch = unsafe { lock.read_single_unchecked() }; + if ch == config.chars.eof { + break; + } if ch == config.chars.erase { pos = pos.saturating_sub(1); continue; @@ -112,32 +115,25 @@ impl PtyHalf { } } - fn read( - &self, - config: &TerminalOptions, - eof: &AtomicBool, - buffer: &mut [u8], - ) -> Result { - if buffer.is_empty() || eof.load(Ordering::Acquire) { + fn read(&self, config: &TerminalOptions, buffer: &mut [u8]) -> Result { + if buffer.is_empty() { return Ok(0); } if let Some(lock) = self.try_begin_read(config) { - input_discipline(lock, config, eof, buffer) + input_discipline(lock, config, buffer) } else { - block!(self.read_async(config, eof, buffer).await)? + block!(self.read_async(config, buffer).await)? } } fn read_async<'a>( &'a self, config: &'a TerminalOptions, - eof: &'a AtomicBool, buffer: &'a mut [u8], ) -> impl Future> + 'a { struct F<'f> { config: &'f TerminalOptions, - eof: &'f AtomicBool, half: &'f PtyHalf, buffer: &'f mut [u8], } @@ -148,12 +144,9 @@ impl PtyHalf { fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { self.half.notify.register(cx.waker()); - if self.eof.load(Ordering::Acquire) { + if let Some(lock) = self.half.try_begin_read(self.config) { self.half.notify.remove(cx.waker()); - Poll::Ready(Ok(0)) - } else if let Some(lock) = self.half.try_begin_read(self.config) { - self.half.notify.remove(cx.waker()); - Poll::Ready(input_discipline(lock, self.config, self.eof, self.buffer)) + Poll::Ready(input_discipline(lock, self.config, self.buffer)) } else { Poll::Pending } @@ -162,21 +155,15 @@ impl PtyHalf { F { half: self, - eof, buffer, config, } } - fn poll_read( - &self, - config: &TerminalOptions, - eof: &AtomicBool, - cx: &mut Context<'_>, - ) -> Poll> { + fn poll_read(&self, config: &TerminalOptions, cx: &mut Context<'_>) -> Poll> { self.notify.register(cx.waker()); - if eof.load(Ordering::Acquire) || self.try_begin_read(config).is_some() { + if self.try_begin_read(config).is_some() { self.notify.remove(cx.waker()); Poll::Ready(Ok(())) } else { @@ -184,15 +171,15 @@ impl PtyHalf { } } - fn read_raw(&self, eof: &AtomicBool, buffer: &mut [u8]) -> Result { - if buffer.is_empty() || eof.load(Ordering::Acquire) { + fn read_raw(&self, buffer: &mut [u8]) -> Result { + if buffer.is_empty() { return Ok(0); } let mut lock = self.ring.lock(); if lock.is_readable() { let mut pos = 0; - while !eof.load(Ordering::Acquire) && lock.is_readable() { + while lock.is_readable() { buffer[pos] = unsafe { lock.read_single_unchecked() }; pos += 1; } @@ -203,10 +190,10 @@ impl PtyHalf { } } - fn poll_raw(&self, eof: &AtomicBool, cx: &mut Context<'_>) -> Poll> { + fn poll_raw(&self, cx: &mut Context<'_>) -> Poll> { self.notify.register(cx.waker()); - if eof.load(Ordering::Acquire) || self.ring.lock().is_readable() { + if self.ring.lock().is_readable() { self.notify.remove(cx.waker()); Poll::Ready(Ok(())) } else { @@ -234,7 +221,6 @@ impl PseudoTerminal { master_to_slave: PtyHalf::new(), slave_to_master: PtyHalf::new(), size: IrqSafeRwLock::new(size), - eof: AtomicBool::new(false), }); let master = PseudoTerminalMaster { pty: pty.clone() }; @@ -288,10 +274,7 @@ impl PseudoTerminal { } if byte == config.chars.eof { - self.slave_to_master.notify.wake_all(); - self.master_to_slave.notify.wake_all(); - self.eof.store(true, Ordering::Release); - + self.master_to_slave.putc(byte, true); return; } @@ -316,21 +299,21 @@ impl PseudoTerminal { } fn read_from_slave(&self, buf: &mut [u8]) -> Result { - self.slave_to_master.read_raw(&self.eof, buf) + self.slave_to_master.read_raw(buf) } fn read_from_master(&self, buf: &mut [u8]) -> Result { let config = self.config.read(); - self.master_to_slave.read(&config, &self.eof, buf) + self.master_to_slave.read(&config, buf) } fn poll_from_slave(&self, cx: &mut Context<'_>) -> Poll> { - self.slave_to_master.poll_raw(&self.eof, cx) + self.slave_to_master.poll_raw(cx) } fn poll_from_master(&self, cx: &mut Context<'_>) -> Poll> { let config = self.config.read(); - self.master_to_slave.poll_read(&config, &self.eof, cx) + self.master_to_slave.poll_read(&config, cx) } fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { @@ -402,16 +385,14 @@ impl PseudoTerminalMaster { impl Drop for PseudoTerminalMaster { fn drop(&mut self) { - self.pty.eof.store(true, Ordering::Release); - self.pty.master_to_slave.notify.wake_all(); - self.pty.slave_to_master.notify.wake_all(); + self.pty + .master_to_slave + .putc(self.pty.config.read().chars.eof, true); } } impl Drop for PseudoTerminalSlave { fn drop(&mut self) { - self.pty.eof.store(true, Ordering::Release); - self.pty.master_to_slave.notify.wake_all(); - self.pty.slave_to_master.notify.wake_all(); + // TODO } } diff --git a/lib/vmalloc/src/lib.rs b/lib/vmalloc/src/lib.rs index 70b918ab..d4a408b8 100644 --- a/lib/vmalloc/src/lib.rs +++ b/lib/vmalloc/src/lib.rs @@ -20,13 +20,14 @@ type PfnIndex = u64; /// Metadata associated with an allocated memory region. The [Eq] trait is used to coalesce "equal" /// regions if they "touch". -pub trait RangeData: Eq {} +pub trait RangeData: Eq + Clone {} fn ie(from: PfnIndex, to: PfnIndex) -> InclusiveInterval { InclusiveInterval::from(from..to) } /// Main virtual memory allocator +#[derive(Clone)] pub struct VirtualMemoryAllocator { map: DiscreteRangeMap, D>, outer_range: InclusiveInterval, @@ -41,6 +42,14 @@ impl VirtualMemoryAllocator { } } + /// XXX + pub fn regions(&self) -> impl Iterator, &D)> { + self.map.iter().map(|(range, data)| { + let range = range.start() as usize..range.end() as usize + 1; + (range, data) + }) + } + /// Allocates a contiguous range of virtual address space and associates metadata with it pub fn allocate(&mut self, page_count: usize, data: D) -> Result { let start_pfn = self @@ -80,10 +89,7 @@ impl VirtualMemoryAllocator { start_pfn: usize, page_count: usize, mut release: F, - ) -> Result<(), Error> - where - D: Clone, - { + ) -> Result<(), Error> { let end_pfn = (start_pfn + page_count) as PfnIndex; let start_pfn = start_pfn as PfnIndex; @@ -94,4 +100,15 @@ impl VirtualMemoryAllocator { release(origin.start() as _, range, data) }) } + + /// XXX + pub fn clear, D) -> Result<(), Error>>( + &mut self, + mut release: F, + ) -> Result<(), Error> { + self.map.drain().try_for_each(|(range, data)| { + let range = range.start() as usize..range.end() as usize + 1; + release(range, data) + }) + } } diff --git a/src/arch/x86_64/context.S b/src/arch/x86_64/context.S index d89ff421..f9b0a65c 100644 --- a/src/arch/x86_64/context.S +++ b/src/arch/x86_64/context.S @@ -54,11 +54,29 @@ .global __x86_64_task_enter_user .global __x86_64_task_enter_kernel +.global __x86_64_task_enter_from_fork .global __x86_64_enter_task .global __x86_64_switch_task .section .text +__x86_64_task_enter_from_fork: + xorq %rax, %rax + + xorq %rcx, %rcx + xorq %r11, %r11 + + popq %rdi + popq %rsi + popq %rdx + popq %r10 + popq %r8 + popq %r9 + + swapgs + + iretq + __x86_64_task_enter_user: // User stack pointer popq %rcx diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 195ea309..3f96bb09 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -2,9 +2,16 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; -use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw}; +use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; +use tock_registers::interfaces::Readable; -use crate::{arch::x86_64::mem::KERNEL_TABLES, mem::phys, task::context::TaskContextImpl}; +use crate::{ + arch::x86_64::{mem::KERNEL_TABLES, registers::CR3}, + mem::phys, + task::context::TaskContextImpl, +}; + +use super::syscall::SyscallFrame; struct StackBuilder { base: usize, @@ -77,6 +84,54 @@ impl StackBuilder { unsafe impl Sync for TaskContext {} +impl TaskContext { + /// XXX + pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result { + const USER_TASK_PAGES: usize = 8; + + let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + // iretq frame + stack.push(0x1B); + stack.push(frame.user_sp as _); + stack.push(0x200); + stack.push(0x23); + stack.push(frame.user_ip as _); + + stack.push(frame.args[5] as _); // r9 + stack.push(frame.args[4] as _); // r8 + stack.push(frame.args[3] as _); // r10 + stack.push(frame.args[2] as _); // rdx + stack.push(frame.args[1] as _); // rsi + stack.push(frame.args[0] as _); // rdi + + // callee-saved registers + stack.push(__x86_64_task_enter_from_fork as _); + + stack.push(cr3 as _); + + stack.push(frame.rbp as _); + stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame + stack.push(frame.r15 as _); + stack.push(frame.r14 as _); + stack.push(frame.r13 as _); + stack.push(frame.r12 as _); + stack.push(frame.rbx as _); + + let sp = stack.build(); + let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; + debugln!("Fork TSS.RSP0 = {:#x}, cr3 = {:#x}", rsp0, cr3); + + Ok(Self { + inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + stack_base, + stack_size: USER_TASK_PAGES * 0x1000, + }) + } +} + impl TaskContextImpl for TaskContext { const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; const USER_STACK_EXTRA_ALIGN: usize = 8; @@ -157,6 +212,7 @@ impl TaskContextImpl for TaskContext { extern "C" { fn __x86_64_task_enter_kernel(); fn __x86_64_task_enter_user(); + fn __x86_64_task_enter_from_fork(); fn __x86_64_enter_task(to: *mut Inner) -> !; fn __x86_64_switch_task(to: *mut Inner, from: *mut Inner); } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 71f3e4df..3664bdf1 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -2,9 +2,13 @@ use core::{arch::global_asm, mem::size_of_val}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; +use tock_registers::interfaces::Readable; use crate::{ - arch::{x86_64::apic, CpuAccess}, + arch::{ + x86_64::{apic, registers::CR3}, + CpuAccess, + }, task::{context::TaskFrame, thread::Thread, Cpu}, }; @@ -285,10 +289,12 @@ static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { let thread = Thread::current(); + let cr3 = CR3.get(); warnln!("{:?} in {} {:?}", kind, thread.id, thread.name); warnln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); warnln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); + warnln!("CR3 = {:#x}", cr3); match kind { ExceptionKind::PageFault => { diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 7675b6b4..7fa2e8a5 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -2,35 +2,51 @@ use core::arch::global_asm; -use abi::{arch::SavedFrame, process::SignalEntryData, syscall::SyscallFunction}; +use abi::{arch::SavedFrame, error::Error, process::SignalEntryData, syscall::SyscallFunction}; +use kernel_util::mem::address::PhysicalAddress; use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ arch::x86_64::registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, syscall::raw_syscall_handler, - task::{context::TaskFrame, thread::Thread}, + task::{ + context::{ForkFrame, TaskFrame}, + process::Process, + thread::Thread, + TaskContext, + }, }; /// Set of registers saved when taking a syscall instruction #[derive(Debug)] #[repr(C)] -struct SyscallFrame { - rax: u64, - args: [u64; 6], +pub(super) struct SyscallFrame { + pub rax: u64, + pub args: [u64; 6], - rcx: u64, - r11: u64, + pub rcx: u64, + pub r11: u64, - user_ip: u64, - user_sp: u64, - user_flags: u64, + pub user_ip: u64, + pub user_sp: u64, + pub user_flags: u64, - rbx: u64, - rbp: u64, - r12: u64, - r13: u64, - r14: u64, - r15: u64, + pub rbx: u64, + pub rbp: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, +} + +impl ForkFrame for SyscallFrame { + unsafe fn fork(&self, address_space: u64) -> Result { + TaskContext::from_syscall_frame(self, address_space) + } + + fn set_return_value(&mut self, value: u64) { + self.rax = value; + } } impl TaskFrame for SyscallFrame { @@ -117,6 +133,12 @@ fn syscall_inner(frame: &mut SyscallFrame) { return; } } + if frame.rax == usize::from(SyscallFunction::Fork) as u64 { + unsafe { + Process::raw_fork(frame); + return; + } + } let result = raw_syscall_handler(frame.rax, &frame.args); diff --git a/src/main.rs b/src/main.rs index e1ec8010..1870b640 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,7 +24,8 @@ exact_size_is_empty, inline_const, maybe_uninit_uninit_array, - const_maybe_uninit_uninit_array + const_maybe_uninit_uninit_array, + never_type )] #![allow( clippy::new_without_default, diff --git a/src/mem/process.rs b/src/mem/process.rs index bab29089..1fb65e27 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,15 +1,21 @@ //! Process address space structures and management functions +use core::ops::Range; + use abi::error::Error; use cfg_if::cfg_if; use kernel_util::{ - mem::{table::EntryLevelExt, PageProvider}, + mem::{ + pointer::{PhysicalRef, PhysicalRefMut}, + table::EntryLevelExt, + PageProvider, + }, sync::IrqSafeSpinlock, }; use vfs::FileRef; use vmalloc::{RangeData, VirtualMemoryAllocator}; -use crate::{arch::L3, mem::phys}; +use crate::{arch::L3, debug, mem::phys}; use super::{table::MapAttributes, PhysicalAddress}; @@ -252,6 +258,70 @@ impl Inner { Ok(()) } + + unsafe fn clear(&mut self) -> Result<(), Error> { + self.allocator.clear(|pfn_range, backing| { + let origin_pfn = pfn_range.start; + for pfn in pfn_range { + let offset = ((pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; + + let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; + let phys = unsafe { self.table.unmap_page(virt)? }; + + backing.release_page(offset, phys)?; + } + + Ok(()) + }) + } + + fn clone_range( + &mut self, + source: &Self, + pfn_range: Range, + backing: &VirtualRangeBacking, + ) -> Result<(), Error> { + self.allocator + .insert(pfn_range.start, pfn_range.len(), backing.clone()) + .unwrap(); + + match backing { + VirtualRangeBacking::Anonymous => { + let start = pfn_range.start * ProcessAddressSpaceImpl::PAGE_SIZE; + let end = pfn_range.end * ProcessAddressSpaceImpl::PAGE_SIZE; + debugln!("fork: clone_range({:#x?})", start..end); + // Clone the data + for i in pfn_range { + let address = i * ProcessAddressSpaceImpl::PAGE_SIZE; + let (src_page, attrs) = source.table.translate(address).unwrap(); + let dst_page = clone_page(address, src_page)?; + unsafe { + self.table + .map_page( + address, + dst_page, + MapAttributes::USER_READ + | MapAttributes::USER_WRITE + | MapAttributes::NON_GLOBAL, + ) + .unwrap(); + } + } + Ok(()) + } + VirtualRangeBacking::File(f) => { + todo!(); + } + } + } +} + +fn clone_page(src_virt: usize, src_phys: PhysicalAddress) -> Result { + let dst_page = phys::alloc_page()?; + let src_map = unsafe { PhysicalRef::<[u8; 4096]>::map(src_phys) }; + let mut dst_map = unsafe { PhysicalRefMut::<[u8; 4096]>::map(dst_page) }; + dst_map.copy_from_slice(src_map.as_ref()); + Ok(dst_page) } impl ProcessAddressSpace { @@ -267,6 +337,37 @@ impl ProcessAddressSpace { }) } + /// XXX + pub fn fork(&self) -> Result { + let src_inner = self.inner.lock(); + let new_table = ProcessAddressSpaceImpl::new()?; + let mut new_inner = Inner { + allocator: VirtualMemoryAllocator::new( + ProcessAddressSpaceImpl::LOWER_LIMIT_PFN, + ProcessAddressSpaceImpl::UPPER_LIMIT_PFN, + ), + table: new_table, + }; + + debugln!("fork address space!"); + + for (range, backing) in src_inner.allocator.regions() { + // If they are present in existing allocator, there should be no + // problem adding them to a new one + new_inner.clone_range(&src_inner, range, backing)?; + } + + for (range, _) in new_inner.allocator.regions() { + let start = range.start * ProcessAddressSpaceImpl::PAGE_SIZE; + let end = range.end * ProcessAddressSpaceImpl::PAGE_SIZE; + debugln!("forked region: {:#x?}", start..end); + } + + Ok(Self { + inner: IrqSafeSpinlock::new(new_inner), + }) + } + /// Allocates a region of virtual memory within the address space and maps the pages to the /// ones returned from `get_page` function pub fn allocate( @@ -350,4 +451,10 @@ impl ProcessAddressSpace { pub fn as_address_with_asid(&self) -> u64 { self.inner.lock().table.as_address_with_asid() } + + /// XXX + pub fn clear(&self) -> Result<(), Error> { + let mut inner = self.inner.lock(); + unsafe { inner.clear() } + } } diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 44e91ec3..ffc40279 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -8,7 +8,7 @@ use abi::{ path::Path, process::ProgramArgumentInner, }; -use alloc::{string::String, sync::Arc, vec::Vec}; +use alloc::{borrow::ToOwned, string::String, sync::Arc, vec::Vec}; use kernel_util::mem::pointer::PhysicalRefMut; use vfs::{FileRef, IoContext, Read, Seek}; @@ -86,8 +86,8 @@ unsafe impl<'a> Placer for BufferPlacer<'a> { fn setup_program_env( space: &ProcessAddressSpace, virt: usize, - args: &[&str], - env: &[&str], + args: &Vec, + envs: &Vec, ) -> Result { // TODO growing buffer let phys_page = space.map_single( @@ -98,19 +98,24 @@ fn setup_program_env( let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; let mut placer = BufferPlacer::new(virt, &mut buffer); - let in_kernel = ProgramArgumentInner { args, env }; + let args = args.iter().map(String::as_ref).collect::>(); + let envs = envs.iter().map(String::as_ref).collect::>(); + + let in_kernel = ProgramArgumentInner { + args: &args, + env: &envs, + }; let in_user = in_kernel.place_ref(&mut placer)?; Ok(in_user as *const _ as usize) } -fn setup_binary>( - name: S, - space: ProcessAddressSpace, - image: ProcessImage, - args: &[&str], - envs: &[&str], -) -> Result<(Arc, Arc), Error> { +fn setup_context( + space: &ProcessAddressSpace, + image: &ProcessImage, + args: &Vec, + envs: &Vec, +) -> Result { const USER_STACK_PAGES: usize = 16; let virt_stack_base = 0x3000000; @@ -148,18 +153,25 @@ fn setup_binary>( let tls_address = proc::elf::clone_tls(&space, &image)?; - let context = TaskContext::user( + TaskContext::user( image.entry, arg, space.as_address_with_asid(), user_sp, tls_address, - )?; + ) +} +fn setup_binary>( + name: S, + space: ProcessAddressSpace, + image: ProcessImage, + args: &Vec, + envs: &Vec, +) -> Result<(Arc, Arc), Error> { + let context = setup_context(&space, &image, args, envs)?; let (process, main) = Process::new_with_main(name, Arc::new(space), context, Some(image)); - infoln!("exec::setup_binary -> {:?}", process.id()); - Ok((process, main)) } @@ -175,13 +187,13 @@ fn load_binary( } } -/// Loads a program from given `path` -pub fn load>( +fn xxx_load_program>( + space: &ProcessAddressSpace, ioctx: &mut IoContext, path: P, - args: &[&str], - envs: &[&str], -) -> Result<(Arc, Arc), Error> { + args: Vec, + envs: Vec, +) -> Result<(ProcessImage, Vec, Vec), Error> { let mut head = [0; 256]; let path = path.as_ref(); let file = ioctx.open_exec(None, path)?; @@ -196,19 +208,52 @@ pub fn load>( && let Some((shebang, _)) = shebang.split_once(|&ch| ch == b'\n') { let shebang = core::str::from_utf8(shebang).map_err(|_| Error::InvalidFile)?; - let mut shebang_args = shebang.split(' ').collect::>(); + let mut shebang_args = shebang.split(' ').map(|s| s.to_owned()).collect::>(); if shebang_args.is_empty() || shebang_args.len() >= 8 { return Err(Error::UnrecognizedExecutable); } - shebang_args.extend_from_slice(args); + shebang_args.extend_from_slice(&args); - return load(ioctx, shebang_args[0], &shebang_args, envs); + return xxx_load_program(space, ioctx, shebang_args[0].clone(), shebang_args, envs); } file.seek(SeekFrom::Start(0))?; - let space = ProcessAddressSpace::new()?; let image = load_binary(head, file, &space)?; - setup_binary(path.display(), space, image, args, envs) + Ok((image, args, envs)) +} + +/// Loads a program from given `path` +pub fn load>( + ioctx: &mut IoContext, + path: P, + args: &[&str], + envs: &[&str], +) -> Result<(Arc, Arc), Error> { + let path = path.as_ref(); + let args = args.into_iter().map(|&s| s.to_owned()).collect(); + let envs = envs.into_iter().map(|&s| s.to_owned()).collect(); + + let space = ProcessAddressSpace::new()?; + let (image, args, envs) = xxx_load_program(&space, ioctx, path, args, envs)?; + setup_binary(path.display(), space, image, &args, &envs) +} + +/// XXX +pub fn load_into>( + process: &Process, + path: P, + args: Vec, + envs: Vec, +) -> Result<(TaskContext, ProcessImage), Error> { + let mut io = process.io.lock(); + // Have to make the Path owned, going to drop the address space from which it came + let path = path.as_ref().to_owned(); + let space = process.space(); + space.clear()?; + let (image, args, envs) = xxx_load_program(&space, io.ioctx(), &path, args, envs)?; + let context = setup_context(&space, &image, &args, &envs)?; + + Ok((context, image)) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index e35981b4..9b9ceb8b 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -9,10 +9,13 @@ use abi::{ TerminalSize, }, mem::MappingSource, - process::{ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions}, + process::{ + ExecveOptions, ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, + ThreadSpawnOptions, + }, syscall::SyscallFunction, }; -use alloc::{boxed::Box, sync::Arc}; +use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; @@ -25,7 +28,7 @@ use crate::{ proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, - thread::Thread, + thread::{CurrentThread, Thread}, }, }; @@ -57,6 +60,23 @@ fn run_with_io_at< f(at, io) } +fn sys_execve(thread: CurrentThread, options: &ExecveOptions) -> Result { + let process = thread.process(); + + // Clone the options into the kernel + let mut argv = Vec::new(); + let mut envp = Vec::new(); + + for &arg in options.arguments { + argv.push(arg.to_owned()); + } + for &env in options.environment { + envp.push(env.to_owned()); + } + + process.exec(options.program, argv, envp) +} + fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { let thread = Thread::current(); let process = thread.process(); @@ -448,6 +468,23 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) }) } + SyscallFunction::CloneFd => { + let source_fd = RawFd::from(args[0] as u32); + let target_fd = RawFd::from(args[1] as u32); + + run_with_io(process, |mut io| { + let file = io.files.file(source_fd)?.clone(); + + let fd = if target_fd != RawFd::NONE { + io.files.set_file(target_fd, file)?; + target_fd + } else { + io.files.place_file(file, true)? + }; + + Ok(fd.0 as usize) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; @@ -507,6 +544,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(pid as _) }) } + SyscallFunction::Exec => { + let options = arg_user_ref::(args[0] as usize)?; + sys_execve(thread, options)?; + } SyscallFunction::SpawnThread => { let options = arg_user_ref::(args[0] as usize)?; let id = process.spawn_thread(options)?; @@ -602,6 +643,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } + + _ => unreachable!(), } } @@ -619,5 +662,11 @@ pub fn raw_syscall_handler(func: u64, args: &[u64]) -> u64 { // func, // args // ); - syscall_handler(func, args).into_syscall_result() as u64 + let result = syscall_handler(func, args); + + if let &Err(error) = &result { + warnln!("{:?}: {:?}", func, error); + } + + result.into_syscall_result() as u64 } diff --git a/src/task/context.rs b/src/task/context.rs index 79268e1f..3dc3cdf4 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -3,7 +3,7 @@ use abi::{arch::SavedFrame, error::Error}; use alloc::boxed::Box; use cfg_if::cfg_if; -use kernel_util::thread::Termination; +use kernel_util::{mem::address::PhysicalAddress, thread::Termination}; use crate::task::thread::Thread; @@ -47,6 +47,15 @@ pub trait TaskFrame { fn user_ip(&self) -> usize; } +/// XXX +pub trait ForkFrame: Sized { + /// XXX + unsafe fn fork(&self, address_space: u64) -> Result; + + /// XXX + fn set_return_value(&mut self, value: u64); +} + /// Platform-specific task context implementation pub trait TaskContextImpl: Sized { /// Number of bytes to offset the signal stack pointer by diff --git a/src/task/process.rs b/src/task/process.rs index b55d81b9..b8524ac4 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -9,21 +9,25 @@ use core::{ }; use abi::{ - error::Error, + error::{Error, SyscallResult}, process::{ExitCode, Signal, ThreadSpawnOptions}, }; use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; use futures_util::Future; -use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock}; -use vfs::NodeRef; +use kernel_util::{ + runtime::QueueWaker, + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, +}; +use vfs::{IoContext, NodeRef}; use crate::{ mem::process::ProcessAddressSpace, proc::{self, io::ProcessIo}, - task::context::TaskContextImpl, + task::{context::TaskContextImpl, sched::CpuQueue}, }; use super::{ + context::ForkFrame, sync::UserspaceMutex, thread::{Thread, ThreadId}, TaskContext, @@ -52,7 +56,7 @@ pub struct ProcessId(u64); // | ??? | Data .....| /// Describes Thread-Local Storage of a process -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ProcessTlsInfo { /// Location of the TLS master copy within the process's memory pub master_copy_base: usize, @@ -61,7 +65,7 @@ pub struct ProcessTlsInfo { } /// Describes TLS layout for a program image -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ProcessTlsLayout { /// Data offset from the TLS base pub data_offset: usize, @@ -126,6 +130,7 @@ impl ProcessTlsLayout { } /// Describes information about a program's image in memory +#[derive(Clone)] pub struct ProcessImage { /// Entry point address pub entry: usize, @@ -142,6 +147,9 @@ struct ProcessInner { session_terminal: Option, threads: Vec>, mutexes: BTreeMap>, + + space: Arc, + image: Option, } /// Describes a process within the system @@ -149,9 +157,7 @@ pub struct Process { name: String, id: ProcessId, - space: Arc, - inner: IrqSafeSpinlock, - image: Option, + inner: IrqSafeRwLock, exit_waker: QueueWaker, /// Process I/O information @@ -176,16 +182,16 @@ impl Process { name, id, - image, - - space: space.clone(), - inner: IrqSafeSpinlock::new(ProcessInner { + inner: IrqSafeRwLock::new(ProcessInner { state: ProcessState::Running, session_id: id, group_id: id, session_terminal: None, threads: Vec::new(), mutexes: BTreeMap::new(), + + image, + space: space.clone(), }), exit_waker: QueueWaker::new(), @@ -194,7 +200,7 @@ impl Process { // Create "main" thread let thread = Thread::new_uthread(process.clone(), space, context); - process.inner.lock().threads.push(thread.clone()); + process.inner.write().threads.push(thread.clone()); PROCESSES.lock().insert(id, process.clone()); @@ -208,14 +214,15 @@ impl Process { self.id(), options ); + let mut inner = self.inner.write(); - let tls_address = if let Some(image) = self.image.as_ref() { - proc::elf::clone_tls(&self.space, image)? + let tls_address = if let Some(image) = inner.image.as_ref() { + proc::elf::clone_tls(&inner.space, image)? } else { 0 }; - let space = self.space.clone(); + let space = inner.space.clone(); let context = TaskContext::user( options.entry as _, options.argument as _, @@ -226,13 +233,81 @@ impl Process { let thread = Thread::new_uthread(self.clone(), space, context); let id = thread.id; - self.inner.lock().threads.push(thread.clone()); + inner.threads.push(thread.clone()); thread.enqueue(); Ok(id) } + unsafe fn fork_inner(&self, frame: &F) -> Result { + let src_inner = self.inner.read(); + let new_space = src_inner.space.fork()?; + let new_context = frame.fork(new_space.as_address_with_asid())?; + + let (new_process, new_main) = + Self::new_with_main(self.name(), Arc::new(new_space), new_context, None); + + { + let mut src_io = self.io.lock(); + + let new_ioctx = IoContext::inherit(src_io.ioctx()); + let mut new_io = new_process.io.lock(); + new_io.set_ioctx(new_ioctx); + + for (old_fd, old_file) in src_io.files.iter() { + let new_file = old_file.send()?; + new_io.files.set_file(*old_fd, new_file)?; + } + } + + new_process.inherit(self)?; + + infoln!("Process::fork -> {:?}", new_process.id()); + new_main.enqueue_to(CpuQueue::for_cpu(1)); + + Ok(new_process.id()) + } + + /// XXX + pub unsafe fn raw_fork(frame: &mut F) { + let src_thread = Thread::current(); + let src_process = src_thread.process(); + + let rax = src_process + .fork_inner(frame) + .map(|ProcessId(p)| p as u32) + .into_syscall_result(); + + frame.set_return_value(rax as _); + } + + /// XXX + pub fn exec(&self, program: &str, argv: Vec, envp: Vec) -> Result { + if self.inner.read().threads.len() != 1 { + todo!(); + } + + let (context, image) = proc::exec::load_into(self, program, argv, envp)?; + let mut inner = self.inner.write(); + let main = &inner.threads[0]; + + let old_context = unsafe { main.replace_context(context) }; + let new_context = main.context.as_ptr(); + inner.image = Some(image); + + drop(inner); + + // TODO old context is leaked + unsafe { (&*new_context).switch(&old_context) } + unreachable!() + } + + /// XXX + pub fn space(&self) -> Arc { + self.inner.read().space.clone() + } + /// Returns the [ProcessId] of this process pub fn id(&self) -> ProcessId { self.id @@ -240,12 +315,12 @@ impl Process { /// Returns the process group ID of the process pub fn group_id(&self) -> ProcessId { - self.inner.lock().group_id + self.inner.read().group_id } /// Returns the process session ID of the process pub fn session_id(&self) -> ProcessId { - self.inner.lock().session_id + self.inner.read().session_id } /// Returns the process name @@ -255,35 +330,35 @@ impl Process { /// Changes the process's group ID pub fn set_group_id(&self, id: ProcessId) { - self.inner.lock().group_id = id; + self.inner.write().group_id = id; } /// Changes the process's session ID pub fn set_session_id(&self, id: ProcessId) { - self.inner.lock().session_id = id; + self.inner.write().session_id = id; } // Resources /// Returns the current session terminal of the process, if set pub fn session_terminal(&self) -> Option { - self.inner.lock().session_terminal.clone() + self.inner.read().session_terminal.clone() } /// Changes the current session terminal of the process pub fn set_session_terminal(&self, node: NodeRef) { - self.inner.lock().session_terminal.replace(node); + self.inner.write().session_terminal.replace(node); } /// Resets the current session terminal of the process pub fn clear_session_terminal(&self) -> Option { - self.inner.lock().session_terminal.take() + self.inner.write().session_terminal.take() } /// Inherits the process information from the `parent` pub fn inherit(&self, parent: &Process) -> Result<(), Error> { - let mut our_inner = self.inner.lock(); - let their_inner = parent.inner.lock(); + let mut our_inner = self.inner.write(); + let their_inner = parent.inner.read(); our_inner.session_id = their_inner.session_id; our_inner.group_id = their_inner.group_id; @@ -296,7 +371,7 @@ impl Process { /// Returns the [ExitCode] of the process, if it has exited pub fn get_exit_status(&self) -> Option { - match self.inner.lock().state { + match self.inner.read().state { ProcessState::Running => None, ProcessState::Terminated(x) => Some(x), } @@ -331,7 +406,7 @@ impl Process { /// Handles exit of a single child thread pub fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { debugln!("Thread {} of process {}: {:?}", thread, self.id, code); - let mut inner = self.inner.lock(); + let mut inner = self.inner.write(); // TODO make this cleaner let old_len = inner.threads.len(); @@ -356,7 +431,7 @@ impl Process { /// Raises a signal for the specified process pub fn raise_signal(self: &Arc, signal: Signal) { - let thread = self.inner.lock().threads[0].clone(); + let thread = self.inner.read().threads[0].clone(); thread.raise_signal(signal); } @@ -364,7 +439,7 @@ impl Process { pub fn signal_group(group_id: ProcessId, signal: Signal) { let processes = PROCESSES.lock(); for (_, proc) in processes.iter() { - let inner = proc.inner.lock(); + let inner = proc.inner.read(); if !matches!(inner.state, ProcessState::Terminated(_)) && inner.group_id == group_id { debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); drop(inner); @@ -376,7 +451,7 @@ impl Process { /// Returns a [UserspaceMutex] associated with the `address`. If one does not exist, will /// create it. pub fn get_or_insert_mutex(&self, address: usize) -> Arc { - let mut inner = self.inner.lock(); + let mut inner = self.inner.write(); inner .mutexes .entry(address) @@ -393,7 +468,7 @@ impl Process { /// Terminates all children of the process, `except` one pub async fn terminate_others(&self, except: ThreadId) { - let mut inner = self.inner.lock(); + let mut inner = self.inner.write(); for thread in inner.threads.iter() { if thread.id == except { diff --git a/src/task/sched.rs b/src/task/sched.rs index c19eb529..0f96cfd5 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -1,5 +1,7 @@ //! Per-CPU queue implementation +use core::cell::Cell; + use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; @@ -20,7 +22,7 @@ use super::{ pub struct CpuQueue { queue: SegQueue, index: usize, - idle: TaskContext, + idle: Cell, } impl CpuQueue { @@ -32,7 +34,7 @@ impl CpuQueue { Self { queue: SegQueue::new(), index, - idle, + idle: Cell::new(idle), } } @@ -50,7 +52,7 @@ impl CpuQueue { pub unsafe fn enter(&self) -> ! { let _guard = IrqGuard::acquire(); - self.idle.enter() + (&*self.idle.as_ptr()).enter() } /// "Pushes" a thread to the queue for execution @@ -151,9 +153,12 @@ impl CpuQueue { cpu.set_current_thread_id(next_id); + let current_ctx = current_ctx.as_ptr(); + let next_ctx = next_ctx.as_ptr(); + if !core::ptr::eq(current_ctx, next_ctx) { // Perform the switch - next_ctx.switch(current_ctx); + (&*next_ctx).switch(&*current_ctx); true } else { diff --git a/src/task/thread.rs b/src/task/thread.rs index 4e0f5e81..37557644 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -3,6 +3,7 @@ //! Thread data structures and management use core::{ + cell::Cell, fmt, mem::size_of, ops::Deref, @@ -91,7 +92,7 @@ pub struct Thread { pub id: ThreadId, pub name: Option, pub sched: IrqSafeSpinlock, - pub context: TaskContext, + pub context: Cell, process: Option>, space: Option>, @@ -103,6 +104,8 @@ pub struct Thread { pub affinity: ThreadAffinity, } +unsafe impl Sync for Thread {} + impl GlobalThreadList { pub const fn new() -> Self { Self { @@ -161,7 +164,7 @@ impl Thread { in_queue: false, queue: None, }), - context, + context: Cell::new(context), process, space, @@ -203,6 +206,10 @@ impl Thread { ) } + pub unsafe fn replace_context(&self, context: TaskContext) -> TaskContext { + self.context.replace(context) + } + // Get/Set /// Updates the thread affinity to run on a specific set of CPUs @@ -432,7 +439,8 @@ impl CurrentThread { let inner = self.inner.lock(); let Some(entry) = inner.signal_entry.as_ref() else { - todo!(); + drop(inner); + self.exit_process(ExitCode::BySignal(signal)); }; // TODO check if really in a syscall, lol From cb5f4c72574eebd22c66d075aee2390281a41343 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 15 Jan 2024 18:15:59 +0200 Subject: [PATCH 146/211] fs: fix PTY allocation, better RingBuffer management --- driver/fs/memfs/src/bvec.rs | 12 ++++++- driver/fs/memfs/src/file.rs | 4 +++ lib/kernel-util/src/lib.rs | 3 +- lib/kernel-util/src/util/ring.rs | 58 +++++++++++++++++++++++--------- lib/vfs/src/file/mod.rs | 10 +++--- lib/vfs/src/ioctx.rs | 2 +- lib/vfs/src/node/impls.rs | 4 +++ lib/vfs/src/node/ops.rs | 6 +++- lib/vfs/src/node/traits.rs | 6 ++++ lib/vfs/src/pty.rs | 25 +++++++------- lib/vmalloc/src/lib.rs | 4 +-- 11 files changed, 94 insertions(+), 40 deletions(-) diff --git a/driver/fs/memfs/src/bvec.rs b/driver/fs/memfs/src/bvec.rs index 92c05adb..f6f91ad4 100644 --- a/driver/fs/memfs/src/bvec.rs +++ b/driver/fs/memfs/src/bvec.rs @@ -146,7 +146,7 @@ impl<'a, A: BlockAllocator> BVec<'a, A> { } fn ensure_write_capacity(&mut self, pos: usize, need_to_write: usize) -> Result<(), Error> { - let current_capacity = self.capacity * block::SIZE; + let current_capacity = self.capacity; let need_capacity = (core::cmp::max(pos + need_to_write, self.size) + block::SIZE - 1) / block::SIZE; @@ -218,6 +218,16 @@ impl<'a, A: BlockAllocator> BVec<'a, A> { Ok(doff) } + /// Resize the block vector to requested size + pub fn truncate(&mut self, new_size: u64) -> Result<(), Error> { + let new_size: usize = new_size.try_into().unwrap(); + let requested_capacity = (new_size + block::SIZE - 1) / block::SIZE; + self.resize(requested_capacity)?; + // TODO fill with zeros if resizing larger? + self.size = new_size; + Ok(()) + } + unsafe fn index_unchecked(&self, mut index: usize) -> &BlockData<'a, A> { if index < L0_BLOCKS { return &self.l0[index]; diff --git a/driver/fs/memfs/src/file.rs b/driver/fs/memfs/src/file.rs index 78e95920..70146b4a 100644 --- a/driver/fs/memfs/src/file.rs +++ b/driver/fs/memfs/src/file.rs @@ -65,6 +65,10 @@ impl RegularImpl for FileNode { self.data.lock().write(pos, buf) } + fn truncate(&self, node: &NodeRef, new_size: u64) -> Result<(), Error> { + self.data.lock().truncate(new_size) + } + fn close(&self, _node: &NodeRef, _instance: Option<&InstanceData>) -> Result<(), Error> { Ok(()) } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index c3627a67..7197417c 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -10,7 +10,8 @@ let_chains, allocator_api, maybe_uninit_uninit_array, - const_maybe_uninit_uninit_array + const_maybe_uninit_uninit_array, + new_uninit )] use device_api::interrupt::MessageInterruptController; diff --git a/lib/kernel-util/src/util/ring.rs b/lib/kernel-util/src/util/ring.rs index dd371e53..1fb8c70b 100644 --- a/lib/kernel-util/src/util/ring.rs +++ b/lib/kernel-util/src/util/ring.rs @@ -2,28 +2,47 @@ use core::mem::MaybeUninit; +use alloc::boxed::Box; +use yggdrasil_abi::error::Error; + /// Ring buffer base -pub struct RingBuffer { +pub struct RingBuffer { rd: usize, wr: usize, - data: [MaybeUninit; N], + capacity: usize, + data: Option]>>, } -impl RingBuffer { - /// Constructs an empty [RingBuffer] - pub const fn new() -> Self { +impl RingBuffer { + /// Constructs an empty [RingBuffer]. + /// + /// NOTE does not allocate until the first write operation. Any read accesses + /// without checking readable count will cause a panic. + pub const fn with_capacity(capacity: usize) -> Self { Self { rd: 0, wr: 0, - data: MaybeUninit::uninit_array(), + capacity, + data: None, } } + /// Constructs an empty [RingBuffer], trying to allocate its data + pub fn try_with_capacity(capacity: usize) -> Result { + let data = Box::try_new_uninit_slice(capacity).map_err(|_| Error::OutOfMemory)?; + Ok(Self { + rd: 0, + wr: 0, + capacity, + data: Some(data), + }) + } + const fn readable_count_at(&self, at: usize) -> usize { if at <= self.wr { self.wr - at } else { - self.wr + (N - at) + self.wr + (self.capacity - at) } } @@ -37,14 +56,14 @@ impl RingBuffer { if at <= self.wr { (self.wr - at) > 0 } else { - (self.wr + (N - at)) > 0 + (self.wr + (self.capacity - at)) > 0 } } /// Returns `true` if the ring is not full #[inline] pub fn is_writable(&self) -> bool { - (self.wr + 1) % N != self.rd + (self.wr + 1) % self.capacity != self.rd } /// Returns `true` if the ring contains given element @@ -52,10 +71,11 @@ impl RingBuffer { where T: PartialEq, { + let data = self.data.as_ref().unwrap(); let count = self.readable_count_at(self.rd); for i in 0..count { - if unsafe { self.data[(self.rd + i) % N].assume_init_ref() == t } { + if unsafe { data[(self.rd + i) % self.capacity].assume_init_ref() == t } { return true; } } @@ -73,8 +93,9 @@ impl RingBuffer { where T: Copy, { - let res = self.data[self.rd].assume_init(); - self.rd = (self.rd + 1) % N; + let data = self.data.as_ref().unwrap(); + let res = data[self.rd].assume_init(); + self.rd = (self.rd + 1) % self.capacity; res } @@ -87,10 +108,11 @@ impl RingBuffer { where T: Copy, { - let mut pos = (self.rd + pos) % N; + let data = self.data.as_ref().unwrap(); + let mut pos = (self.rd + pos) % self.capacity; let mut off = 0; while off < buffer.len() && self.is_readable_at(pos) { - buffer[off] = self.data[pos].assume_init(); + buffer[off] = data[pos].assume_init(); pos += 1; off += 1; } @@ -100,7 +122,11 @@ impl RingBuffer { /// Writes a single entry to the buffer #[inline] pub fn write(&mut self, ch: T) { - self.data[self.wr].write(ch); - self.wr = (self.wr + 1) % N; + let data = self + .data + .get_or_insert_with(|| Box::new_uninit_slice(self.capacity)); + + data[self.wr].write(ch); + self.wr = (self.wr + 1) % self.capacity; } } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 5634dd39..6d670e85 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -107,12 +107,12 @@ impl File { pub fn new_pseudo_terminal( config: TerminalOptions, size: TerminalSize, - ) -> (Arc, Arc) { - let (master, slave) = PseudoTerminal::new(config, size); - ( + ) -> Result<(Arc, Arc), Error> { + let (master, slave) = PseudoTerminal::new(config, size)?; + Ok(( Arc::new(Self::PtyMaster(master)), Arc::new(Self::PtySlave(slave)), - ) + )) } pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { @@ -423,7 +423,7 @@ impl FileSet { self.map.retain(predicate); } - /// XXX + /// Returns an iterator over the file set pub fn iter(&self) -> impl Iterator { self.map.iter() } diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index b6458655..c29a4487 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -215,7 +215,7 @@ impl IoContext { path: P, ) -> Result { let node = self.find(at, path, true, true)?; - let access = self.check_access(Action::Execute, &node)?; + let access = self.check_access(Action::Read, &node)?; node.open(OpenOptions::READ, access) } diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index 0f752b61..91afa247 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -163,6 +163,10 @@ where ) -> Result { Err(Error::ReadOnly) } + + fn truncate(&self, _node: &NodeRef, _new_size: u64) -> Result<(), Error> { + Err(Error::ReadOnly) + } } // Read-write FnNode diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs index 11c1b3fa..a40308a3 100644 --- a/lib/vfs/src/node/ops.rs +++ b/lib/vfs/src/node/ops.rs @@ -29,12 +29,16 @@ impl Node { match &self.data { NodeImpl::Regular(imp) => { let (pos, instance) = imp.open(self, opts)?; + if opts.contains(OpenOptions::TRUNCATE) { + imp.truncate(self, 0)?; + } Ok(File::regular(self.clone(), pos, instance, opts)) } NodeImpl::Block(dev) => File::block(dev.clone(), self.clone(), opts), NodeImpl::Char(dev) => File::char(dev.clone(), self.clone(), opts), // TODO: maybe merge open_directory and open? - NodeImpl::Directory(_) | NodeImpl::Symlink(_) => todo!(), + NodeImpl::Directory(_) => Err(Error::IsADirectory), + NodeImpl::Symlink(_) => todo!(), } } diff --git a/lib/vfs/src/node/traits.rs b/lib/vfs/src/node/traits.rs index 823e3189..aea24a78 100644 --- a/lib/vfs/src/node/traits.rs +++ b/lib/vfs/src/node/traits.rs @@ -56,6 +56,7 @@ pub trait RegularImpl: CommonImpl { ) -> Result { Err(Error::NotImplemented) } + /// Writes data to the file from given buffer fn write( &self, @@ -66,6 +67,11 @@ pub trait RegularImpl: CommonImpl { ) -> Result { Err(Error::NotImplemented) } + + /// Resizes the file to requested size + fn truncate(&self, node: &NodeRef, new_size: u64) -> Result<(), Error> { + Err(Error::NotImplemented) + } } /// Directory implementation diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index fa172abe..f2d763de 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -4,7 +4,6 @@ // TODO handle werase key use core::{ pin::Pin, - sync::atomic::{AtomicBool, Ordering}, task::{Context, Poll}, }; @@ -26,10 +25,10 @@ use yggdrasil_abi::{ process::Signal, }; -const CAPACITY: usize = 1024; +const CAPACITY: usize = 8192; struct PtyHalf { - ring: IrqSafeSpinlock>, + ring: IrqSafeSpinlock>, notify: QueueWaker, } @@ -53,7 +52,7 @@ pub struct PseudoTerminalMaster { } fn input_discipline( - mut lock: IrqSafeSpinlockGuard>, + mut lock: IrqSafeSpinlockGuard>, config: &TerminalOptions, buffer: &mut [u8], ) -> Result { @@ -89,17 +88,17 @@ fn input_discipline( } impl PtyHalf { - pub fn new() -> Self { - Self { - ring: IrqSafeSpinlock::new(RingBuffer::new()), + pub fn new() -> Result { + Ok(Self { + ring: IrqSafeSpinlock::new(RingBuffer::try_with_capacity(CAPACITY)?), notify: QueueWaker::new(), - } + }) } fn try_begin_read<'a>( &'a self, config: &TerminalOptions, - ) -> Option>> { + ) -> Option>> { let lock = self.ring.lock(); let ready = if config.is_canonical() { @@ -214,19 +213,19 @@ impl PseudoTerminal { pub fn new( config: TerminalOptions, size: TerminalSize, - ) -> (PseudoTerminalMaster, PseudoTerminalSlave) { + ) -> Result<(PseudoTerminalMaster, PseudoTerminalSlave), Error> { let pty = Arc::new(Self { config: IrqSafeRwLock::new(config), signal_pgroup: IrqSafeRwLock::new(None), - master_to_slave: PtyHalf::new(), - slave_to_master: PtyHalf::new(), + master_to_slave: PtyHalf::new()?, + slave_to_master: PtyHalf::new()?, size: IrqSafeRwLock::new(size), }); let master = PseudoTerminalMaster { pty: pty.clone() }; let slave = PseudoTerminalSlave { pty }; - (master, slave) + Ok((master, slave)) } fn putc_from_slave(&self, byte: u8) { diff --git a/lib/vmalloc/src/lib.rs b/lib/vmalloc/src/lib.rs index d4a408b8..716265ea 100644 --- a/lib/vmalloc/src/lib.rs +++ b/lib/vmalloc/src/lib.rs @@ -42,7 +42,7 @@ impl VirtualMemoryAllocator { } } - /// XXX + /// Returns an iterator over the regions within the allocator pub fn regions(&self) -> impl Iterator, &D)> { self.map.iter().map(|(range, data)| { let range = range.start() as usize..range.end() as usize + 1; @@ -101,7 +101,7 @@ impl VirtualMemoryAllocator { }) } - /// XXX + /// Removes all allocations, invoking a function on each of them pub fn clear, D) -> Result<(), Error>>( &mut self, mut release: F, From 129019de6c37f8273bedd048d2a73988b0bf5686 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 15 Jan 2024 18:16:32 +0200 Subject: [PATCH 147/211] block/nvme: fix temporary warnings --- driver/block/nvme/src/lib.rs | 2 +- driver/block/nvme/src/queue.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index 3c133c9b..9a0c6cb0 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(strict_provenance, const_trait_impl, let_chains, if_let_guard)] +#![feature(strict_provenance, const_trait_impl, let_chains, if_let_guard, effects)] #![allow(missing_docs)] #![no_std] diff --git a/driver/block/nvme/src/queue.rs b/driver/block/nvme/src/queue.rs index 050df88a..91124583 100644 --- a/driver/block/nvme/src/queue.rs +++ b/driver/block/nvme/src/queue.rs @@ -289,7 +289,7 @@ impl QueuePair { let mut inner = self.inner.lock(); match inner.completed.remove(&command_id) { - Some(result) if let Some(error) = result.error() => todo!(), + Some(result) if let Some(_error) = result.error() => todo!(), Some(_) => Poll::Ready(Ok(())), None => Poll::Pending, } From ae6195094cf4e45d149be7657b5aa3eaa992396f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 15 Jan 2024 18:17:16 +0200 Subject: [PATCH 148/211] proc: better handling of `at` in syscalls, UpdateMetadata call --- driver/fs/memfs/src/file.rs | 2 +- src/arch/x86_64/context.rs | 11 +++------ src/arch/x86_64/exception.rs | 15 +++++++++--- src/arch/x86_64/mem/mod.rs | 9 ++++---- src/arch/x86_64/mem/process.rs | 3 ++- src/arch/x86_64/syscall.rs | 1 - src/debug.rs | 4 ++-- src/device/input.rs | 9 +++----- src/device/tty.rs | 4 ++-- src/mem/process.rs | 24 ++++++------------- src/proc/elf.rs | 16 +++++++++++-- src/proc/exec.rs | 21 ++++++++++++----- src/syscall/mod.rs | 42 +++++++++++++++++++++------------- src/task/context.rs | 15 +++++++----- src/task/process.rs | 24 +++++++++++++------ src/task/sched.rs | 4 ++-- src/task/thread.rs | 26 +++++++++++++++++++-- 17 files changed, 144 insertions(+), 86 deletions(-) diff --git a/driver/fs/memfs/src/file.rs b/driver/fs/memfs/src/file.rs index 70146b4a..800e71c5 100644 --- a/driver/fs/memfs/src/file.rs +++ b/driver/fs/memfs/src/file.rs @@ -65,7 +65,7 @@ impl RegularImpl for FileNode { self.data.lock().write(pos, buf) } - fn truncate(&self, node: &NodeRef, new_size: u64) -> Result<(), Error> { + fn truncate(&self, _node: &NodeRef, new_size: u64) -> Result<(), Error> { self.data.lock().truncate(new_size) } diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 3f96bb09..d3d0965b 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -2,14 +2,9 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; -use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; -use tock_registers::interfaces::Readable; +use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw}; -use crate::{ - arch::x86_64::{mem::KERNEL_TABLES, registers::CR3}, - mem::phys, - task::context::TaskContextImpl, -}; +use crate::{arch::x86_64::mem::KERNEL_TABLES, mem::phys, task::context::TaskContextImpl}; use super::syscall::SyscallFrame; @@ -85,7 +80,7 @@ impl StackBuilder { unsafe impl Sync for TaskContext {} impl TaskContext { - /// XXX + /// Constructs a new task context from a "forked" syscall frame pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result { const USER_TASK_PAGES: usize = 8; diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 3664bdf1..a6bb3a8b 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -1,5 +1,5 @@ //! x86-64 exception and interrupt handling -use core::{arch::global_asm, mem::size_of_val}; +use core::{arch::global_asm, mem::size_of, ptr::addr_of}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; use tock_registers::interfaces::Readable; @@ -307,9 +307,18 @@ fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { thread.raise_signal(Signal::MemoryAccessViolation); } + ExceptionKind::GeneralProtectionFault => { + thread.raise_signal(Signal::MemoryAccessViolation); + } + ExceptionKind::FpuException => { + todo!() + } ExceptionKind::InvalidOpcode => { thread.raise_signal(Signal::Aborted); } + ExceptionKind::Breakpoint => { + todo!() + } _ => todo!("No handler for exception: {:?}", kind), } } @@ -377,8 +386,8 @@ pub unsafe fn init_exceptions(cpu_index: usize) { } let idtr = Pointer { - limit: size_of_val(&IDT) as u16 - 1, - offset: &IDT as *const _ as usize, + limit: (IDT.len() * size_of::()) as u16 - 1, + offset: addr_of!(IDT) as usize, }; core::arch::asm!("wbinvd; lidt ({0})", in(reg) &idtr, options(att_syntax)); diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index bd4d055c..a2886899 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -2,6 +2,7 @@ use core::{ alloc::Layout, ops::{Deref, DerefMut}, + ptr::addr_of, }; use abi::error::Error; @@ -324,10 +325,10 @@ fn clone_kernel_tables(dst: &mut PageTable) { /// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ... pub(super) unsafe fn init_fixed_tables() { // TODO this could be built in compile-time too? - let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; - let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; - let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; - let ram_mapping_l1_phys = &RAM_MAPPING_L1 as *const _ as usize - KERNEL_VIRT_OFFSET; + let early_mapping_l3_phys = addr_of!(EARLY_MAPPING_L3) as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = addr_of!(DEVICE_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = addr_of!(HEAP_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + let ram_mapping_l1_phys = addr_of!(RAM_MAPPING_L1) as usize - KERNEL_VIRT_OFFSET; for i in 0..DEVICE_MAPPING_L3_COUNT { let device_mapping_l3_phys = PhysicalAddress::from_raw( diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index aba47e2e..9352c4b1 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -29,7 +29,8 @@ pub struct ProcessAddressSpaceImpl { impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { const PAGE_SIZE: usize = L3::SIZE; - const LOWER_LIMIT_PFN: usize = 8; + // Start with 8GiB + const LOWER_LIMIT_PFN: usize = (8 << 30) / Self::PAGE_SIZE; // 16GiB VM limit const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE; diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 7fa2e8a5..6f5b469a 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -3,7 +3,6 @@ use core::arch::global_asm; use abi::{arch::SavedFrame, error::Error, process::SignalEntryData, syscall::SyscallFunction}; -use kernel_util::mem::address::PhysicalAddress; use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ diff --git a/src/debug.rs b/src/debug.rs index c717a66a..94a5fd79 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -15,7 +15,7 @@ const RING_LOGGER_CAPACITY: usize = 65536; struct SimpleLogger; struct RingLoggerInner { - data: RingBuffer, + data: RingBuffer, } /// Logger sink which collects output to an internal ring buffer @@ -168,7 +168,7 @@ impl RingLoggerSink { const fn new() -> Self { Self { inner: IrqSafeSpinlock::new(RingLoggerInner { - data: RingBuffer::new(), + data: RingBuffer::with_capacity(RING_LOGGER_CAPACITY), }), } } diff --git a/src/device/input.rs b/src/device/input.rs index dc6cc515..20539110 100644 --- a/src/device/input.rs +++ b/src/device/input.rs @@ -30,7 +30,7 @@ use vfs::{CharDevice, FileReadiness}; // } pub struct MpscChannel { - queue: IrqSafeSpinlock>, + queue: IrqSafeSpinlock>, notify: AtomicWaker, } @@ -53,7 +53,7 @@ pub struct KeyboardDevice; impl MpscChannel { pub const fn new() -> Self { Self { - queue: IrqSafeSpinlock::new(RingBuffer::new()), + queue: IrqSafeSpinlock::new(RingBuffer::with_capacity(128)), notify: AtomicWaker::new(), } } @@ -63,10 +63,7 @@ impl MpscChannel { self.notify.wake(); } - pub fn poll_read( - &self, - cx: &mut Context<'_>, - ) -> Poll>> { + pub fn poll_read(&self, cx: &mut Context<'_>) -> Poll>> { let lock = self.queue.lock(); if lock.is_readable() { return Poll::Ready(lock); diff --git a/src/device/tty.rs b/src/device/tty.rs index cf55e8d8..3a54f8bc 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -17,7 +17,7 @@ use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBu use crate::task::process::{Process, ProcessId}; struct TerminalRing { - buffer: IrqSafeSpinlock>, + buffer: IrqSafeSpinlock>, eof: AtomicBool, notify: QueueWaker, } @@ -27,7 +27,7 @@ impl TerminalRing { const fn new() -> Self { Self { - buffer: IrqSafeSpinlock::new(RingBuffer::new()), + buffer: IrqSafeSpinlock::new(RingBuffer::with_capacity(TerminalRing::CAPACITY)), eof: AtomicBool::new(false), notify: QueueWaker::new(), } diff --git a/src/mem/process.rs b/src/mem/process.rs index 1fb65e27..061bcebe 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -15,7 +15,7 @@ use kernel_util::{ use vfs::FileRef; use vmalloc::{RangeData, VirtualMemoryAllocator}; -use crate::{arch::L3, debug, mem::phys}; +use crate::{arch::L3, mem::phys}; use super::{table::MapAttributes, PhysicalAddress}; @@ -294,29 +294,21 @@ impl Inner { for i in pfn_range { let address = i * ProcessAddressSpaceImpl::PAGE_SIZE; let (src_page, attrs) = source.table.translate(address).unwrap(); - let dst_page = clone_page(address, src_page)?; + let dst_page = clone_page(src_page)?; unsafe { - self.table - .map_page( - address, - dst_page, - MapAttributes::USER_READ - | MapAttributes::USER_WRITE - | MapAttributes::NON_GLOBAL, - ) - .unwrap(); + self.table.map_page(address, dst_page, attrs).unwrap(); } } Ok(()) } - VirtualRangeBacking::File(f) => { + VirtualRangeBacking::File(_f) => { todo!(); } } } } -fn clone_page(src_virt: usize, src_phys: PhysicalAddress) -> Result { +fn clone_page(src_phys: PhysicalAddress) -> Result { let dst_page = phys::alloc_page()?; let src_map = unsafe { PhysicalRef::<[u8; 4096]>::map(src_phys) }; let mut dst_map = unsafe { PhysicalRefMut::<[u8; 4096]>::map(dst_page) }; @@ -337,7 +329,7 @@ impl ProcessAddressSpace { }) } - /// XXX + /// Performs a "fork" operation of the address space, cloning all the mappings into a new one pub fn fork(&self) -> Result { let src_inner = self.inner.lock(); let new_table = ProcessAddressSpaceImpl::new()?; @@ -417,8 +409,6 @@ impl ProcessAddressSpace { attributes: MapAttributes, ) -> Result { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - // XXX - // assert!(physical.is_aligned_for::()); self.inner.lock().map_single(address, backing, attributes) } @@ -452,7 +442,7 @@ impl ProcessAddressSpace { self.inner.lock().table.as_address_with_asid() } - /// XXX + /// Removes all allocations and their associated mappings from the address space pub fn clear(&self) -> Result<(), Error> { let mut inner = self.inner.lock(); unsafe { inner.clear() } diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 8d5c0fb1..902c5b59 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -7,11 +7,15 @@ use elf::{ segment::ProgramHeader, ElfStream, ParseError, }; -use kernel_util::mem::pointer::{PhysicalRef, PhysicalRefMut}; +use kernel_util::mem::{ + pointer::{PhysicalRef, PhysicalRefMut}, + table::EntryLevelExt, +}; use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ + arch::L3, mem::{ process::{ProcessAddressSpace, VirtualRangeBacking}, table::MapAttributes, @@ -273,12 +277,16 @@ pub fn load_elf_from_file( ) -> Result { let file = FileReader { file: &file }; let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; - + let mut image_end = 0; let mut tls = None; for phdr in elf.segments() { match phdr.p_type { PT_LOAD => { + if phdr.p_vaddr + phdr.p_memsz > image_end { + image_end = phdr.p_vaddr + phdr.p_memsz; + } + load_segment(space, phdr, &file)?; } PT_TLS => { @@ -289,6 +297,10 @@ pub fn load_elf_from_file( } } + let image_end = (image_end as usize).page_align_up::(); + + debugln!("Loaded image end: {:#x}", image_end); + Ok(ProcessImage { entry: elf.ehdr.e_entry as usize, tls, diff --git a/src/proc/exec.rs b/src/proc/exec.rs index ffc40279..f8555f22 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -83,6 +83,8 @@ unsafe impl<'a> Placer for BufferPlacer<'a> { } } +// args, envs are passed as Vec to ensure kernel ownership +#[allow(clippy::ptr_arg)] fn setup_program_env( space: &ProcessAddressSpace, virt: usize, @@ -129,7 +131,7 @@ fn setup_context( MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; - let arg = setup_program_env(&space, virt_args_base, args, envs)?; + let arg = setup_program_env(space, virt_args_base, args, envs)?; debugln!( "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", @@ -147,11 +149,18 @@ fn setup_context( #[allow(clippy::reversed_empty_ranges)] for i in 0..TaskContext::USER_STACK_EXTRA_ALIGN / 8 { unsafe { - ptr.add(i).write_foreign_volatile(&space, 0xDEADC0DE); + ptr.add(i).write_foreign_volatile(space, 0xDEADC0DE); } } - let tls_address = proc::elf::clone_tls(&space, &image)?; + let tls_address = proc::elf::clone_tls(space, image)?; + + debugln!( + "Create User TaskContext: entry = {:#x}, rsp3 = {:#x}, stack: {:#x?}", + image.entry, + user_sp, + virt_stack_base..virt_stack_base + USER_STACK_PAGES * 0x1000 + ); TaskContext::user( image.entry, @@ -219,7 +228,7 @@ fn xxx_load_program>( file.seek(SeekFrom::Start(0))?; - let image = load_binary(head, file, &space)?; + let image = load_binary(head, file, space)?; Ok((image, args, envs)) } @@ -232,8 +241,8 @@ pub fn load>( envs: &[&str], ) -> Result<(Arc, Arc), Error> { let path = path.as_ref(); - let args = args.into_iter().map(|&s| s.to_owned()).collect(); - let envs = envs.into_iter().map(|&s| s.to_owned()).collect(); + let args = args.iter().map(|&s| s.to_owned()).collect(); + let envs = envs.iter().map(|&s| s.to_owned()).collect(); let space = ProcessAddressSpace::new()?; let (image, args, envs) = xxx_load_program(&space, ioctx, path, args, envs)?; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 9b9ceb8b..0f22dbcc 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -4,9 +4,9 @@ use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, io::{ - DeviceRequest, DirectoryEntry, FileAttr, FileMode, MessageDestination, OpenOptions, - PollControl, RawFd, ReceivedMessageMetadata, SeekFrom, SentMessage, TerminalOptions, - TerminalSize, + DeviceRequest, DirectoryEntry, FileAttr, FileMetadataUpdate, FileMode, MessageDestination, + OpenOptions, PollControl, RawFd, ReceivedMessageMetadata, SeekFrom, SentMessage, + TerminalOptions, TerminalSize, }, mem::MappingSource, process::{ @@ -40,22 +40,21 @@ fn run_with_io) -> T>(proc: &Proces f(io) } -fn run_with_io_at< - T, - F: FnOnce(Option, IrqSafeSpinlockGuard) -> Result, ->( +fn run_with_io_at) -> Result>( proc: &Process, at: Option, f: F, ) -> Result { - let io = proc.io.lock(); + let mut io = proc.io.lock(); let at = at .map(|fd| { io.files .file(fd) .and_then(|f| f.node().ok_or(Error::InvalidFile).cloned()) }) - .transpose()?; + .transpose()? + // at = None + .unwrap_or_else(|| io.ioctx().cwd().clone()); f(at, io) } @@ -167,7 +166,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let mode = FileMode::from(args[4] as u32); run_with_io_at(process, at, |at, mut io| { - let file = io.ioctx().open(at, path, opts, mode)?; + let file = io.ioctx().open(Some(at), path, opts, mode)?; // TODO NO_CTTY? if process.session_terminal().is_none() @@ -187,7 +186,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; run_with_io_at(process, at, |at, mut io| { - let node = io.ioctx().find(at, path, true, true)?; + let node = io.ioctx().find(Some(at), path, true, true)?; let access = io.ioctx().check_access(vfs::Action::Read, &node)?; let file = node.open_directory(access)?; let fd = io.files.place_file(file, true)?; @@ -270,9 +269,10 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io_at(process, at, |at, mut io| { let node = if path.is_empty() { - at.ok_or(Error::InvalidArgument)? + at + // at.ok_or(Error::InvalidArgument)? } else { - io.ioctx().find(None, path, follow, true)? + io.ioctx().find(Some(at), path, follow, true)? }; let metadata = node.metadata()?; @@ -287,13 +287,23 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) }) } + SyscallFunction::UpdateMetadata => { + let at = arg_option_fd(args[0] as u32); + let _path = arg_user_str(args[1] as usize, args[2] as usize)?; + let _update = arg_user_ref::(args[3] as usize)?; + let _follow = args[4] != 0; + + run_with_io_at(process, at, |_at, _io| { + todo!(); + }) + } SyscallFunction::CreateDirectory => { let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; let mode = FileMode::from(args[3] as u32); run_with_io_at(process, at, |at, mut io| { - io.ioctx().create_directory(at, path, mode)?; + io.ioctx().create_directory(Some(at), path, mode)?; Ok(0) }) } @@ -302,7 +312,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; run_with_io_at(process, at, |at, mut io| { - io.ioctx().remove_file(at, path)?; + io.ioctx().remove_file(Some(at), path)?; Ok(0) }) } @@ -459,7 +469,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let output = arg_user_mut::>(args[2] as usize)?; run_with_io(process, |mut io| { - let (master, slave) = File::new_pseudo_terminal(*options, *size); + let (master, slave) = File::new_pseudo_terminal(*options, *size)?; let master_fd = io.files.place_file(master, true)?; let slave_fd = io.files.place_file(slave, true)?; diff --git a/src/task/context.rs b/src/task/context.rs index 3dc3cdf4..a90f7c21 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -3,12 +3,10 @@ use abi::{arch::SavedFrame, error::Error}; use alloc::boxed::Box; use cfg_if::cfg_if; -use kernel_util::{mem::address::PhysicalAddress, thread::Termination}; +use kernel_util::thread::Termination; use crate::task::thread::Thread; -// use crate::task::process::Process; - cfg_if! { if #[cfg(target_arch = "aarch64")] { pub use crate::arch::aarch64::{context::TaskContext, cpu::Cpu}; @@ -47,12 +45,17 @@ pub trait TaskFrame { fn user_ip(&self) -> usize; } -/// XXX +/// Interface for performing context fork operations pub trait ForkFrame: Sized { - /// XXX + /// Constructs a "forked" task context by copying the registers from this one and supplying a + /// new address space to it. + /// + /// # Safety + /// + /// Unsafe: accepts raw frames and address space address. unsafe fn fork(&self, address_space: u64) -> Result; - /// XXX + /// Replaces the return value inside the frame with a new one fn set_return_value(&mut self, value: u64); } diff --git a/src/task/process.rs b/src/task/process.rs index b8524ac4..11f4543e 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -245,8 +245,12 @@ impl Process { let new_space = src_inner.space.fork()?; let new_context = frame.fork(new_space.as_address_with_asid())?; - let (new_process, new_main) = - Self::new_with_main(self.name(), Arc::new(new_space), new_context, None); + let (new_process, new_main) = Self::new_with_main( + self.name(), + Arc::new(new_space), + new_context, + src_inner.image.clone(), + ); { let mut src_io = self.io.lock(); @@ -269,7 +273,13 @@ impl Process { Ok(new_process.id()) } - /// XXX + /// Performs a "fork" operation on the process, creating an identical copy of it, cloning + /// the register state from provided [ForkFrame]. + /// + /// # Safety + /// + /// Unsafe: frame must be a valid frame to be forked, the function is not yet stable and does + /// not yet properly fork all the necessary context details. pub unsafe fn raw_fork(frame: &mut F) { let src_thread = Thread::current(); let src_process = src_thread.process(); @@ -282,7 +292,7 @@ impl Process { frame.set_return_value(rax as _); } - /// XXX + /// Replaces the process address space with a new one, loaded from the specified program pub fn exec(&self, program: &str, argv: Vec, envp: Vec) -> Result { if self.inner.read().threads.len() != 1 { todo!(); @@ -299,11 +309,11 @@ impl Process { drop(inner); // TODO old context is leaked - unsafe { (&*new_context).switch(&old_context) } + unsafe { (*new_context).switch(&old_context) } unreachable!() } - /// XXX + /// Returns the address space of the process pub fn space(&self) -> Arc { self.inner.read().space.clone() } @@ -420,7 +430,7 @@ impl Process { inner.state = ProcessState::Terminated(code); - // XXX + inner.space.clear().unwrap(); self.io.lock().handle_exit(); drop(inner); diff --git a/src/task/sched.rs b/src/task/sched.rs index 0f96cfd5..3d8ac7e6 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -52,7 +52,7 @@ impl CpuQueue { pub unsafe fn enter(&self) -> ! { let _guard = IrqGuard::acquire(); - (&*self.idle.as_ptr()).enter() + (*self.idle.as_ptr()).enter() } /// "Pushes" a thread to the queue for execution @@ -158,7 +158,7 @@ impl CpuQueue { if !core::ptr::eq(current_ctx, next_ctx) { // Perform the switch - (&*next_ctx).switch(&*current_ctx); + (*next_ctx).switch(&*current_ctx); true } else { diff --git a/src/task/thread.rs b/src/task/thread.rs index 37557644..0cf92f74 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -1,5 +1,3 @@ -// TODO XXX TODO XXX -#![allow(missing_docs)] //! Thread data structures and management use core::{ @@ -61,6 +59,7 @@ pub enum ThreadId { #[repr(C)] pub struct CurrentThread(Arc, IrqGuard); +/// Mask describing CPUs a thread is allowed to be scheduled to #[derive(Debug)] #[repr(transparent)] pub struct ThreadAffinity(AtomicU64); @@ -78,10 +77,14 @@ struct GlobalThreadList { data: BTreeMap>, } +/// Provides details about how the thread is scheduled onto CPUs pub struct ThreadSchedulingInfo { + /// Current state pub state: ThreadState, + /// Is the thread present in any queue pub in_queue: bool, + /// Queue into which the thread was last put pub queue: Option<&'static CpuQueue>, } @@ -89,9 +92,13 @@ static THREADS: IrqSafeRwLock = IrqSafeRwLock::new(GlobalThrea /// Describes a single thread within the system pub struct Thread { + /// Unique thread ID pub id: ThreadId, + /// Thread name pub name: Option, + /// Scheduling information pub sched: IrqSafeSpinlock, + /// Low-level context details pub context: Cell, process: Option>, @@ -101,6 +108,7 @@ pub struct Thread { signal_queue: SegQueue, pub(super) terminated: AtomicBool, pub(super) exit_notify: Arc, + /// CPU scheduling affinity mask pub affinity: ThreadAffinity, } @@ -126,21 +134,26 @@ impl GlobalThreadList { } impl ThreadAffinity { + /// Mask value for a thread to be scheduled onto any CPU pub const ANY_CPU: u64 = u64::MAX; + /// Constructs an affinity mask allowing a thread to be scheduled onto any CPU pub const fn any_cpu() -> Self { Self(AtomicU64::new(Self::ANY_CPU)) } + /// Constructs an affinity mask targeting a single CPU pub const fn only_cpu(index: usize) -> Self { Self(AtomicU64::new(1 << index)) } + /// Returns the current value of the thread's CPU affinity mask #[inline] pub fn get(&self) -> u64 { self.0.load(Ordering::Relaxed) } + /// Updates the thread's CPU affinity mask #[inline] pub fn set(&self, value: u64) { debug_assert_ne!(value, 0); @@ -206,6 +219,13 @@ impl Thread { ) } + /// Replaces the current context of the thread with a new one. + /// + /// TODO: how do I drop the old one within exec(), lol + /// + /// # Safety + /// + /// Unsafe: directly sets the thread's context. pub unsafe fn replace_context(&self, context: TaskContext) -> TaskContext { self.context.replace(context) } @@ -378,6 +398,7 @@ impl Thread { thread.map(|t| CurrentThread(t, cpu.into_guard())) } + /// Returns a thread for given `id`, if such exists pub fn get(id: ThreadId) -> Option> { THREADS.read().get(id).cloned() } @@ -414,6 +435,7 @@ impl CurrentThread { self.exit(code) } + /// Suspends a thread from further execution until it is awoken pub fn suspend(&self) -> Result<(), Error> { self.dequeue(ThreadState::Suspended); From ed5286ccd0c2522429a2b3fb555dc91df96517a0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 16 Jan 2024 16:11:09 +0200 Subject: [PATCH 149/211] proc: add Get/SetProcessInfo stubs --- src/arch/x86_64/exception.rs | 51 ++++++++++++++++++++++++++++++++++-- src/proc/exec.rs | 2 +- src/syscall/mod.rs | 6 ++++- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index a6bb3a8b..0922109d 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -9,6 +9,7 @@ use crate::{ x86_64::{apic, registers::CR3}, CpuAccess, }, + debug, task::{context::TaskFrame, thread::Thread, Cpu}, }; @@ -138,6 +139,53 @@ struct Pointer { offset: usize, } +impl ExceptionFrame { + fn dump(&self, level: debug::LogLevel) { + log_print_raw!(level, " CS:RIP = {:#x}:{:#x}\n", self.cs, self.rip); + log_print_raw!(level, " SS:RSP = {:#x}:{:#x}\n", self.ss, self.rsp); + + log_print_raw!( + level, + "RAX = {:#018x}, RCX = {:#018x}\n", + self.rax, + self.rcx + ); + log_print_raw!( + level, + "RDX = {:#018x}, RBX = {:#018x}\n", + self.rdx, + self.rbx + ); + log_print_raw!( + level, + "RSI = {:#018x}, RDI = {:#018x}\n", + self.rsi, + self.rdi + ); + log_print_raw!(level, "RBP = {:#018x}\n\n", self.rbp); + + log_print_raw!(level, " R8 = {:#018x}, R9 = {:#018x}\n", self.r8, self.r9); + log_print_raw!( + level, + "R10 = {:#018x}, R11 = {:#018x}\n", + self.r10, + self.r11 + ); + log_print_raw!( + level, + "R12 = {:#018x}, R13 = {:#018x}\n", + self.r12, + self.r13 + ); + log_print_raw!( + level, + "R14 = {:#018x}, R15 = {:#018x}\n", + self.r14, + self.r15 + ); + } +} + impl TaskFrame for IrqFrame { fn store(&self) -> SavedFrame { SavedFrame { @@ -292,8 +340,7 @@ fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { let cr3 = CR3.get(); warnln!("{:?} in {} {:?}", kind, thread.id, thread.name); - warnln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); - warnln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); + frame.dump(debug::LogLevel::Warning); warnln!("CR3 = {:#x}", cr3); match kind { diff --git a/src/proc/exec.rs b/src/proc/exec.rs index f8555f22..fadb1f72 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -118,7 +118,7 @@ fn setup_context( args: &Vec, envs: &Vec, ) -> Result { - const USER_STACK_PAGES: usize = 16; + const USER_STACK_PAGES: usize = 32; let virt_stack_base = 0x3000000; // 0x1000 of guard page diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 0f22dbcc..f1d27c7d 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -282,6 +282,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result size, ty: node.ty(), mode: metadata.mode, + uid: metadata.uid, + gid: metadata.gid, }); Ok(0) @@ -617,6 +619,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result process.set_group_id(group_id); Ok(0) } + SyscallFunction::GetProcessInfo => todo!(), + SyscallFunction::SetProcessInfo => todo!(), SyscallFunction::StartSession => { let session_terminal = process.clear_session_terminal(); @@ -654,7 +658,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } - _ => unreachable!(), + SyscallFunction::Fork => unreachable!(), } } From b006a108ee5f1820ed3397dc3c0f6f3120c070be Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 16 Jan 2024 17:36:43 +0200 Subject: [PATCH 150/211] vfs/pty: proper PTY closing --- lib/vfs/src/pty.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index f2d763de..6ba5ad51 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -4,6 +4,7 @@ // TODO handle werase key use core::{ pin::Pin, + sync::atomic::{AtomicBool, Ordering}, task::{Context, Poll}, }; @@ -19,8 +20,8 @@ use kernel_util::{ use yggdrasil_abi::{ error::Error, io::{ - DeviceRequest, TerminalInputOptions, TerminalLineOptions, TerminalOptions, - TerminalOutputOptions, TerminalSize, + DeviceRequest, TerminalControlCharacters, TerminalInputOptions, TerminalLineOptions, + TerminalOptions, TerminalOutputOptions, TerminalSize, }, process::Signal, }; @@ -38,6 +39,7 @@ pub struct PseudoTerminal { signal_pgroup: IrqSafeRwLock>, slave_to_master: PtyHalf, master_to_slave: PtyHalf, + slave_closed: AtomicBool, size: IrqSafeRwLock, } /// Slave part of a PTY device @@ -102,7 +104,7 @@ impl PtyHalf { let lock = self.ring.lock(); let ready = if config.is_canonical() { - lock.is_readable() && lock.contains(&b'\n') + lock.is_readable() && (lock.contains(&b'\n') || lock.contains(&config.chars.eof)) } else { lock.is_readable() }; @@ -170,8 +172,8 @@ impl PtyHalf { } } - fn read_raw(&self, buffer: &mut [u8]) -> Result { - if buffer.is_empty() { + fn read_raw(&self, closed: &AtomicBool, buffer: &mut [u8]) -> Result { + if buffer.is_empty() || closed.load(Ordering::Acquire) { return Ok(0); } @@ -179,7 +181,8 @@ impl PtyHalf { if lock.is_readable() { let mut pos = 0; while lock.is_readable() { - buffer[pos] = unsafe { lock.read_single_unchecked() }; + let ch = unsafe { lock.read_single_unchecked() }; + buffer[pos] = ch; pos += 1; } @@ -189,10 +192,10 @@ impl PtyHalf { } } - fn poll_raw(&self, cx: &mut Context<'_>) -> Poll> { + fn poll_raw(&self, closed: &AtomicBool, cx: &mut Context<'_>) -> Poll> { self.notify.register(cx.waker()); - if self.ring.lock().is_readable() { + if closed.load(Ordering::Acquire) || self.ring.lock().is_readable() { self.notify.remove(cx.waker()); Poll::Ready(Ok(())) } else { @@ -219,6 +222,7 @@ impl PseudoTerminal { signal_pgroup: IrqSafeRwLock::new(None), master_to_slave: PtyHalf::new()?, slave_to_master: PtyHalf::new()?, + slave_closed: AtomicBool::new(false), size: IrqSafeRwLock::new(size), }); @@ -298,7 +302,7 @@ impl PseudoTerminal { } fn read_from_slave(&self, buf: &mut [u8]) -> Result { - self.slave_to_master.read_raw(buf) + self.slave_to_master.read_raw(&self.slave_closed, buf) } fn read_from_master(&self, buf: &mut [u8]) -> Result { @@ -307,7 +311,7 @@ impl PseudoTerminal { } fn poll_from_slave(&self, cx: &mut Context<'_>) -> Poll> { - self.slave_to_master.poll_raw(cx) + self.slave_to_master.poll_raw(&self.slave_closed, cx) } fn poll_from_master(&self, cx: &mut Context<'_>) -> Poll> { @@ -392,6 +396,7 @@ impl Drop for PseudoTerminalMaster { impl Drop for PseudoTerminalSlave { fn drop(&mut self) { - // TODO + self.pty.slave_closed.store(true, Ordering::Release); + self.pty.slave_to_master.notify.wake_all(); } } From 9bd29970f85d09e1b9400cf80a9269404622e30b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 16 Jan 2024 23:44:14 +0200 Subject: [PATCH 151/211] dev/pty: rewrite pty implementation --- lib/vfs/src/lib.rs | 2 +- lib/vfs/src/pty.rs | 363 +++++++++++++++++++++++++++------------------ 2 files changed, 217 insertions(+), 148 deletions(-) diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 186f7c70..1e63cfd9 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -3,7 +3,7 @@ #![cfg_attr(not(test), no_std)] #![allow(clippy::new_ret_no_self, clippy::new_without_default)] #![deny(missing_docs)] -#![feature(if_let_guard, maybe_uninit_slice, trait_alias, let_chains)] +#![feature(if_let_guard, maybe_uninit_slice, trait_alias, let_chains, new_uninit)] #[cfg(test)] extern crate hosted_tests; diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 6ba5ad51..2f0ab550 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -1,14 +1,14 @@ //! Pseudo-terminal devices -// TODO handle erase key // TODO handle werase key use core::{ + mem::MaybeUninit, pin::Pin, sync::atomic::{AtomicBool, Ordering}, task::{Context, Poll}, }; -use alloc::sync::Arc; +use alloc::{boxed::Box, sync::Arc}; use futures_util::Future; use kernel_util::{ block, @@ -28,18 +28,32 @@ use yggdrasil_abi::{ const CAPACITY: usize = 8192; -struct PtyHalf { +struct PtySlaveToMasterHalf { ring: IrqSafeSpinlock>, notify: QueueWaker, + shutdown: AtomicBool, +} + +struct MasterToSlaveBuffer { + pending: Box<[MaybeUninit]>, + ready: RingBuffer, + + position: usize, +} + +struct PtyMasterToSlaveHalf { + // Actual data to be read by the slave + buffer: IrqSafeSpinlock, + signal_pgroup: IrqSafeRwLock>, + // Notified when data is available in the buffer + notify: QueueWaker, } /// Pseudo-terminal shared device pub struct PseudoTerminal { config: IrqSafeRwLock, - signal_pgroup: IrqSafeRwLock>, - slave_to_master: PtyHalf, - master_to_slave: PtyHalf, - slave_closed: AtomicBool, + slave_to_master: PtySlaveToMasterHalf, + master_to_slave: PtyMasterToSlaveHalf, size: IrqSafeRwLock, } /// Slave part of a PTY device @@ -53,101 +67,114 @@ pub struct PseudoTerminalMaster { pty: Arc, } -fn input_discipline( - mut lock: IrqSafeSpinlockGuard>, - config: &TerminalOptions, - buffer: &mut [u8], -) -> Result { +fn read_all(source: &mut RingBuffer, target: &mut [u8], eof: Option) -> usize { let mut pos = 0; - - if !config.is_canonical() { - while pos < buffer.len() && lock.is_readable() { - buffer[pos] = unsafe { lock.read_single_unchecked() }; - pos += 1; + while pos < target.len() && source.is_readable() { + let ch = unsafe { source.read_single_unchecked() }; + if eof.map(|eof| eof == ch).unwrap_or(false) { + break; } - Ok(pos) - } else { - while pos < buffer.len() && lock.is_readable() { - let ch = unsafe { lock.read_single_unchecked() }; + target[pos] = ch; + pos += 1; + } + pos +} - if ch == config.chars.eof { - break; - } - if ch == config.chars.erase { - pos = pos.saturating_sub(1); - continue; - } - - buffer[pos] = ch; - pos += 1; - - if ch == b'\n' { - break; - } +impl MasterToSlaveBuffer { + pub fn write_pending(&mut self, byte: u8) { + if self.position == self.pending.len() { + // TODO not sure how to handle this case + todo!(); } - Ok(pos) + + self.pending[self.position].write(byte); + self.position += 1; + } + + pub fn erase_pending(&mut self) -> bool { + if self.position != 0 { + self.position -= 1; + true + } else { + false + } + } + + pub fn write_ready(&mut self, byte: u8) { + self.ready.write(byte); + } + + pub fn flush(&mut self) { + if self.position == 0 { + return; + } + log::debug!("m->s: flush {}", self.position); + + for &ch in unsafe { MaybeUninit::slice_assume_init_ref(&self.pending[..self.position]) } { + self.ready.write(ch); + } + + self.position = 0; } } -impl PtyHalf { - pub fn new() -> Result { +impl PtyMasterToSlaveHalf { + pub fn with_capacity(capacity: usize) -> Result { Ok(Self { - ring: IrqSafeSpinlock::new(RingBuffer::try_with_capacity(CAPACITY)?), + buffer: IrqSafeSpinlock::new(MasterToSlaveBuffer { + pending: Box::new_uninit_slice(256), + ready: RingBuffer::try_with_capacity(capacity)?, + position: 0, + }), + signal_pgroup: IrqSafeRwLock::new(None), notify: QueueWaker::new(), }) } - fn try_begin_read<'a>( - &'a self, - config: &TerminalOptions, - ) -> Option>> { - let lock = self.ring.lock(); - - let ready = if config.is_canonical() { - lock.is_readable() && (lock.contains(&b'\n') || lock.contains(&config.chars.eof)) + pub fn read(&self, buf: &mut [u8], eof: Option) -> Result { + if let Some(mut lock) = self.try_read() { + Ok(read_all(&mut lock.ready, buf, eof)) } else { - lock.is_readable() - }; - - if ready { - Some(lock) - } else { - None + block!(self.read_async(buf, eof).await)? } } - fn read(&self, config: &TerminalOptions, buffer: &mut [u8]) -> Result { - if buffer.is_empty() { - return Ok(0); - } + pub fn poll(&self, cx: &mut Context<'_>) -> Poll> { + todo!() + } - if let Some(lock) = self.try_begin_read(config) { - input_discipline(lock, config, buffer) - } else { - block!(self.read_async(config, buffer).await)? - } + pub fn flush(&self) { + self.buffer.lock().flush(); + self.notify.wake_all(); + } + + fn try_read(&self) -> Option> { + let lock = self.buffer.lock(); + lock.ready.is_readable().then_some(lock) } fn read_async<'a>( &'a self, - config: &'a TerminalOptions, buffer: &'a mut [u8], + eof: Option, ) -> impl Future> + 'a { struct F<'f> { - config: &'f TerminalOptions, - half: &'f PtyHalf, + this: &'f PtyMasterToSlaveHalf, buffer: &'f mut [u8], + eof: Option, } impl<'f> Future for F<'f> { type Output = Result; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.half.notify.register(cx.waker()); + let eof = self.eof; + self.this.notify.register(cx.waker()); - if let Some(lock) = self.half.try_begin_read(self.config) { - self.half.notify.remove(cx.waker()); - Poll::Ready(input_discipline(lock, self.config, self.buffer)) + if let Some(mut lock) = self.this.try_read() { + self.this.notify.remove(cx.waker()); + let count = read_all(&mut lock.ready, self.buffer, eof); + Poll::Ready(Ok(count)) } else { Poll::Pending } @@ -155,47 +182,51 @@ impl PtyHalf { } F { - half: self, + this: self, buffer, - config, + eof, } } +} - fn poll_read(&self, config: &TerminalOptions, cx: &mut Context<'_>) -> Poll> { - self.notify.register(cx.waker()); - - if self.try_begin_read(config).is_some() { - self.notify.remove(cx.waker()); - Poll::Ready(Ok(())) - } else { - Poll::Pending - } +impl PtySlaveToMasterHalf { + pub fn with_capacity(capacity: usize) -> Result { + Ok(Self { + ring: IrqSafeSpinlock::new(RingBuffer::try_with_capacity(capacity)?), + notify: QueueWaker::new(), + shutdown: AtomicBool::new(false), + }) } - fn read_raw(&self, closed: &AtomicBool, buffer: &mut [u8]) -> Result { - if buffer.is_empty() || closed.load(Ordering::Acquire) { + pub fn handle_input(&self, byte: u8, config: &TerminalOutputOptions) { + let mut lock = self.ring.lock(); + + // if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { + // self.slave_to_master.putc(b'\r', true); + // } + + lock.write(byte); + + self.notify.wake_one(); + } + + pub fn read(&self, buf: &mut [u8]) -> Result { + if self.shutdown.load(Ordering::Acquire) { return Ok(0); } - let mut lock = self.ring.lock(); - if lock.is_readable() { - let mut pos = 0; - while lock.is_readable() { - let ch = unsafe { lock.read_single_unchecked() }; - buffer[pos] = ch; - pos += 1; - } - - Ok(pos) + if let Some(mut lock) = self.try_read() { + let count = read_all(&mut lock, buf, None); + Ok(count) } else { todo!() } } - fn poll_raw(&self, closed: &AtomicBool, cx: &mut Context<'_>) -> Poll> { + pub fn poll(&self, cx: &mut Context<'_>) -> Poll> { self.notify.register(cx.waker()); - if closed.load(Ordering::Acquire) || self.ring.lock().is_readable() { + if self.shutdown.load(Ordering::Acquire) || self.ring.lock().is_readable() { self.notify.remove(cx.waker()); Poll::Ready(Ok(())) } else { @@ -203,11 +234,9 @@ impl PtyHalf { } } - fn putc(&self, byte: u8, signal: bool) { - self.ring.lock().write(byte); - if signal { - self.notify.wake_all(); - } + fn try_read(&self) -> Option>> { + let lock = self.ring.lock(); + lock.is_readable().then_some(lock) } } @@ -217,12 +246,13 @@ impl PseudoTerminal { config: TerminalOptions, size: TerminalSize, ) -> Result<(PseudoTerminalMaster, PseudoTerminalSlave), Error> { + let master_to_slave = PtyMasterToSlaveHalf::with_capacity(CAPACITY)?; + let slave_to_master = PtySlaveToMasterHalf::with_capacity(CAPACITY)?; + let pty = Arc::new(Self { config: IrqSafeRwLock::new(config), - signal_pgroup: IrqSafeRwLock::new(None), - master_to_slave: PtyHalf::new()?, - slave_to_master: PtyHalf::new()?, - slave_closed: AtomicBool::new(false), + master_to_slave, + slave_to_master, size: IrqSafeRwLock::new(size), }); @@ -234,57 +264,75 @@ impl PseudoTerminal { fn putc_from_slave(&self, byte: u8) { let config = self.config.read(); - - if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { - self.slave_to_master.putc(b'\r', true); - } - - self.slave_to_master.putc(byte, true) + self.slave_to_master.handle_input(byte, &config.output) } fn putc_from_master(&self, mut byte: u8) { let config = self.config.read(); + let mut buffer = self.master_to_slave.buffer.lock(); + if byte == b'\r' && config.input.contains(TerminalInputOptions::CR_TO_NL) { byte = b'\n'; } - if byte == b'\n' { - if config.is_echo_newline() { - if config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { - self.slave_to_master.putc(b'\r', true); + if config.is_canonical() { + // Canonical line processing + + // Echo back + if byte == config.chars.erase { + let echo = + buffer.erase_pending() && config.line.contains(TerminalLineOptions::ECHO_ERASE); + + if echo { + for &ch in b"\x1B[D \x1B[D" { + self.slave_to_master.handle_input(ch, &config.output); + } } - self.slave_to_master.putc(byte, true); - } - } else if config.line.contains(TerminalLineOptions::ECHO) { - if byte.is_ascii_control() { - if byte != config.chars.erase && byte != config.chars.werase { - self.slave_to_master.putc(b'^', false); - self.slave_to_master.putc(byte + 0x40, true); + + return; + } else if byte == b'\n' { + // TODO NL_TO_CRNL + if config.is_echo_newline() { + self.slave_to_master.handle_input(byte, &config.output); + } + } else if byte.is_ascii_control() { + if config.line.contains(TerminalLineOptions::ECHO) { + self.slave_to_master.handle_input(b'^', &config.output); + self.slave_to_master + .handle_input(byte + 0x40, &config.output); } } else { - self.slave_to_master.putc(byte, true); + if config.line.contains(TerminalLineOptions::ECHO) { + self.slave_to_master.handle_input(byte, &config.output); + } } - } - if byte == config.chars.interrupt && config.line.contains(TerminalLineOptions::SIGNAL) { - self.slave_to_master.notify.wake_all(); + if byte == config.chars.interrupt { + if config.line.contains(TerminalLineOptions::SIGNAL) { + self.slave_to_master.notify.wake_all(); - if let Some(group_id) = *self.signal_pgroup.read() { - signal_process_group(group_id, Signal::Interrupted); - return; + if let Some(group_id) = *self.master_to_slave.signal_pgroup.read() { + signal_process_group(group_id, Signal::Interrupted); + self.master_to_slave.notify.wake_all(); + return; + } + } else { + buffer.write_pending(byte); + } + } else { + buffer.write_pending(byte); } - } - if byte == config.chars.eof { - self.master_to_slave.putc(byte, true); - return; + if byte == b'\n' || byte == config.chars.eof { + buffer.flush(); + self.master_to_slave.notify.wake_one(); + } + } else { + // Raw line processing + buffer.write_ready(byte); + self.master_to_slave.notify.wake_one(); } - - self.master_to_slave.putc( - byte, - !config.is_canonical() || byte == b'\n' || byte == b'\r', - ); } fn write_from_slave(&self, buf: &[u8]) -> Result { @@ -302,27 +350,40 @@ impl PseudoTerminal { } fn read_from_slave(&self, buf: &mut [u8]) -> Result { - self.slave_to_master.read_raw(&self.slave_closed, buf) + self.slave_to_master.read(buf) } fn read_from_master(&self, buf: &mut [u8]) -> Result { - let config = self.config.read(); - self.master_to_slave.read(&config, buf) + let eof = { + let config = self.config.read(); + config.is_canonical().then_some(config.chars.eof) + }; + self.master_to_slave.read(buf, eof) } fn poll_from_slave(&self, cx: &mut Context<'_>) -> Poll> { - self.slave_to_master.poll_raw(&self.slave_closed, cx) + self.slave_to_master.poll(cx) } fn poll_from_master(&self, cx: &mut Context<'_>) -> Poll> { + self.master_to_slave.poll(cx) + } + + fn close_master(&self) { let config = self.config.read(); - self.master_to_slave.poll_read(&config, cx) + let mut buffer = self.master_to_slave.buffer.lock(); + + buffer.write_ready(config.chars.eof); + self.master_to_slave.notify.wake_one(); } fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { match req { DeviceRequest::SetTerminalGroup(group_id) => { - self.signal_pgroup.write().replace(*group_id); + self.master_to_slave + .signal_pgroup + .write() + .replace(*group_id); Ok(()) } DeviceRequest::GetTerminalOptions(options) => { @@ -330,6 +391,7 @@ impl PseudoTerminal { Ok(()) } DeviceRequest::SetTerminalOptions(options) => { + self.master_to_slave.flush(); *self.config.write() = *options; Ok(()) } @@ -337,6 +399,12 @@ impl PseudoTerminal { size.write(*self.size.read()); Ok(()) } + DeviceRequest::SetTerminalSize(size) => { + // TODO SIGWINCH? + // TODO validate + *self.size.write() = *size; + Ok(()) + } _ => Err(Error::InvalidOperation), } } @@ -388,15 +456,16 @@ impl PseudoTerminalMaster { impl Drop for PseudoTerminalMaster { fn drop(&mut self) { - self.pty - .master_to_slave - .putc(self.pty.config.read().chars.eof, true); + self.pty.close_master(); } } impl Drop for PseudoTerminalSlave { fn drop(&mut self) { - self.pty.slave_closed.store(true, Ordering::Release); + self.pty + .slave_to_master + .shutdown + .store(true, Ordering::Release); self.pty.slave_to_master.notify.wake_all(); } } From 5d8067991dca171da9dc32b0a04af02251fef754 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 20 Jan 2024 19:40:27 +0200 Subject: [PATCH 152/211] net: Basic UDP/ICMP over IPv4 networking using virtio-net --- Cargo.lock | 1055 +++++++++++++++++++++++ Cargo.toml | 2 + driver/bus/pci/src/capability.rs | 141 +++ driver/bus/pci/src/lib.rs | 33 + driver/bus/pci/src/space/mod.rs | 11 +- driver/fs/kernel-fs/src/devfs.rs | 84 +- driver/net/core/Cargo.toml | 16 + driver/net/core/src/ethernet.rs | 100 +++ driver/net/core/src/interface.rs | 116 +++ driver/net/core/src/l3/arp.rs | 142 +++ driver/net/core/src/l3/ip.rs | 148 ++++ driver/net/core/src/l3/mod.rs | 187 ++++ driver/net/core/src/l4/icmp.rs | 104 +++ driver/net/core/src/l4/mod.rs | 2 + driver/net/core/src/l4/udp.rs | 79 ++ driver/net/core/src/lib.rs | 139 +++ driver/net/core/src/queue.rs | 53 ++ driver/net/core/src/socket.rs | 198 +++++ driver/net/core/src/types.rs | 225 +++++ driver/virtio/core/Cargo.toml | 21 + driver/virtio/core/src/error.rs | 20 + driver/virtio/core/src/lib.rs | 54 ++ driver/virtio/core/src/queue.rs | 344 ++++++++ driver/virtio/core/src/transport/mod.rs | 92 ++ driver/virtio/core/src/transport/pci.rs | 119 +++ driver/virtio/net/Cargo.toml | 24 + driver/virtio/net/src/lib.rs | 316 +++++++ lib/kernel-util/src/api.rs | 2 + lib/kernel-util/src/mem/device.rs | 34 + lib/kernel-util/src/mem/mod.rs | 11 +- lib/kernel-util/src/util/ring.rs | 7 +- lib/vfs/src/file/mod.rs | 52 +- lib/vfs/src/lib.rs | 2 + lib/vfs/src/node/impls.rs | 4 + lib/vfs/src/socket.rs | 53 ++ src/arch/mod.rs | 8 + src/arch/x86_64/mem/mod.rs | 17 + src/arch/x86_64/mod.rs | 11 + src/init.rs | 2 + src/syscall/mod.rs | 43 +- 40 files changed, 4054 insertions(+), 17 deletions(-) create mode 100644 Cargo.lock create mode 100644 driver/net/core/Cargo.toml create mode 100644 driver/net/core/src/ethernet.rs create mode 100644 driver/net/core/src/interface.rs create mode 100644 driver/net/core/src/l3/arp.rs create mode 100644 driver/net/core/src/l3/ip.rs create mode 100644 driver/net/core/src/l3/mod.rs create mode 100644 driver/net/core/src/l4/icmp.rs create mode 100644 driver/net/core/src/l4/mod.rs create mode 100644 driver/net/core/src/l4/udp.rs create mode 100644 driver/net/core/src/lib.rs create mode 100644 driver/net/core/src/queue.rs create mode 100644 driver/net/core/src/socket.rs create mode 100644 driver/net/core/src/types.rs create mode 100644 driver/virtio/core/Cargo.toml create mode 100644 driver/virtio/core/src/error.rs create mode 100644 driver/virtio/core/src/lib.rs create mode 100644 driver/virtio/core/src/queue.rs create mode 100644 driver/virtio/core/src/transport/mod.rs create mode 100644 driver/virtio/core/src/transport/pci.rs create mode 100644 driver/virtio/net/Cargo.toml create mode 100644 driver/virtio/net/src/lib.rs create mode 100644 lib/vfs/src/socket.rs diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..245875f0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1055 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aarch64-cpu" +version = "9.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac42a04a61c19fc8196dd728022a784baecc5d63d7e256c01ad1b3fbfab26287" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "accessor" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd8b2abd55bf1f9cffbf00fd594566c51a9d31402553284920c1309ca8351086" + +[[package]] +name = "acpi" +version = "4.1.1" +source = "git+https://github.com/alnyan/acpi.git?branch=acpi-system#5efda0f9c17bccb922cf2b6b5fd73088bb887965" +dependencies = [ + "bit_field", + "log", + "rsdp", +] + +[[package]] +name = "acpi-system" +version = "0.1.0" +source = "git+https://github.com/alnyan/acpi-system.git#17192e5f603fdc7d78c9b83885e8a9d9aac45af7" +dependencies = [ + "acpi", + "aml", + "bit_field", + "enum-map", + "log", +] + +[[package]] +name = "aml" +version = "0.16.4" +source = "git+https://github.com/alnyan/acpi.git?branch=acpi-system#5efda0f9c17bccb922cf2b6b5fd73088bb887965" +dependencies = [ + "bit_field", + "bitvec", + "byteorder", + "log", + "spinning_top", +] + +[[package]] +name = "atomic_enum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "btree_monstrousity" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d0977e9c15f276380f16f2e9594257c258172b23af39ffd2e4cf5971cb38c7" +dependencies = [ + "cfg-if", + "rustversion", +] + +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "device-api" +version = "0.1.0" +dependencies = [ + "device-api-macros", + "yggdrasil-abi", +] + +[[package]] +name = "device-api-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "discrete_range_map" +version = "0.6.2" +source = "git+https://git.alnyan.me/yggdrasil/discrete_range_map.git#10fd79828d2918bd079a11c2b2c623eede170c3f" +dependencies = [ + "btree_monstrousity", + "either", + "itertools 0.12.0", + "serde", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "elf" +version = "0.7.2" +source = "git+https://git.alnyan.me/yggdrasil/yggdrasil-elf.git#419cd311de2e9514b5033677cde9a33f7d0ba4a2" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "endian-type-rs" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6419a5c75e40011b9fe0174db3fe24006ab122fbe1b7e9cc5974b338a755c76" + +[[package]] +name = "enum-map" +version = "2.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fdt-rs" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99a40cabc11c8258822a593f5c51f2d9f4923e715ca9e2a0630cf77ae15f390b" +dependencies = [ + "endian-type-rs", + "fallible-iterator", + "memoffset 0.5.6", + "num-derive", + "num-traits", + "rustc_version", + "static_assertions", + "unsafe_unwrap", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "hosted-tests" +version = "0.1.0" +dependencies = [ + "yggdrasil-abi", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + +[[package]] +name = "kernel-fs" +version = "0.1.0" +dependencies = [ + "kernel-util", + "log", + "vfs", + "ygg_driver_block", + "yggdrasil-abi", +] + +[[package]] +name = "kernel-util" +version = "0.1.0" +dependencies = [ + "crossbeam-queue", + "device-api", + "futures-util", + "log", + "yggdrasil-abi", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "linked_list_allocator" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afa463f5405ee81cdb9cc2baf37e08ec7e4c8209442b5d72c04cfb2cd6e6286" +dependencies = [ + "spinning_top", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memfs" +version = "0.1.0" +dependencies = [ + "kernel-util", + "log", + "static_assertions", + "vfs", + "yggdrasil-abi", +] + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memtables" +version = "0.1.0" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.4.1", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rsdp" +version = "2.0.0" +source = "git+https://github.com/alnyan/acpi.git?branch=acpi-system#5efda0f9c17bccb922cf2b6b5fd73088bb887965" +dependencies = [ + "log", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.195" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.195" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "spinning_top" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9eb1a2f4c41445a3a0ff9abc5221c5fcd28e1f13cd7c0397706f9ac938ddb0" +dependencies = [ + "lock_api", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "tock-registers" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unsafe_unwrap" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1230ec65f13e0f9b28d789da20d2d419511893ea9dac2c1f4ef67b8b14e5da80" + +[[package]] +name = "uuid" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "vfs" +version = "0.1.0" +dependencies = [ + "futures-util", + "hosted-tests", + "kernel-util", + "log", + "ygg_driver_block", + "yggdrasil-abi", +] + +[[package]] +name = "vmalloc" +version = "0.1.0" +dependencies = [ + "discrete_range_map", + "itertools 0.11.0", + "proptest", + "yggdrasil-abi", +] + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xhci" +version = "0.9.2" +source = "git+https://github.com/rust-osdev/xhci.git#6144829ff13c2499dc812cea09df4e5a1d0862cc" +dependencies = [ + "accessor", + "bit_field", + "num-derive", + "num-traits", + "paste", +] + +[[package]] +name = "yboot-proto" +version = "0.1.0" +source = "git+https://git.alnyan.me/yggdrasil/yboot-proto.git#edf5db1b1d5d5dcb9d737cd7ec23b2e124b3b027" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "ygg_driver_ahci" +version = "0.1.0" +dependencies = [ + "bytemuck", + "device-api", + "futures-util", + "kernel-fs", + "kernel-util", + "log", + "memoffset 0.9.0", + "static_assertions", + "tock-registers", + "vfs", + "ygg_driver_block", + "ygg_driver_pci", + "yggdrasil-abi", +] + +[[package]] +name = "ygg_driver_block" +version = "0.1.0" +dependencies = [ + "bytemuck", + "futures-util", + "kernel-util", + "log", + "static_assertions", + "uuid", + "yggdrasil-abi", +] + +[[package]] +name = "ygg_driver_nvme" +version = "0.1.0" +dependencies = [ + "bytemuck", + "device-api", + "futures-util", + "kernel-fs", + "kernel-util", + "log", + "static_assertions", + "tock-registers", + "vfs", + "ygg_driver_block", + "ygg_driver_pci", + "yggdrasil-abi", +] + +[[package]] +name = "ygg_driver_pci" +version = "0.1.0" +dependencies = [ + "acpi", + "bitflags 2.4.1", + "device-api", + "kernel-util", + "log", + "yggdrasil-abi", +] + +[[package]] +name = "ygg_driver_virtio_net" +version = "0.1.0" +dependencies = [ + "ygg_driver_pci", +] + +[[package]] +name = "yggdrasil-abi" +version = "0.1.0" +source = "git+https://git.alnyan.me/yggdrasil/yggdrasil-abi.git#d2173e1e9e92e668399cf83aeb32e09823826821" + +[[package]] +name = "yggdrasil-kernel" +version = "0.1.0" +dependencies = [ + "aarch64-cpu", + "acpi", + "acpi-system", + "aml", + "atomic_enum", + "bitflags 2.4.1", + "bytemuck", + "cfg-if", + "crossbeam-queue", + "device-api", + "device-api-macros", + "elf", + "fdt-rs", + "futures-util", + "git-version", + "kernel-fs", + "kernel-util", + "linked_list_allocator", + "log", + "memfs", + "memtables", + "spinning_top", + "static_assertions", + "tock-registers", + "vfs", + "vmalloc", + "xhci", + "yboot-proto", + "ygg_driver_ahci", + "ygg_driver_block", + "ygg_driver_nvme", + "ygg_driver_pci", + "ygg_driver_virtio_net", + "yggdrasil-abi", +] diff --git a/Cargo.toml b/Cargo.toml index cf505ff5..c69cca0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ device-api-macros = { path = "lib/device-api/macros" } # Drivers ygg_driver_block = { path = "driver/block/core" } +ygg_driver_net_core = { path = "driver/net/core" } kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } @@ -57,6 +58,7 @@ xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_nvme = { path = "driver/block/nvme" } ygg_driver_ahci = { path = "driver/block/ahci" } +ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] } [features] default = ["fb_console"] diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index cb62c932..387e070d 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -11,12 +11,48 @@ use yggdrasil_abi::error::Error; use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; use crate::PciBaseAddress; +pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized { + fn from_space_offset(space: &'s S, offset: usize) -> Self; + + fn space(&self) -> &'s S; + fn offset(&self) -> usize; + + fn bar_index(&self) -> Option { + let value = self.space().read_u8(self.offset() + 4); + (value <= 0x5).then_some(value as _) + } + + fn bar_offset(&self) -> usize { + let value = self.space().read_u32(self.offset() + 8); + value as _ + } + + fn length(&self) -> usize { + let value = self.space().read_u32(self.offset() + 12); + value as _ + } +} + +pub trait VirtioCapability { + const CFG_TYPE: u8; + const MIN_LEN: usize = 0; + type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>; +} + /// MSI-X capability query pub struct MsiXCapability; /// MSI capability query pub struct MsiCapability; +// VirtIO-over-PCI capabilities +/// VirtIO PCI configuration access +pub struct VirtioDeviceConfigCapability; +/// VirtIO common configuration +pub struct VirtioCommonConfigCapability; +/// VirtIO notify configuration +pub struct VirtioNotifyConfigCapability; + /// Represents an entry in MSI-X vector table #[repr(C)] pub struct MsiXEntry { @@ -44,6 +80,39 @@ pub struct MsiData<'s, S: PciConfigurationSpace + ?Sized + 's> { offset: usize, } +pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + +pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + +pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + +impl PciCapability for T { + const ID: PciCapabilityId = PciCapabilityId::VendorSpecific; + type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>; + + fn check(space: &S, offset: usize, len: usize) -> bool { + let cfg_type = space.read_u8(offset + 3); + cfg_type == T::CFG_TYPE && len >= T::MIN_LEN + } + + fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( + space: &'s S, + offset: usize, + _len: usize, + ) -> Self::CapabilityData<'s, S> { + T::Output::from_space_offset(space, offset) + } +} + impl PciCapability for MsiXCapability { const ID: PciCapabilityId = PciCapabilityId::MsiX; type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>; @@ -51,6 +120,7 @@ impl PciCapability for MsiXCapability { fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( space: &'s S, offset: usize, + _len: usize, ) -> Self::CapabilityData<'s, S> { MsiXData { space, offset } } @@ -63,11 +133,82 @@ impl PciCapability for MsiCapability { fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( space: &'s S, offset: usize, + _len: usize, ) -> Self::CapabilityData<'s, S> { MsiData { space, offset } } } +impl VirtioCapability for VirtioDeviceConfigCapability { + const CFG_TYPE: u8 = 0x04; + type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>; +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S> + for VirtioDeviceConfigData<'s, S> +{ + fn from_space_offset(space: &'s S, offset: usize) -> Self { + Self { space, offset } + } + + fn space(&self) -> &'s S { + self.space + } + + fn offset(&self) -> usize { + self.offset + } +} + +impl VirtioCapability for VirtioCommonConfigCapability { + const CFG_TYPE: u8 = 0x01; + type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>; +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S> + for VirtioCommonConfigData<'s, S> +{ + fn from_space_offset(space: &'s S, offset: usize) -> Self { + Self { space, offset } + } + + fn space(&self) -> &'s S { + self.space + } + + fn offset(&self) -> usize { + self.offset + } +} + +impl VirtioCapability for VirtioNotifyConfigCapability { + const CFG_TYPE: u8 = 0x02; + const MIN_LEN: usize = 0x14; + type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>; +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> { + pub fn offset_multiplier(&self) -> usize { + self.space.read_u32(self.offset + 16) as usize + } +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S> + for VirtioNotifyConfigData<'s, S> +{ + fn from_space_offset(space: &'s S, offset: usize) -> Self { + Self { space, offset } + } + + fn space(&self) -> &'s S { + self.space + } + + fn offset(&self) -> usize { + self.offset + } +} + impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { // TODO use pending bits as well /// Maps and returns the vector table associated with the device's MSI-X capability diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index c01b1dbb..c76ae9ad 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -73,6 +73,8 @@ pub enum PciBaseAddress { pub enum PciCapabilityId { /// MSI (32-bit or 64-bit) Msi = 0x05, + /// Vendor-specific capability + VendorSpecific = 0x09, /// MSI-X MsiX = 0x11, /// Unknown capability missing from this list @@ -86,10 +88,15 @@ pub trait PciCapability { /// Wrapper for accessing the capability data structure type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a>; + fn check(space: &S, offset: usize, len: usize) -> bool { + true + } + /// Constructs an access wrapper for this capability with given offset fn data<'s, S: PciConfigurationSpace + ?Sized + 's>( space: &'s S, offset: usize, + len: usize, ) -> Self::CapabilityData<'s, S>; } @@ -104,6 +111,7 @@ pub struct PciDeviceInfo { pub enum PciMatch { Generic(fn(&PciDeviceInfo) -> bool), + Vendor(u16, u16), Class(u8, Option, Option), } @@ -134,6 +142,15 @@ pub struct PciBusManager { segments: Vec, } +impl PciBaseAddress { + pub fn as_memory(self) -> usize { + match self { + Self::Memory(address) => address, + _ => panic!("Not a memory BAR"), + } + } +} + impl PciBusSegment { fn probe_config_space(&self, address: PciAddress) -> Result, Error> { match self.ecam_phys_base { @@ -344,6 +361,9 @@ impl PciMatch { pub fn check_device(&self, info: &PciDeviceInfo, class: u8, subclass: u8, prog_if: u8) -> bool { match self { Self::Generic(f) => f(info), + &Self::Vendor(vendor_, device_) => { + info.config_space.vendor_id() == vendor_ && info.config_space.device_id() == device_ + } &Self::Class(class_, Some(subclass_), Some(prog_if_)) => { class_ == class && subclass_ == subclass && prog_if_ == prog_if } @@ -367,6 +387,19 @@ pub fn register_class_driver( }); } +pub fn register_vendor_driver( + name: &'static str, + vendor_id: u16, + device_id: u16, + probe: fn(&PciDeviceInfo) -> Result<&'static dyn Device, Error>, +) { + PCI_DRIVERS.lock().push(PciDriver { + name, + check: PciMatch::Vendor(vendor_id, device_id), + probe, + }); +} + pub fn register_generic_driver( name: &'static str, check: fn(&PciDeviceInfo) -> bool, diff --git a/driver/bus/pci/src/space/mod.rs b/driver/bus/pci/src/space/mod.rs index 182f911f..206d5326 100644 --- a/driver/bus/pci/src/space/mod.rs +++ b/driver/bus/pci/src/space/mod.rs @@ -79,12 +79,13 @@ pub struct CapabilityIterator<'s, S: PciConfigurationSpace + ?Sized> { } impl<'s, S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'s, S> { - type Item = (PciCapabilityId, usize); + type Item = (PciCapabilityId, usize, usize); fn next(&mut self) -> Option { let offset = self.current? & !0x3; let id = unsafe { core::mem::transmute(self.space.read_u8(offset)) }; + let len = self.space.read_u8(offset + 2); let next_pointer = self.space.read_u8(offset + 1); self.current = if next_pointer != 0 { @@ -93,7 +94,7 @@ impl<'s, S: PciConfigurationSpace + ?Sized> Iterator for CapabilityIterator<'s, None }; - Some((id, offset)) + Some((id, offset, len as usize)) } } @@ -301,9 +302,9 @@ pub trait PciConfigurationSpace { /// Locates a capability within this configuration space fn capability(&self) -> Option> { - self.capability_iter().find_map(|(id, offset)| { - if id == C::ID { - Some(C::data(self, offset)) + self.capability_iter().find_map(|(id, offset, len)| { + if id == C::ID && C::check(self, offset, len) { + Some(C::data(self, offset, len)) } else { None } diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 1adc7203..7833f6aa 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -1,9 +1,15 @@ //! Device virtual file system -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::{ + net::IpAddr, + sync::atomic::{AtomicUsize, Ordering}, +}; -use alloc::{format, string::String}; +use alloc::{format, string::String, sync::Arc}; use kernel_util::util::OneTimeInit; -use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; +use vfs::{ + impls::{mdir, FnValueNode, MemoryDirectory, ReadOnlyFnValueNode}, + CharDevice, Node, NodeFlags, NodeRef, +}; use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; @@ -16,11 +22,28 @@ pub enum CharDeviceType { TtySerial, } +/// Describes the kind of a network device +#[derive(Debug)] +pub enum NetworkDeviceType { + /// Ethernet device + Ethernet, +} + +pub trait NetworkConfigInterface: Send + Sync { + fn address(&self) -> Result; + fn set_address(&self, addr: IpAddr) -> Result<(), Error>; + + fn mac(&self) -> [u8; 6]; +} + static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); +static NET_ROOT: OneTimeInit = OneTimeInit::new(); /// Sets up the device filesystem pub fn init() { - let root = MemoryDirectory::empty(); + let net = MemoryDirectory::empty(); + NET_ROOT.init(net.clone()); + let root = mdir([("net", net)]); DEVFS_ROOT.init(root); } @@ -55,6 +78,59 @@ pub fn add_named_block_device>( DEVFS_ROOT.get().add_child(name, node) } +pub fn add_network_config>(name: S, node: NodeRef) -> Result<(), Error> { + let name = name.into(); + NET_ROOT.get().add_child(name, node) +} + +pub fn add_named_network_device>( + name: S, + dev: Arc, +) -> Result<(), Error> { + let name = name.into(); + log::info!("Add network device: {}", name); + + let read_dev = dev.clone(); + let write_dev = dev.clone(); + let address = FnValueNode::new( + move || read_dev.address(), + move |value| write_dev.set_address(value), + ); + + let read_dev = dev.clone(); + let mac = ReadOnlyFnValueNode::new(move || { + let bytes = read_dev.mac(); + let mut out = String::new(); + for (i, byte) in bytes.into_iter().enumerate() { + if i != 0 { + out.push(':'); + } + out.push_str(&format!("{:02X}", byte)); + } + Ok(out) + }); + + let dev_dir = mdir([("address", address), ("mac", mac)]); + + NET_ROOT.get().add_child(name, dev_dir) +} + +pub fn add_network_device( + dev: Arc, + kind: NetworkDeviceType, +) -> Result<(), Error> { + static ETH_COUNT: AtomicUsize = AtomicUsize::new(0); + + let (count, prefix) = match kind { + NetworkDeviceType::Ethernet => (Ð_COUNT, "eth"), + }; + + let value = count.fetch_add(1, Ordering::AcqRel); + let name = format!("{}{}", prefix, value); + + add_named_network_device(name, dev) +} + pub fn add_block_device_partition>( base_name: S, index: usize, diff --git a/driver/net/core/Cargo.toml b/driver/net/core/Cargo.toml new file mode 100644 index 00000000..7c6be1f8 --- /dev/null +++ b/driver/net/core/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ygg_driver_net_core" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } +vfs = { path = "../../../lib/vfs" } + +kernel-fs = { path = "../../fs/kernel-fs" } + +log = "0.4.20" +bytemuck = { version = "1.14.0", features = ["derive"] } diff --git a/driver/net/core/src/ethernet.rs b/driver/net/core/src/ethernet.rs new file mode 100644 index 00000000..c7653dab --- /dev/null +++ b/driver/net/core/src/ethernet.rs @@ -0,0 +1,100 @@ +use core::{fmt, mem::size_of}; + +use bytemuck::{Pod, Zeroable}; +use kernel_util::mem::PageBox; +use yggdrasil_abi::error::Error; + +use crate::{ + interface::{self, NetworkInterface}, + l3, + types::{MacAddress, NetValue, Value}, +}; + +pub struct L2Packet { + pub interface_id: u32, + + pub source_address: MacAddress, + pub destination_address: MacAddress, + + pub l2_offset: usize, + pub l3_offset: usize, + + pub data: PageBox<[u8]>, +} + +impl L2Packet { + pub fn ethernet_frame(&self) -> &EthernetFrame { + bytemuck::from_bytes( + &self.data[self.l2_offset..self.l2_offset + size_of::()], + ) + } + + pub fn l2_data(&self) -> &[u8] { + &self.data[self.l3_offset..] + } +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C)] +pub struct EthernetFrame { + pub destination_mac: MacAddress, + pub source_mac: MacAddress, + pub ethertype: Value, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Pod, Zeroable)] +#[repr(transparent)] +pub struct EtherType(u16); + +impl EtherType { + pub const ARP: Self = Self(0x0806); + pub const IPV4: Self = Self(0x0800); +} + +impl NetValue for EtherType { + fn to_network_order(self) -> Value { + Value(Self(self.0.to_be())) + } + + fn from_network_order(value: Value) -> Self { + Self(u16::from_be(value.0 .0)) + } +} + +pub fn send_l2( + interface: &NetworkInterface, + source_mac: MacAddress, + destination_mac: MacAddress, + ethertype: EtherType, + l2_data: &T, +) -> Result<(), Error> { + let l2_frame = EthernetFrame { + source_mac, + destination_mac, + ethertype: ethertype.to_network_order(), + }; + + log::debug!( + "send_l2: {} -> {}", + l2_frame.source_mac, + l2_frame.destination_mac + ); + + interface.send_l2(&l2_frame, bytemuck::bytes_of(l2_data)) +} + +pub fn handle(packet: L2Packet) { + let frame = packet.ethernet_frame(); + let ty = EtherType::from_network_order(frame.ethertype); + + match ty { + EtherType::ARP => l3::arp::handle_packet(packet), + EtherType::IPV4 => l3::ip::handle_v4_packet(packet), + p => { + log::debug!( + "Unrecognized L2 protocol: {:#06x}", + bytemuck::cast::<_, u16>(p) + ); + } + } +} diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs new file mode 100644 index 00000000..4a843119 --- /dev/null +++ b/driver/net/core/src/interface.rs @@ -0,0 +1,116 @@ +use core::{ + mem::size_of, + net::{IpAddr, Ipv4Addr}, + sync::atomic::{AtomicU32, Ordering}, +}; + +use alloc::{collections::BTreeMap, sync::Arc, vec}; +use bytemuck::Pod; +use kernel_fs::devfs::{self, NetworkConfigInterface, NetworkDeviceType}; +// TODO: link state management? +use kernel_util::{ + mem::PageBox, + sync::{mutex::Mutex, spin_rwlock::IrqSafeRwLock}, +}; +use yggdrasil_abi::error::Error; + +use crate::{ + ethernet::EthernetFrame, + l3::{self, arp::ARP_TABLE, IpFrame, Route}, + types::{MacAddress, SubnetAddress, SubnetV4Address}, +}; + +pub trait NetworkDevice: Sync { + fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error>; + fn packet_prefix_size(&self) -> usize; + + fn read_hardware_address(&self) -> MacAddress; +} + +pub struct NetworkInterface { + pub(crate) device: &'static dyn NetworkDevice, + pub(crate) mac: MacAddress, + + pub(crate) address: IrqSafeRwLock>, + pub(crate) id: u32, +} + +impl NetworkConfigInterface for NetworkInterface { + fn address(&self) -> Result { + if let Some(address) = self.address.read().as_ref() { + Ok(*address) + } else { + Err(Error::InvalidOperation) + } + } + + fn set_address(&self, addr: IpAddr) -> Result<(), Error> { + // Remove old ARP entry, if exists + ARP_TABLE.write().remove_ip(self.id, addr); + *self.address.write() = Some(addr); + ARP_TABLE.write().insert_ip(self.id, self.mac, addr, true); + match addr { + IpAddr::V4(v4) => { + l3::add_route(Route { + subnet: SubnetAddress::V4(SubnetV4Address::from_address_mask(v4, 24)), + interface: self.id, + // TODO manual configuration for this + gateway: Some(IpAddr::V4(Ipv4Addr::new(11, 0, 0, 1))), + }); + } + IpAddr::V6(_) => todo!(), + } + Ok(()) + } + + fn mac(&self) -> [u8; 6] { + self.mac.into() + } +} + +static ETH_INTERFACES: IrqSafeRwLock>> = + IrqSafeRwLock::new(BTreeMap::new()); +static ETH_INTERFACE_ID: AtomicU32 = AtomicU32::new(1); + +impl NetworkInterface { + pub fn get(id: u32) -> Result, Error> { + ETH_INTERFACES + .read() + .get(&id) + .cloned() + .ok_or(Error::DoesNotExist) + } + + pub fn send_l2(&self, l2_frame: &EthernetFrame, l2_data: &[u8]) -> Result<(), Error> { + let l2_offset = self.device.packet_prefix_size(); + let l2_data_offset = l2_offset + size_of::(); + + let mut packet = PageBox::new_slice(0, l2_data_offset + l2_data.len())?; + + packet[l2_offset..l2_data_offset].copy_from_slice(bytemuck::bytes_of(l2_frame)); + packet[l2_data_offset..].copy_from_slice(l2_data); + + self.device.transmit(packet) + } +} + +pub fn register_interface(dev: &'static dyn NetworkDevice) -> u32 { + let mac = dev.read_hardware_address(); + + let id = ETH_INTERFACE_ID.fetch_add(1, Ordering::SeqCst); + + let iface = NetworkInterface { + device: dev, + mac, + address: IrqSafeRwLock::new(None), + id, + }; + log::info!("Registered network interface #{}: {}", id, mac); + + let interface = Arc::new(iface); + + devfs::add_network_device(interface.clone(), NetworkDeviceType::Ethernet).unwrap(); + ETH_INTERFACES.write().insert(id, interface); + + id +} diff --git a/driver/net/core/src/l3/arp.rs b/driver/net/core/src/l3/arp.rs new file mode 100644 index 00000000..b89a1656 --- /dev/null +++ b/driver/net/core/src/l3/arp.rs @@ -0,0 +1,142 @@ +use core::{ + borrow::Borrow, + mem::size_of, + net::{IpAddr, Ipv4Addr}, +}; + +use alloc::{collections::BTreeMap, sync::Arc}; +use bytemuck::{Pod, Zeroable}; +use kernel_util::{mem::PageBox, sync::spin_rwlock::IrqSafeRwLock}; +use yggdrasil_abi::error::Error; + +use crate::{ + ethernet::{self, EtherType, EthernetFrame}, + interface::NetworkInterface, + types::{MacAddress, NetValue, Value}, + L2Packet, +}; + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, packed)] +struct ArpFrame { + pub hardware_type: Value, + pub protocol: Value, + pub hardware_size: u8, + pub protocol_size: u8, + pub opcode: Value, + pub sender_mac: MacAddress, + pub sender_ip: Value, + pub target_mac: MacAddress, + // TODO handle IPv6 + pub target_ip: Value, +} + +pub struct ArpTable { + entries_v4: BTreeMap<(u32, Ipv4Addr), (MacAddress, bool)>, + reverse_v4: BTreeMap<(u32, MacAddress), (Ipv4Addr, bool)>, +} + +impl ArpTable { + pub const fn new() -> Self { + Self { + reverse_v4: BTreeMap::new(), + entries_v4: BTreeMap::new(), + } + } + + // TODO multiple IPs per MAC? + pub fn insert_v4(&mut self, interface: u32, mac: MacAddress, ip: Ipv4Addr, own: bool) { + if !self.reverse_v4.contains_key(&(interface, mac)) { + log::debug!("Insert {} <-> {}", mac, ip); + } + + self.reverse_v4.insert((interface, mac), (ip, own)); + self.entries_v4.insert((interface, ip), (mac, own)); + } + + pub fn insert_ip(&mut self, interface: u32, mac: MacAddress, ip: IpAddr, own: bool) { + match ip { + IpAddr::V4(v4) => self.insert_v4(interface, mac, v4, own), + IpAddr::V6(_) => todo!(), + } + } + + pub fn remove_ip(&mut self, interface: u32, ip: IpAddr) { + match ip { + IpAddr::V4(v4) => { + let entry = self.entries_v4.remove(&(interface, v4)); + if let Some((mac, _)) = entry { + self.reverse_v4.remove(&(interface, mac)).unwrap(); + } + } + IpAddr::V6(_) => todo!(), + } + } + + pub fn get_v4(&self, interface: u32, ip: Ipv4Addr) -> Option<(MacAddress, bool)> { + self.entries_v4.get(&(interface, ip)).cloned() + } +} + +pub(crate) static ARP_TABLE: IrqSafeRwLock = IrqSafeRwLock::new(ArpTable::new()); + +pub fn lookup(interface: u32, ip: IpAddr) -> Option { + // TODO send a request if MAC is not yet in the table + match ip { + IpAddr::V4(v4) => Some(ARP_TABLE.read().get_v4(interface, v4)?.0), + IpAddr::V6(_) => todo!(), + } +} + +fn send_reply(interface_id: u32, arp: &ArpFrame, target_mac: MacAddress) -> Result<(), Error> { + let interface = NetworkInterface::get(interface_id)?; + let reply = ArpFrame { + protocol: arp.protocol, + hardware_type: arp.hardware_type, + hardware_size: arp.hardware_size, + protocol_size: arp.protocol_size, + opcode: 2u16.to_network_order(), + + sender_mac: target_mac, + sender_ip: arp.target_ip, + + target_ip: arp.sender_ip, + target_mac: arp.sender_mac, + }; + + ethernet::send_l2( + &interface, + target_mac, + arp.sender_mac, + EtherType::ARP, + &reply, + ) +} + +pub fn handle_packet(packet: L2Packet) { + let arp: &ArpFrame = bytemuck::from_bytes(&packet.l2_data()[..size_of::()]); + let proto = EtherType::from_network_order(arp.protocol); + let opcode = u16::from_network_order(arp.opcode); + + let (target_address, sender_address) = match proto { + EtherType::IPV4 => ( + Ipv4Addr::from(u32::from_network_order(arp.target_ip)), + Ipv4Addr::from(u32::from_network_order(arp.sender_ip)), + ), + _ => { + log::warn!("TODO: unhandled ARP proto: {:#x?}", proto); + return; + } + }; + + let mut table = ARP_TABLE.write(); + table.insert_v4(packet.interface_id, arp.sender_mac, sender_address, false); + + if opcode == 1 { + // Don't answer with non-owned addresses + if let Some((mac, true)) = table.get_v4(packet.interface_id, target_address) { + // Reply with own address + send_reply(packet.interface_id, arp, mac).ok(); + } + } +} diff --git a/driver/net/core/src/l3/ip.rs b/driver/net/core/src/l3/ip.rs new file mode 100644 index 00000000..88945bb9 --- /dev/null +++ b/driver/net/core/src/l3/ip.rs @@ -0,0 +1,148 @@ +use core::{ + fmt, + mem::size_of, + net::{IpAddr, Ipv4Addr}, +}; + +use alloc::sync::Arc; +use bytemuck::{Pod, Zeroable}; +use yggdrasil_abi::error::Error; + +use crate::{ + ethernet::EthernetFrame, + interface::NetworkInterface, + l3, + l4::udp, + types::{InetChecksum, NetValue, Value}, + L2Packet, L3Packet, ACCEPT_QUEUE, +}; + +use super::IpFrame; + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C)] +pub struct Ipv4Frame { + pub version_length: u8, + pub dscp_flags: u8, + pub total_length: Value, + pub id: Value, + pub flags_frag: Value, + pub ttl: u8, + pub protocol: Protocol, + pub header_checksum: Value, + pub source_address: Value, + pub destination_address: Value, +} + +#[derive(Clone, Copy, PartialEq, Eq, Pod, Zeroable)] +#[repr(transparent)] +pub struct Protocol(pub u8); + +impl Protocol { + pub const ICMP: Self = Self(1); + pub const UDP: Self = Self(17); +} + +impl fmt::Debug for Protocol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Self::ICMP => f.write_str("ICMP"), + Self::UDP => f.write_str("UDP"), + _ => f.write_str(""), + } + } +} + +impl Ipv4Frame { + fn header_length(&self) -> usize { + core::cmp::min( + (self.version_length & 0xF) << 2, + size_of::() as u8, + ) as usize + } + + fn total_length(&self) -> usize { + u16::from_network_order(self.total_length) as usize + } +} + +impl IpFrame for Ipv4Frame { + fn destination_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::from(u32::from_network_order( + self.destination_address, + ))) + } + + fn source_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::from(u32::from_network_order(self.source_address))) + } + + fn data_length(&self) -> usize { + self.total_length() + .checked_sub(self.header_length()) + .unwrap_or(0) + } +} + +pub fn handle_v4_packet(packet: L2Packet) { + let Ok(interface) = NetworkInterface::get(packet.interface_id) else { + log::debug!("Invalid interface ID in L2 packet"); + return; + }; + + let l2_data = packet.l2_data(); + let l3_frame: &Ipv4Frame = bytemuck::from_bytes(&l2_data[..size_of::()]); + let header_length = l3_frame.header_length(); + let l3_data = &l2_data[size_of::()..]; + + let is_input = interface + .address + .read() + .map(|address| address == l3_frame.destination_ip()) + .unwrap_or(false); + + if is_input { + // Extract ports from L4 proto + let (source_port, destination_port) = match l3_frame.protocol { + Protocol::UDP => { + // TODO check size + let l4_frame: &udp::UdpFrame = + bytemuck::from_bytes(&l3_data[..size_of::()]); + ( + Some(u16::from_network_order(l4_frame.source_port)), + Some(u16::from_network_order(l4_frame.destination_port)), + ) + } + Protocol::ICMP => (None, None), + _ => (None, None), + }; + + let l3_packet = L3Packet { + interface_id: packet.interface_id, + + protocol: l3_frame.protocol, + + source_address: l3_frame.source_ip(), + destination_address: l3_frame.destination_ip(), + + source_port, + destination_port, + + l2_offset: packet.l2_offset, + l3_offset: packet.l3_offset, + l4_offset: packet.l3_offset + header_length, + data_length: l3_frame.data_length(), + + data: packet.data, + }; + + ACCEPT_QUEUE.receive_packet(l3_packet).ok(); + } else { + // TODO forwarding + log::debug!( + "Dropped forwarded IPv4: {} -> {}", + l3_frame.source_ip(), + l3_frame.destination_ip() + ); + } +} diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs new file mode 100644 index 00000000..321b93b4 --- /dev/null +++ b/driver/net/core/src/l3/mod.rs @@ -0,0 +1,187 @@ +use core::{ + fmt, + mem::size_of, + net::{IpAddr, Ipv4Addr}, + str::FromStr, +}; + +use alloc::{sync::Arc, vec::Vec}; +use bytemuck::Pod; +use kernel_util::{mem::PageBox, sync::spin_rwlock::IrqSafeRwLock}; +use yggdrasil_abi::error::Error; + +use crate::{ + ethernet::{EtherType, EthernetFrame}, + interface::NetworkInterface, + l3::ip::Ipv4Frame, + l4, + types::{InetChecksum, NetValue, SubnetAddress}, +}; + +use self::ip::Protocol; + +pub mod arp; +pub mod ip; + +pub struct L3Packet { + pub interface_id: u32, + + pub protocol: Protocol, + + pub source_address: IpAddr, + pub destination_address: IpAddr, + + pub source_port: Option, + pub destination_port: Option, + + pub l2_offset: usize, + pub l3_offset: usize, + pub l4_offset: usize, + pub data_length: usize, + + pub data: PageBox<[u8]>, +} + +pub trait IpFrame: Pod { + fn destination_ip(&self) -> IpAddr; + fn source_ip(&self) -> IpAddr; + fn data_length(&self) -> usize; +} + +// TODO use range map for this? +pub struct Route { + pub subnet: SubnetAddress, + pub interface: u32, + pub gateway: Option, +} + +static ROUTES: IrqSafeRwLock> = IrqSafeRwLock::new(Vec::new()); + +impl L3Packet { + pub fn l3_data(&self) -> &[u8] { + &self.data[self.l4_offset..] + } +} + +pub fn lookup_route(address: IpAddr) -> Option<(u32, Option)> { + let routes = ROUTES.read(); + for route in routes.iter() { + if route.subnet.contains(&address) { + return Some((route.interface, route.gateway)); + } + } + None +} + +pub fn list_routes(mut f: F) { + let routes = ROUTES.read(); + for route in routes.iter() { + f(route); + } +} + +pub fn add_route(route: Route) -> Result<(), Error> { + // TODO check for conflicts + log::debug!("Add route: {}", route); + ROUTES.write().push(route); + Ok(()) +} + +impl fmt::Display for Route { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} ", self.subnet)?; + if let Some(gw) = self.gateway { + write!(f, " via {}", gw)?; + } + Ok(()) + } +} + +pub fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { + match l3_packet.protocol { + Protocol::UDP => l4::udp::handle(l3_packet), + Protocol::ICMP => l4::icmp::handle(l3_packet), + _ => todo!(), + } +} + +pub fn send_l4_ip( + destination_ip: IpAddr, + protocol: ip::Protocol, + l4_frame: &L4, + l4_data: &[u8], +) -> Result<(), Error> { + send_l4_ip_opt(destination_ip, protocol, l4_frame, &[], l4_data) +} + +pub fn send_l4_ip_opt( + destination_ip: IpAddr, + protocol: ip::Protocol, + l4_frame: &L4, + l4_options: &[u8], + l4_data: &[u8], +) -> Result<(), Error> { + let IpAddr::V4(destination_v4) = destination_ip else { + log::debug!("Destination: {}", destination_ip); + todo!(); + }; + + // Lookup route to destination + let (iface_id, gateway) = lookup_route(destination_ip).unwrap(); + let iface = NetworkInterface::get(iface_id).unwrap(); + + let Some(IpAddr::V4(gateway_v4)) = gateway else { + todo!(); + }; + let Some(IpAddr::V4(source_v4)) = *iface.address.read() else { + todo!(); + }; + + // Lookup gateway MAC + let gateway_mac = arp::lookup(iface.id, IpAddr::V4(gateway_v4)).unwrap(); + + let l2_offset = iface.device.packet_prefix_size(); + let l3_offset = l2_offset + size_of::(); + let l4_offset = l3_offset + size_of::(); + let l4_data_offset = l4_offset + size_of::(); + + let mut packet = PageBox::new_slice(0, l4_data_offset + l4_data.len())?; + + let l2_frame: &mut EthernetFrame = bytemuck::from_bytes_mut(&mut packet[l2_offset..l3_offset]); + + l2_frame.source_mac = iface.mac; + l2_frame.destination_mac = gateway_mac; + l2_frame.ethertype = EtherType::IPV4.to_network_order(); + + let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); + + l3_frame.source_address = u32::to_network_order(u32::from(source_v4)); + l3_frame.destination_address = u32::to_network_order(u32::from(destination_v4)); + l3_frame.protocol = protocol; + l3_frame.version_length = 0x45; + l3_frame.dscp_flags = 0; + l3_frame.total_length = u16::to_network_order( + (size_of::() + size_of::() + l4_data.len()) + .try_into() + .unwrap(), + ); + // Disable fragmentation + l3_frame.flags_frag = u16::to_network_order(0x4000); + l3_frame.id = u16::to_network_order(0); + l3_frame.header_checksum = u16::to_network_order(0); + l3_frame.ttl = 64; + + let l3_frame_bytes = &packet[l3_offset..l4_offset]; + let mut ip_checksum = InetChecksum::new(); + ip_checksum.add_bytes(l3_frame_bytes, true); + let ip_checksum = ip_checksum.finish(); + + let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); + l3_frame.header_checksum = u16::to_network_order(ip_checksum); + + packet[l4_offset..l4_data_offset].copy_from_slice(bytemuck::bytes_of(l4_frame)); + packet[l4_data_offset..l4_data_offset + l4_options.len()].copy_from_slice(l4_options); + packet[l4_data_offset + l4_options.len()..].copy_from_slice(l4_data); + + iface.device.transmit(packet) +} diff --git a/driver/net/core/src/l4/icmp.rs b/driver/net/core/src/l4/icmp.rs new file mode 100644 index 00000000..d4a0c95c --- /dev/null +++ b/driver/net/core/src/l4/icmp.rs @@ -0,0 +1,104 @@ +use core::{ + mem::size_of, + net::{IpAddr, Ipv4Addr}, +}; + +use bytemuck::{Pod, Zeroable}; +use yggdrasil_abi::error::Error; + +use crate::{ + l3::{self, ip::Protocol}, + types::{InetChecksum, NetValue, Value}, + L3Packet, +}; + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C)] +pub struct IcmpV4Frame { + ty: u8, + code: u8, + checksum: Value, + rest: Value, +} + +fn send_v4_reply( + destination_ip: Ipv4Addr, + icmp_frame: &IcmpV4Frame, + icmp_data: &[u8], +) -> Result<(), Error> { + let mut reply_frame = IcmpV4Frame { + ty: 0, + code: 0, + checksum: u16::to_network_order(0), + rest: icmp_frame.rest, + }; + + if icmp_data.len() % 2 != 0 { + todo!(); + } + + let l4_bytes = bytemuck::bytes_of(&reply_frame); + let mut checksum = InetChecksum::new(); + checksum.add_bytes(l4_bytes, true); + checksum.add_bytes(icmp_data, true); + + reply_frame.checksum = checksum.finish().to_network_order(); + + l3::send_l4_ip( + IpAddr::V4(destination_ip), + Protocol::ICMP, + &reply_frame, + icmp_data, + ) +} + +fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), Error> { + if l3_packet.data_length < size_of::() { + log::debug!("Truncated ICMPv4 packet"); + return Err(Error::MissingData); + } + + if l3_packet.data_length - size_of::() > 576 { + log::debug!("ICMPv4 packet too large"); + return Err(Error::MissingData); + } + + let l3_data = l3_packet.l3_data(); + let icmp_frame: &IcmpV4Frame = bytemuck::from_bytes(&l3_data[..size_of::()]); + let icmp_data = &l3_data[size_of::()..l3_packet.data_length]; + + match (icmp_frame.ty, icmp_frame.code) { + (8, 0) => send_v4_reply(source_address, icmp_frame, icmp_data), + _ => { + log::debug!( + "Ignoring unknown ICMPv4 type:code: {}:{}", + icmp_frame.ty, + icmp_frame.code + ); + return Ok(()); + } + } +} + +pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { + match l3_packet.source_address { + IpAddr::V4(v4) => handle_v4(v4, l3_packet), + IpAddr::V6(_) => todo!(), + } +} + +// fn send_icmp_v4_reply( +// l3_frame: &Ipv4Frame, +// icmp_frame: &IcmpV4Frame, +// icmp_data: &[u8], +// ) -> Result<(), Error> { +// +// l3::send_l4_ip(l3_frame.source_ip(), Protocol::ICMP, &l4_frame, icmp_data) +// } +// +// pub fn handle_icmp_v4_packet( +// interface: Arc, +// l3_frame: &Ipv4Frame, +// l3_data: &[u8], +// ) { +// } diff --git a/driver/net/core/src/l4/mod.rs b/driver/net/core/src/l4/mod.rs new file mode 100644 index 00000000..2b647e5c --- /dev/null +++ b/driver/net/core/src/l4/mod.rs @@ -0,0 +1,2 @@ +pub mod icmp; +pub mod udp; diff --git a/driver/net/core/src/l4/udp.rs b/driver/net/core/src/l4/udp.rs new file mode 100644 index 00000000..66b2beeb --- /dev/null +++ b/driver/net/core/src/l4/udp.rs @@ -0,0 +1,79 @@ +use core::{ + mem::size_of, + net::{IpAddr, SocketAddr}, +}; + +use bytemuck::{Pod, Zeroable}; +use yggdrasil_abi::error::Error; + +use crate::{ + l3::{self, ip::Protocol, IpFrame}, + socket::{self, UdpSocket}, + types::{NetValue, Value}, + L3Packet, +}; + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub struct UdpFrame { + pub source_port: Value, + pub destination_port: Value, + pub length: Value, + pub checksum: Value, +} + +impl UdpFrame { + fn data_length(&self) -> usize { + (u16::from_network_order(self.length) as usize) + .checked_sub(0) + .unwrap_or(0) + } +} + +pub fn send( + source_port: u16, + destination_ip: IpAddr, + destination_port: u16, + data: &[u8], +) -> Result<(), Error> { + let length: u16 = (data.len() + size_of::()).try_into().unwrap(); + let udp_frame = UdpFrame { + source_port: source_port.to_network_order(), + destination_port: destination_port.to_network_order(), + length: length.to_network_order(), + checksum: 0u16.to_network_order(), + }; + + l3::send_l4_ip(destination_ip, Protocol::UDP, &udp_frame, data) +} + +pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { + if l3_packet.data_length < size_of::() { + log::warn!("Truncated UDP frame received"); + return Err(Error::MissingData); + } + + let l3_data = l3_packet.l3_data(); + + let udp_frame: &UdpFrame = bytemuck::from_bytes(&l3_data[..size_of::()]); + let data_size = core::cmp::min( + udp_frame.data_length(), + l3_packet.data_length - size_of::(), + ); + let udp_data = &l3_data[size_of::()..data_size + size_of::()]; + + let source = SocketAddr::new( + l3_packet.source_address, + u16::from_network_order(udp_frame.source_port), + ); + let destination = SocketAddr::new( + l3_packet.destination_address, + u16::from_network_order(udp_frame.destination_port), + ); + + if let Some(socket) = UdpSocket::get(&destination) { + socket.packet_received(source, udp_data); + } + + Ok(()) +} diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs new file mode 100644 index 00000000..bce65a9a --- /dev/null +++ b/driver/net/core/src/lib.rs @@ -0,0 +1,139 @@ +#![feature(map_try_insert)] +#![no_std] + +extern crate alloc; + +use core::{ + fmt, + future::Future, + mem::size_of, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, + pin::Pin, + str::FromStr, + sync::atomic::{AtomicU32, Ordering}, + task::{Context, Poll}, +}; + +use alloc::{collections::BTreeMap, format, string::String, vec::Vec}; +use ethernet::{EtherType, EthernetFrame, L2Packet}; +use interface::{NetworkDevice, NetworkInterface}; +use kernel_fs::devfs; +use kernel_util::{ + mem::PageBox, + runtime::{self, QueueWaker}, + sync::{mutex::Mutex, spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + util::ring::RingBuffer, +}; +use l3::{ip::Protocol, L3Packet, Route}; +use queue::Queue; +use socket::UdpSocket; +use types::{MacAddress, NetValue, SubnetAddress, Value}; +use vfs::{impls::FnValueNode, PacketSocket}; +use yggdrasil_abi::error::Error; + +use crate::types::SubnetV4Address; + +pub mod ethernet; +pub mod l3; +pub mod l4; + +pub mod socket; + +pub mod interface; +pub mod queue; +pub mod types; + +pub use interface::register_interface; + +pub struct Packet { + // TODO info about "received" interface + buffer: PageBox<[u8]>, + offset: usize, + iface: u32, +} + +impl Packet { + #[inline] + pub fn new(buffer: PageBox<[u8]>, offset: usize, iface: u32) -> Self { + Self { + buffer, + offset, + iface, + } + } +} + +static PACKET_QUEUE: Queue = Queue::new(); +static ACCEPT_QUEUE: Queue = Queue::new(); + +#[inline] +pub fn receive_packet(packet: Packet) -> Result<(), Error> { + PACKET_QUEUE.receive_packet(packet) +} + +pub fn start_network_tasks() -> Result<(), Error> { + devfs::add_network_config( + "routes", + FnValueNode::new( + || { + let mut output = String::new(); + l3::list_routes(|route| { + output.push_str(&format!("{}\n", route)); + }); + + Ok(output) + }, + |_value| Err(Error::NotImplemented), + ), + ); + + runtime::spawn(l2_packet_handler_worker())?; + for _ in 0..1 { + runtime::spawn(l3_accept_worker())?; + } + + Ok(()) +} + +async fn l2_packet_handler_worker() { + loop { + let packet = PACKET_QUEUE.wait().await; + + let eth_frame: &EthernetFrame = bytemuck::from_bytes( + &packet.buffer[packet.offset..packet.offset + size_of::()], + ); + + let l2_packet = L2Packet { + interface_id: packet.iface, + + source_address: eth_frame.source_mac, + destination_address: eth_frame.destination_mac, + + l2_offset: packet.offset, + l3_offset: packet.offset + size_of::(), + + data: packet.buffer, + }; + + ethernet::handle(l2_packet); + } +} + +async fn l3_accept_worker() { + loop { + let l3_packet = ACCEPT_QUEUE.wait().await; + + log::debug!( + "INPUT {:?} {}:{:?} -> {}:{:?}: ACCEPT", + l3_packet.protocol, + l3_packet.source_address, + l3_packet.source_port, + l3_packet.destination_address, + l3_packet.destination_port + ); + + if let Err(error) = l3::handle_accepted(l3_packet) { + log::error!("L3 handle error: {:?}", error); + } + } +} diff --git a/driver/net/core/src/queue.rs b/driver/net/core/src/queue.rs new file mode 100644 index 00000000..759754ed --- /dev/null +++ b/driver/net/core/src/queue.rs @@ -0,0 +1,53 @@ +use core::{ + future::Future, + pin::Pin, + task::{Context, Poll}, +}; + +use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBuffer}; +use yggdrasil_abi::error::Error; + +pub struct Queue { + queue: IrqSafeSpinlock>, + notify: QueueWaker, +} + +impl Queue { + pub const fn new() -> Self { + Self { + queue: IrqSafeSpinlock::new(RingBuffer::with_capacity(1024)), + notify: QueueWaker::new(), + } + } + + pub fn receive_packet(&self, packet: T) -> Result<(), Error> { + self.queue.lock().write(packet); + self.notify.wake_one(); + // TODO notify of dropped packets + Ok(()) + } + + pub fn wait(&self) -> impl Future + '_ { + struct F<'f, T> { + queue: &'f Queue, + } + + impl<'f, T> Future for F<'f, T> { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.queue.notify.register(cx.waker()); + let mut lock = self.queue.queue.lock(); + + if lock.is_readable() { + self.queue.notify.remove(cx.waker()); + Poll::Ready(unsafe { lock.read_single_unchecked() }) + } else { + Poll::Pending + } + } + } + + F { queue: self } + } +} diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs new file mode 100644 index 00000000..fe851462 --- /dev/null +++ b/driver/net/core/src/socket.rs @@ -0,0 +1,198 @@ +use core::{ + future::Future, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, + pin::Pin, + sync::atomic::{AtomicBool, Ordering}, + task::{Context, Poll}, +}; + +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; +use kernel_util::{ + block, + runtime::QueueWaker, + sync::{ + mutex::{Mutex, MutexGuard}, + spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, + IrqSafeSpinlock, LockMethod, + }, + util::ring::RingBuffer, +}; +use vfs::{FileReadiness, PacketSocket, Socket}; +use yggdrasil_abi::error::Error; + +use crate::{l4, queue::Queue}; + +pub struct UdpSocket { + local: SocketAddr, + remote: Option, + // TODO just place packets here for one less copy? + receive_queue: Mutex)>>, + receive_notify: QueueWaker, +} + +pub struct SocketTable { + inner: BTreeMap>, +} + +impl SocketTable { + pub const fn new() -> Self { + Self { + inner: BTreeMap::new(), + } + } + + pub fn insert(&mut self, address: SocketAddr, socket: Arc) -> Result<(), Error> { + match self.inner.try_insert(address, socket) { + Ok(_) => Ok(()), + Err(_) => return Err(Error::AlreadyExists), + } + } + + pub fn remove(&mut self, local: SocketAddr) -> Result<(), Error> { + match self.inner.remove(&local) { + Some(_) => Ok(()), + None => Err(Error::DoesNotExist), + } + } + + pub fn get_exact(&self, local: &SocketAddr) -> Option> { + self.inner.get(local).cloned() + } + + pub fn get(&self, local: &SocketAddr) -> Option> { + if let Some(socket) = self.inner.get(local) { + return Some(socket.clone()); + } + + match local { + SocketAddr::V4(v4) => { + let unspec_v4 = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, local.port()); + self.inner.get(&SocketAddr::V4(unspec_v4)).cloned() + } + SocketAddr::V6(_) => todo!(), + } + } +} + +static UDP_SOCKETS: IrqSafeRwLock> = IrqSafeRwLock::new(SocketTable::new()); + +impl UdpSocket { + pub fn bind(address: SocketAddr) -> Result, Error> { + let mut sockets = UDP_SOCKETS.write(); + + if sockets.get(&address).is_some() { + // TODO use network-specific error + return Err(Error::AlreadyExists); + } + + let socket = Arc::new(Self { + local: address, + remote: None, + receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), + receive_notify: QueueWaker::new(), + }); + + sockets.insert(address, socket.clone()); + + log::debug!("UDP socket opened: {}", address); + + Ok(socket) + } + + pub fn connect(address: SocketAddr) -> Result, Error> { + todo!() + } + + pub fn get(local: &SocketAddr) -> Option> { + UDP_SOCKETS.read().get(local) + } + + fn poll_receive( + &self, + cx: &mut Context<'_>, + ) -> Poll)>>, Error>> { + self.receive_notify.register(cx.waker()); + let mut lock = self.receive_queue.lock()?; + if lock.is_readable() { + self.receive_notify.remove(cx.waker()); + Poll::Ready(Ok(lock)) + } else { + Poll::Pending + } + } + + pub fn receive_raw<'a>( + &'a self, + ) -> impl Future), Error>> + 'a { + struct F<'f> { + socket: &'f UdpSocket, + } + + impl<'f> Future for F<'f> { + type Output = Result<(SocketAddr, Vec), Error>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.socket.poll_receive(cx)? { + Poll::Ready(mut lock) => { + let (source, data) = unsafe { lock.read_single_unchecked() }; + Poll::Ready(Ok((source, data))) + } + Poll::Pending => Poll::Pending, + } + } + } + + F { socket: self } + } + + pub fn packet_received(&self, source: SocketAddr, data: &[u8]) -> Result<(), Error> { + let mut lock = self.receive_queue.lock()?; + lock.write((source, Vec::from(data))); + self.receive_notify.wake_one(); + Ok(()) + } +} + +impl FileReadiness for UdpSocket { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + self.poll_receive(cx).map_ok(|_| ()) + } +} + +impl PacketSocket for UdpSocket { + fn send(&self, destination: SocketAddr, data: &[u8]) -> Result { + // TODO check that destnation family matches self family + l4::udp::send( + self.local.port(), + destination.ip(), + destination.port(), + data, + )?; + + Ok(data.len()) + } + + fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error> { + let (source, data) = block!(self.receive_raw().await)??; + if data.len() > buffer.len() { + todo!() + } + buffer[..data.len()].copy_from_slice(&data); + Ok((source, data.len())) + } +} + +impl Socket for UdpSocket { + fn local_address(&self) -> SocketAddr { + self.local + } + + fn remote_address(&self) -> Option { + self.remote + } + + fn close(&self) -> Result<(), Error> { + log::debug!("UDP socket closed: {}", self.local); + UDP_SOCKETS.write().remove(self.local) + } +} diff --git a/driver/net/core/src/types.rs b/driver/net/core/src/types.rs new file mode 100644 index 00000000..fa439d54 --- /dev/null +++ b/driver/net/core/src/types.rs @@ -0,0 +1,225 @@ +use core::{ + fmt, + net::{IpAddr, Ipv4Addr}, + str::FromStr, +}; + +use bytemuck::{Pod, Zeroable}; +use yggdrasil_abi::error::Error; + +#[macro_export] +macro_rules! wrap_value { + ($vis:vis struct $name:ident($ty_vis:vis $ty:ty)) => { + #[derive(Clone, Copy, Debug, bytemuck::Pod, bytemuck::Zeroable)] + $vis struct $name($ty_vis $ty); + + impl $crate::types::NetValue for $name { + fn from_network_order(value: $crate::types::Value) -> Self { + todo!() + } + + fn to_network_order(self) -> $crate::types::Value { + todo!() + } + } + }; +} + +pub trait NetValue: Copy + Eq + Pod + Zeroable { + fn from_network_order(value: Value) -> Self; + fn to_network_order(self) -> Value; +} + +#[derive(Clone, Copy, PartialEq, Eq, Pod, Zeroable)] +#[repr(transparent)] +pub struct Value(pub T); + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Pod, Zeroable)] +#[repr(transparent)] +pub struct MacAddress([u8; 6]); + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct SubnetV4Address { + pub root: Ipv4Addr, + pub mask_bits: u8, + pub mask: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SubnetAddress { + V4(SubnetV4Address), +} + +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = T::from_network_order(*self); + fmt::Debug::fmt(&value, f) + } +} + +impl NetValue for u16 { + fn from_network_order(value: Value) -> Self { + Self::from_be(value.0) + } + + fn to_network_order(self) -> Value { + Value(self.to_be()) + } +} + +impl NetValue for u32 { + fn from_network_order(value: Value) -> Self { + Self::from_be(value.0) + } + + fn to_network_order(self) -> Value { + Value(self.to_be()) + } +} + +impl fmt::Display for MacAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use fmt::Write; + + for (i, &v) in self.0.iter().enumerate() { + if i != 0 { + write!(f, ":")?; + } + write!(f, "{:02X}", v)?; + } + + Ok(()) + } +} + +impl From for [u8; 6] { + fn from(value: MacAddress) -> Self { + value.0 + } +} + +impl From<[u8; 6]> for MacAddress { + fn from(value: [u8; 6]) -> Self { + Self(value) + } +} + +impl SubnetV4Address { + pub const UNSPECIFIED: Self = Self { + root: Ipv4Addr::new(0, 0, 0, 0), + mask: 0, + mask_bits: 0, + }; + + pub fn from_address_mask(root: Ipv4Addr, mask_bits: u8) -> Self { + let root = u32::from(root); + let mask = Self::make_mask(mask_bits); + // Clear masked bits + let root = root & mask; + + Self { + root: Ipv4Addr::from(root), + mask_bits, + mask, + } + } + + pub fn contains(&self, address: &Ipv4Addr) -> bool { + let root = u32::from(self.root); + let address = u32::from(*address); + + (address & self.mask) == root + } + + fn make_mask(bits: u8) -> u32 { + ((1 << bits) - 1) << (32 - bits) + } +} + +impl fmt::Display for SubnetV4Address { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.root, self.mask_bits) + } +} + +impl FromStr for SubnetV4Address { + type Err = Error; + + fn from_str(s: &str) -> Result { + let (addr, mask) = s.split_once('/').ok_or(Error::InvalidArgument)?; + let addr = Ipv4Addr::from_str(addr).map_err(|_| Error::InvalidArgument)?; + let mask = u8::from_str(mask).map_err(|_| Error::InvalidArgument)?; + + Ok(Self::from_address_mask(addr, mask)) + } +} + +impl SubnetAddress { + pub const UNSPECIFIED_V4: Self = Self::V4(SubnetV4Address::UNSPECIFIED); + + pub fn contains(&self, address: &IpAddr) -> bool { + match (self, address) { + (Self::V4(subnet), IpAddr::V4(address)) => subnet.contains(address), + _ => false, + } + } + + pub fn is_v4(&self) -> bool { + true + } +} + +impl From for SubnetAddress { + fn from(value: SubnetV4Address) -> Self { + Self::V4(value) + } +} + +impl FromStr for SubnetAddress { + type Err = Error; + + fn from_str(s: &str) -> Result { + // TODO v6 + let v4 = SubnetV4Address::from_str(s)?; + Ok(Self::from(v4)) + } +} + +impl fmt::Display for SubnetAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::V4(v4) => fmt::Display::fmt(v4, f), + } + } +} + +pub struct InetChecksum { + state: u32, +} + +impl InetChecksum { + pub fn new() -> Self { + Self { state: 0 } + } + + pub fn add_bytes(&mut self, bytes: &[u8], reorder: bool) { + let len = bytes.len(); + for i in 0..len / 2 { + let word = if reorder { + ((bytes[i * 2] as u16) << 8) | (bytes[i * 2 + 1] as u16) + } else { + (bytes[i * 2] as u16) | ((bytes[i * 2 + 1] as u16) << 8) + }; + self.state = self.state.wrapping_add(word as u32); + } + if len % 2 != 0 { + self.state += self.state.wrapping_add(bytes[len - 1] as u32); + } + } + + pub fn finish(self) -> u16 { + let sum = (self.state >> 16) + (self.state & 0xFFFF); + + (!(sum & 0xFFFF)) as u16 + } +} diff --git a/driver/virtio/core/Cargo.toml b/driver/virtio/core/Cargo.toml new file mode 100644 index 00000000..f2d0bc67 --- /dev/null +++ b/driver/virtio/core/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "ygg_driver_virtio_core" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } + +ygg_driver_pci = { path = "../../bus/pci", optional = true } + +log = "0.4.20" +bitflags = "2.4.2" +tock-registers = "0.8.1" + +[features] +default = [] +pci = ["ygg_driver_pci"] diff --git a/driver/virtio/core/src/error.rs b/driver/virtio/core/src/error.rs new file mode 100644 index 00000000..6538e069 --- /dev/null +++ b/driver/virtio/core/src/error.rs @@ -0,0 +1,20 @@ +#[derive(Debug)] +pub enum Error { + OsError(yggdrasil_abi::error::Error), + InvalidPciConfiguration, + NoCommonConfigCapability, + NoNotifyConfigCapability, + NoDeviceConfigCapability, + QueueTooLarge, + InvalidQueueSize, + EmptyTransaction, + QueueFull, + QueueEmpty, + WrongToken, +} + +impl From for Error { + fn from(value: yggdrasil_abi::error::Error) -> Self { + Self::OsError(value) + } +} diff --git a/driver/virtio/core/src/lib.rs b/driver/virtio/core/src/lib.rs new file mode 100644 index 00000000..e0505ccc --- /dev/null +++ b/driver/virtio/core/src/lib.rs @@ -0,0 +1,54 @@ +#![no_std] + +extern crate alloc; + +pub mod error; +pub mod queue; +pub mod transport; + +use bitflags::bitflags; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_structs, + registers::{ReadOnly, ReadWrite, WriteOnly}, +}; + +register_structs! { + pub CommonConfiguration { + (0x00 => device_feature_select: ReadWrite), + (0x04 => device_feature: ReadOnly), + (0x08 => driver_feature_select: ReadWrite), + (0x0C => driver_feature: ReadWrite), + (0x10 => msix_config: ReadWrite), + (0x12 => num_queues: ReadOnly), + (0x14 => device_status: ReadWrite), + (0x15 => config_generation: ReadOnly), + + (0x16 => queue_select: ReadWrite), + (0x18 => queue_size: ReadWrite), + (0x1A => queue_msix_vector: ReadWrite), + (0x1C => queue_enable: ReadWrite), + (0x1E => queue_notify_off: ReadWrite), + (0x20 => queue_desc: ReadWrite), + (0x28 => queue_driver: ReadWrite), + (0x30 => queue_device: ReadWrite), + + (0x38 => @END), + } +} + +bitflags! { + #[derive(Clone, Copy)] + pub struct DeviceStatus: u8 { + const ACKNOWLEDGE = 1 << 0; + const DRIVER = 1 << 1; + const DRIVER_OK = 1 << 2; + const FEATURES_OK = 1 << 3; + const DEVICE_NEEDS_RESET = 1 << 6; + const FAILED = 1 << 7; + } +} + +impl DeviceStatus { + pub const RESET_VALUE: Self = Self::empty(); +} diff --git a/driver/virtio/core/src/queue.rs b/driver/virtio/core/src/queue.rs new file mode 100644 index 00000000..4fd27108 --- /dev/null +++ b/driver/virtio/core/src/queue.rs @@ -0,0 +1,344 @@ +//! VirtIO queue implementation. +//! +//! # Note +//! +//! The code is poorly borrowed from `virtio-drivers` crate. I want to rewrite it properly myself. +use core::{ + mem::MaybeUninit, + sync::atomic::{fence, Ordering}, +}; + +use alloc::boxed::Box; +use kernel_util::mem::{ + address::{AsPhysicalAddress, IntoRaw}, + PageBox, +}; + +use crate::{error::Error, transport::Transport}; + +#[derive(Debug)] +#[repr(C)] +struct Descriptor { + address: u64, + len: u32, + flags: u16, + next: u16, +} + +// Layout: +// { +// flags: u16, +// idx: u16, +// ring: [u16; QUEUE_SIZE], +// used_event: u16 +// } +struct AvailableRing { + data: PageBox<[MaybeUninit]>, +} + +// Layout: +// { +// flags: u16, +// idx: u16, +// ring: [UsedElem; QUEUE_SIZE], +// avail_event: u16, +// _pad: u16 +// } +struct UsedRing { + data: PageBox<[MaybeUninit]>, + + used_count: usize, +} + +pub struct VirtQueue { + descriptor_table: PageBox<[MaybeUninit]>, + available: AvailableRing, + used: UsedRing, + + capacity: usize, + + queue_index: u16, + free_head: u16, + + avail_idx: u16, + last_used_idx: u16, + + msix_vector: u16, +} + +impl AvailableRing { + pub fn with_capacity(capacity: usize) -> Result { + let mut data = PageBox::new_uninit_slice(capacity + 3)?; + + data[1].write(0); + + Ok(Self { data }) + } + + pub fn set_head(&mut self, slot: u16, head: u16) { + self.data[slot as usize + 2].write(head); + } + + pub fn set_index(&mut self, index: u16) { + self.data[1].write(index); + } +} + +impl UsedRing { + pub fn with_capacity(capacity: usize) -> Result { + let mut data = PageBox::new_uninit_slice(capacity * 2 + 2)?; + + data[0].write(0); + + Ok(Self { + data, + used_count: 0, + }) + } + + pub fn read_slot(&self, index: u16) -> (u32, u32) { + let index = unsafe { self.data[1 + index as usize * 2].assume_init() }; + let len = unsafe { self.data[2 + index as usize * 2].assume_init() }; + (index, len) + } + + pub fn index(&self) -> u16 { + unsafe { (self.data[0].assume_init() >> 16) as u16 } + } +} + +impl VirtQueue { + pub fn with_capacity( + transport: &mut T, + index: u16, + capacity: usize, + msix_vector: Option, + ) -> Result { + // TODO check if queue is already set up + + let max_capacity = transport.max_queue_size(index); + + if !capacity.is_power_of_two() || capacity > u16::MAX.into() { + return Err(Error::InvalidQueueSize); + } + + if capacity > max_capacity as usize { + return Err(Error::QueueTooLarge); + } + + let descriptor_table = PageBox::new_uninit_slice(capacity)?; + let available = AvailableRing::with_capacity(capacity)?; + let used = UsedRing::with_capacity(capacity)?; + + transport.set_queue( + index, + capacity as u16, + unsafe { descriptor_table.as_physical_address() }, + unsafe { available.data.as_physical_address() }, + unsafe { used.data.as_physical_address() }, + msix_vector, + ); + + Ok(Self { + descriptor_table, + available, + used, + + capacity, + + queue_index: index, + free_head: 0, + + avail_idx: 0, + last_used_idx: 0, + + msix_vector: msix_vector.unwrap_or(0xFFFF), + }) + } + + pub fn capacity(&self) -> usize { + self.capacity + } + + pub fn with_max_capacity( + transport: &mut T, + index: u16, + capacity: usize, + msix_vector: Option, + ) -> Result { + let max_capacity = transport.max_queue_size(index); + let capacity = capacity.min(max_capacity as usize); + + Self::with_capacity(transport, index, capacity, msix_vector) + } + + pub unsafe fn add<'a, 'b>( + &mut self, + input: &'a [&'b mut PageBox<[u8]>], + output: &'a [&'b PageBox<[u8]>], + ) -> Result { + if input.is_empty() && output.is_empty() { + return Err(Error::EmptyTransaction); + } + let n_desc = input.len() + output.len(); + + if self.used.used_count + 1 > self.capacity || self.used.used_count + n_desc > self.capacity + { + return Err(Error::QueueFull); + } + + let head = self.add_direct(input, output); + let avail_slot = self.avail_idx % self.capacity as u16; + + self.available.set_head(avail_slot, head); + + fence(Ordering::SeqCst); + + self.avail_idx = self.avail_idx.wrapping_add(1); + self.available.set_index(self.avail_idx); + + fence(Ordering::SeqCst); + + Ok(head) + } + + unsafe fn add_direct<'a, 'b>( + &mut self, + input: &'a [&'b mut PageBox<[u8]>], + output: &'a [&'b PageBox<[u8]>], + ) -> u16 { + let head = self.free_head; + let mut last = self.free_head; + + for item in input { + assert_ne!(item.len(), 0); + let desc = &mut self.descriptor_table[usize::from(self.free_head)]; + let next = (self.free_head + 1) % self.capacity as u16; + + let desc = desc.write(Descriptor { + address: item.as_physical_address().into_raw(), + len: item.len().try_into().unwrap(), + // TODO MAGIC + flags: (1 << 0) | (1 << 1), + next, + }); + + last = self.free_head; + self.free_head = next; + } + + for item in output { + assert_ne!(item.len(), 0); + let desc = &mut self.descriptor_table[usize::from(self.free_head)]; + let next = (self.free_head + 1) % self.capacity as u16; + + let desc = desc.write(Descriptor { + address: item.as_physical_address().into_raw(), + len: item.len().try_into().unwrap(), + // TODO + flags: (1 << 0), + next, + }); + + last = self.free_head; + self.free_head = next; + } + + { + let last_desc = self.descriptor_table[last as usize].assume_init_mut(); + + // TODO + last_desc.flags &= !(1 << 0); + } + + self.used.used_count += (input.len() + output.len()); + + head + } + + pub fn add_notify_wait_pop<'a, 'b, T: Transport>( + &mut self, + input: &'a [&'b mut PageBox<[u8]>], + output: &'a [&'b PageBox<[u8]>], + transport: &mut T, + ) -> Result { + let token = unsafe { self.add(input, output) }?; + + transport.notify(self.queue_index); + + while self.is_used_empty() { + core::hint::spin_loop(); + } + + unsafe { self.pop_used(token) } + } + + pub fn is_used_empty(&self) -> bool { + fence(Ordering::SeqCst); + + self.last_used_idx == self.used.index() + } + + pub unsafe fn pop_last_used(&mut self) -> Option<(u16, u32)> { + let token = self.peek_used()?; + let len = self.pop_used(token).unwrap(); + + Some((token, len)) + } + + pub unsafe fn peek_used(&mut self) -> Option { + if !self.is_used_empty() { + let last_used = self.last_used_idx % self.capacity as u16; + Some(self.used.read_slot(last_used).0 as u16) + } else { + None + } + } + + pub unsafe fn pop_used(&mut self, token: u16) -> Result { + if self.is_used_empty() { + return Err(Error::QueueEmpty); + } + + let last_used_slot = self.last_used_idx % self.capacity as u16; + let (index, len) = self.used.read_slot(last_used_slot); + + if index != token as u32 { + return Err(Error::WrongToken); + } + + let freed_count = self.free_descriptor_chain(token); + + self.last_used_idx = self.last_used_idx.wrapping_add(1); + + Ok(len) + } + + unsafe fn free_descriptor_chain(&mut self, head: u16) -> usize { + let mut current_node = Some(self.descriptor_table[usize::from(head)].assume_init_mut()); + let mut count = 0; + + while let Some(current) = current_node { + assert_ne!(current.len, 0); + let next_head = (current.flags & (1 << 0) != 0).then_some(current.next); + + current.address = 0; + current.flags = 0; + current.next = 0; + current.len = 0; + + self.used.used_count -= 1; + count += 1; + + current_node = + next_head.map(|head| self.descriptor_table[usize::from(head)].assume_init_mut()); + } + + self.free_head = head; + count + } + + pub fn msix_vector(&self) -> u16 { + self.msix_vector + } +} diff --git a/driver/virtio/core/src/transport/mod.rs b/driver/virtio/core/src/transport/mod.rs new file mode 100644 index 00000000..b4db67ab --- /dev/null +++ b/driver/virtio/core/src/transport/mod.rs @@ -0,0 +1,92 @@ +use core::mem::size_of; + +use kernel_util::mem::{ + address::{IntoRaw, PhysicalAddress}, + device::DeviceMemoryIo, +}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + registers::{ReadWrite, WriteOnly}, +}; + +use crate::{CommonConfiguration, DeviceStatus}; + +pub mod pci; + +pub trait Transport { + fn common_cfg(&self) -> &CommonConfiguration; + fn notify_cfg(&self) -> &[WriteOnly]; + fn notify_off_mul(&self) -> usize; + fn supports_msix(&self) -> bool; + fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>>; + + fn read_device_features(&mut self) -> u64 { + let cfg = self.common_cfg(); + cfg.device_feature_select.set(0); + let low = cfg.device_feature.get(); + cfg.device_feature_select.set(1); + let high = cfg.device_feature.get(); + + (low as u64) | ((high as u64) << 32) + } + + fn write_driver_features(&mut self, value: u64) { + let cfg = self.common_cfg(); + cfg.driver_feature_select.set(0); + cfg.driver_feature.set(value as u32); + cfg.driver_feature_select.set(1); + cfg.driver_feature.set((value >> 32) as u32); + } + + fn read_device_status(&mut self) -> DeviceStatus { + let cfg = self.common_cfg(); + DeviceStatus::from_bits_retain(cfg.device_status.get()) + } + + fn write_device_status(&mut self, value: DeviceStatus) { + let cfg = self.common_cfg(); + cfg.device_status.set(value.bits()); + } + + fn max_queue_size(&mut self, queue: u16) -> u32 { + let cfg = self.common_cfg(); + cfg.queue_select.set(queue); + cfg.queue_size.get().into() + } + + fn set_queue( + &mut self, + queue: u16, + capacity: u16, + descriptor_table_phys: PhysicalAddress, + available_ring_phys: PhysicalAddress, + used_ring_phys: PhysicalAddress, + msix_vector: Option, + ) { + let cfg = self.common_cfg(); + cfg.queue_select.set(queue); + cfg.queue_size.set(capacity); + cfg.queue_desc.set(descriptor_table_phys.into_raw()); + cfg.queue_driver.set(available_ring_phys.into_raw()); + cfg.queue_device.set(used_ring_phys.into_raw()); + if self.supports_msix() { + cfg.queue_msix_vector.set(msix_vector.unwrap_or(0xFFFF)); + } + cfg.queue_enable.set(1); + } + + fn unset_queue(&mut self, queue: u16) { + todo!() + } + + fn notify(&mut self, queue: u16) { + let cfg = self.common_cfg(); + let notify = self.notify_cfg(); + + cfg.queue_select.set(queue); + let notify_off = cfg.queue_notify_off.get() as usize; + let index = (notify_off * self.notify_off_mul()) / size_of::(); + + notify[index].set(queue); + } +} diff --git a/driver/virtio/core/src/transport/pci.rs b/driver/virtio/core/src/transport/pci.rs new file mode 100644 index 00000000..aac330d1 --- /dev/null +++ b/driver/virtio/core/src/transport/pci.rs @@ -0,0 +1,119 @@ +use kernel_util::mem::{ + address::{FromRaw, PhysicalAddress}, + device::DeviceMemoryIo, +}; +use tock_registers::registers::WriteOnly; +use ygg_driver_pci::{ + capability::{ + MsiXCapability, VirtioCapabilityData, VirtioCommonConfigCapability, + VirtioDeviceConfigCapability, VirtioNotifyConfigCapability, + }, + PciCommandRegister, PciConfigurationSpace, +}; + +use crate::{error::Error, CommonConfiguration}; + +use super::Transport; + +pub struct PciTransport { + common_cfg: DeviceMemoryIo<'static, CommonConfiguration>, + device_cfg: DeviceMemoryIo<'static, [u8]>, + notify_cfg: DeviceMemoryIo<'static, [WriteOnly]>, + notify_cfg_mul: usize, +} + +impl Transport for PciTransport { + fn common_cfg(&self) -> &CommonConfiguration { + &self.common_cfg + } + + fn notify_cfg(&self) -> &[WriteOnly] { + &self.notify_cfg + } + + fn notify_off_mul(&self) -> usize { + self.notify_cfg_mul + } + + fn supports_msix(&self) -> bool { + true + } + + fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>> { + Some(&self.device_cfg) + } +} + +impl PciTransport { + pub fn from_config_space(space: &S) -> Result { + // Transitional devices MUST have a PCI Revision ID of 0. + // Transitional devices MUST have the PCI Subsystem Device ID + // matching the Virtio Device ID, as indicated in section 5. + // Transitional devices MUST have the Transitional PCI + // Device ID in the range 0x1000 to 0x103f. + // TODO check PCI subsystem ID + if space.rev_id() != 0 { + return Err(Error::InvalidPciConfiguration); + } + + let mut cmd = PciCommandRegister::from_bits_retain(space.command()); + cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); + cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; + space.set_command(cmd.bits()); + + // Extract capabilities + + let common_cfg_cap = space + .capability::() + .ok_or(Error::NoCommonConfigCapability)?; + // TODO this is not mandatory + let device_cfg_cap = space + .capability::() + .ok_or(Error::NoDeviceConfigCapability)?; + let notify_cfg_cap = space + .capability::() + .ok_or(Error::NoNotifyConfigCapability)?; + + // TODO MSI/MSI-X + + // Map the regions + + let common_cfg_base = space + .bar(common_cfg_cap.bar_index().unwrap()) + .unwrap() + .as_memory() + + common_cfg_cap.bar_offset(); + let device_cfg_base = space + .bar(device_cfg_cap.bar_index().unwrap()) + .unwrap() + .as_memory() + + device_cfg_cap.bar_offset(); + let device_cfg_len = device_cfg_cap.length(); + let notify_cfg_base = space + .bar(notify_cfg_cap.bar_index().unwrap()) + .unwrap() + .as_memory() + + notify_cfg_cap.bar_offset(); + let notify_cfg_len = notify_cfg_cap.length(); + let notify_cfg_mul = notify_cfg_cap.offset_multiplier(); + + let common_cfg_base = PhysicalAddress::from_raw(common_cfg_base); + let device_cfg_base = PhysicalAddress::from_raw(device_cfg_base); + let notify_cfg_base = PhysicalAddress::from_raw(notify_cfg_base); + + assert_eq!(notify_cfg_len % 2, 0); + + let common_cfg = unsafe { DeviceMemoryIo::map(common_cfg_base) }.unwrap(); + let device_cfg = + unsafe { DeviceMemoryIo::map_slice(device_cfg_base, device_cfg_len) }.unwrap(); + let notify_cfg = + unsafe { DeviceMemoryIo::map_slice(notify_cfg_base, notify_cfg_len / 2) }.unwrap(); + + Ok(Self { + common_cfg, + device_cfg, + notify_cfg, + notify_cfg_mul, + }) + } +} diff --git a/driver/virtio/net/Cargo.toml b/driver/virtio/net/Cargo.toml new file mode 100644 index 00000000..b8008284 --- /dev/null +++ b/driver/virtio/net/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "ygg_driver_virtio_net" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } + +ygg_driver_virtio_core = { path = "../core" } +ygg_driver_net_core = { path = "../../net/core" } +ygg_driver_pci = { path = "../../bus/pci", optional = true } + +log = "0.4.20" +bitflags = "2.4.2" +tock-registers = "0.8.1" +bytemuck = { version = "1.14.0", features = ["derive"] } + +[features] +default = [] +pci = ["ygg_driver_pci", "ygg_driver_virtio_core/pci"] diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs new file mode 100644 index 00000000..9b4b5d6f --- /dev/null +++ b/driver/virtio/net/src/lib.rs @@ -0,0 +1,316 @@ +// TODO use more fancy features of virtio-net, TCP/IP checksum offloading would be nice +#![feature(strict_provenance)] +#![no_std] + +extern crate alloc; + +use core::{ + future::Future, + mem::size_of, + pin::Pin, + ptr::NonNull, + task::{Context, Poll}, +}; + +use alloc::{boxed::Box, collections::BTreeMap}; +use bitflags::Flags; +use bytemuck::{Pod, Zeroable}; +use device_api::{ + interrupt::{InterruptAffinity, InterruptHandler, MsiHandler}, + Device, +}; +use kernel_util::{ + block, + mem::{ + self, + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::{DeviceMemoryIo, RawDeviceMemoryMapping}, + PageBox, + }, + message_interrupt_controller, + runtime::{self, QueueWaker}, + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, + util::{ring::RingBuffer, OneTimeInit}, +}; +use ygg_driver_net_core::{interface::NetworkDevice, types::MacAddress, Packet}; +use ygg_driver_pci::{ + capability::{ + MsiXCapability, MsiXVectorTable, VirtioCapabilityData, VirtioCommonConfigCapability, + VirtioDeviceConfigCapability, VirtioNotifyConfigCapability, + }, + PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, +}; +use ygg_driver_virtio_core::{ + queue::VirtQueue, + transport::{pci::PciTransport, Transport}, + DeviceStatus, +}; +use yggdrasil_abi::error::Error; + +struct Queues { + receive: IrqSafeSpinlock, + transmit: IrqSafeSpinlock, + + configuration_vector: usize, + receive_vector: usize, +} + +pub struct VirtioNet { + transport: IrqSafeSpinlock, + queues: OneTimeInit, + interface_id: OneTimeInit, + + mac: IrqSafeRwLock, + + pending_packets: IrqSafeRwLock>>, + pending_tx_packets: IrqSafeRwLock>>, + + vector_table: IrqSafeRwLock>, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C)] +struct VirtioPacketHeader { + flags: u8, + gso_type: u8, + hdr_len: u16, + gso_size: u16, + csum_start: u16, + csum_offset: u16, +} + +impl Queues { + pub fn try_receive(&self, _index: usize) -> Option<(u16, IrqSafeSpinlockGuard)> { + let mut queue = self.receive.lock(); + let (token, _) = unsafe { queue.pop_last_used() }?; + Some((token, queue)) + } +} + +impl VirtioNet { + const PACKET_SIZE: usize = 4096; + + pub fn new(mut transport: T, vector_table: MsiXVectorTable<'static>) -> Self { + // Read MAC from device config + let device_cfg = transport + .device_cfg() + .expect("virtio-net must have device-specific configuration section"); + let mut mac_bytes = [0; 6]; + mac_bytes.copy_from_slice(&device_cfg[..6]); + let mac = MacAddress::from(mac_bytes); + + Self { + transport: IrqSafeSpinlock::new(transport), + queues: OneTimeInit::new(), + interface_id: OneTimeInit::new(), + + mac: IrqSafeRwLock::new(mac), + + pending_packets: IrqSafeRwLock::new(BTreeMap::new()), + pending_tx_packets: IrqSafeRwLock::new(BTreeMap::new()), + vector_table: IrqSafeRwLock::new(vector_table), + } + } + + pub fn listen(&self, buffers: usize) { + let queues = self.queues.get(); + let mut queue = queues.receive.lock(); + let mut packets = self.pending_packets.write(); + + for _ in 0..buffers { + let mut packet = PageBox::new_slice(0, Self::PACKET_SIZE).unwrap(); + let token = unsafe { queue.add(&[&mut packet], &[]).unwrap() }; + packets.insert(token, packet); + } + } + + fn handle_receive_interrupt(&self, queue: usize) -> bool { + let queues = self.queues.get(); + let interface_id = *self.interface_id.get(); + let mut count = 0; + + while let Some((token, mut queue)) = queues.try_receive(queue) { + let mut pending_packets = self.pending_packets.write(); + let packet = pending_packets.remove(&token).unwrap(); + + let mut buffer = PageBox::new_slice(0, Self::PACKET_SIZE).unwrap(); + + let token = unsafe { queue.add(&[&mut buffer], &[]).unwrap() }; + pending_packets.insert(token, buffer); + + let packet = Packet::new(packet, size_of::(), interface_id); + ygg_driver_net_core::receive_packet(packet).unwrap(); + count += 1 + } + + if count != 0 { + self.transport.lock().notify(1); + } + + count != 0 + } + + fn begin_init(&self) -> Result { + let mut transport = self.transport.lock(); + let mut status = DeviceStatus::RESET_VALUE; + + log::debug!("Reset device"); + transport.write_device_status(status); + status |= DeviceStatus::ACKNOWLEDGE; + transport.write_device_status(status); + status |= DeviceStatus::DRIVER; + transport.write_device_status(status); + + let device_features = transport.read_device_features(); + + // TODO blah blah blah + + transport.write_driver_features(0); + + status |= DeviceStatus::FEATURES_OK; + transport.write_device_status(status); + + if !transport + .read_device_status() + .contains(DeviceStatus::FEATURES_OK) + { + return Err(Error::InvalidOperation); + } + + Ok(status) + } + + fn finish_init(&self, status: DeviceStatus) { + let mut transport = self.transport.lock(); + + transport.write_device_status(status | DeviceStatus::DRIVER_OK); + } + + unsafe fn setup_queues( + &'static self, + receive_count: usize, + transmit_count: usize, + ) -> Result<(), Error> { + // TODO multiqueue capability + assert_eq!(receive_count, 1); + assert_eq!(transmit_count, 1); + + let mut transport = self.transport.lock(); + let mut vt = self.vector_table.write(); + + let msix_range = vt.register_range( + 0, + 1 + receive_count, + message_interrupt_controller(), + InterruptAffinity::Any, + self, + )?; + + // TODO set the configuration vector in virtio common cfg + let receive_vector: u16 = msix_range[1].vector.try_into().unwrap(); + + // Setup the virtqs + let rx = VirtQueue::with_max_capacity(&mut *transport, 0, 128, Some(receive_vector)) + .map_err(cvt_error)?; + let tx = VirtQueue::with_max_capacity(&mut *transport, 1, 128, None).map_err(cvt_error)?; + + self.queues.init(Queues { + receive: IrqSafeSpinlock::new(rx), + transmit: IrqSafeSpinlock::new(tx), + + configuration_vector: msix_range[0].vector, + receive_vector: msix_range[1].vector, + }); + + Ok(()) + } +} + +impl NetworkDevice for VirtioNet { + fn transmit(&self, mut packet: PageBox<[u8]>) -> Result<(), Error> { + let queues = self.queues.get(); + let mut tx = queues.transmit.lock(); + let mut transport = self.transport.lock(); + packet[..size_of::()].fill(0); + let len = tx + .add_notify_wait_pop(&[], &[&packet], &mut *transport) + .unwrap(); + + Ok(()) + } + + fn read_hardware_address(&self) -> MacAddress { + *self.mac.read() + } + + fn packet_prefix_size(&self) -> usize { + size_of::() + } +} + +impl MsiHandler for VirtioNet { + fn handle_msi(&self, vector: usize) -> bool { + let Some(queues) = self.queues.try_get() else { + return false; + }; + + if vector == queues.receive_vector { + self.handle_receive_interrupt(0) + } else { + false + } + } +} + +impl Device for VirtioNet { + fn display_name(&self) -> &'static str { + "VirtIO Network Device" + } + + unsafe fn init(&'static self) -> Result<(), Error> { + let status = self.begin_init()?; + + // TODO multiqueue + self.setup_queues(1, 1)?; + + self.finish_init(status); + + let id = ygg_driver_net_core::register_interface(self); + self.interface_id.init(id); + self.listen(64); + + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + Ok(()) + } +} + +fn cvt_error(error: ygg_driver_virtio_core::error::Error) -> Error { + use ygg_driver_virtio_core::error::Error as VirtioError; + match error { + VirtioError::OsError(err) => err, + _ => Error::InvalidOperation, + } +} + +pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { + let space = &info.config_space; + + let mut msix = space.capability::().unwrap(); + let mut vt = msix.vector_table()?; + + // TODO is this really needed? PCI spec says this is masked on reset, though I'm not sure if + // firmware puts it back in masked state after loading the kernel + vt.mask_all(); + msix.set_function_mask(false); + msix.set_enabled(true); + + let transport = PciTransport::from_config_space(space).unwrap(); + let device = VirtioNet::new(transport, vt); + + let device = Box::leak(Box::new(device)); + + Ok(device) +} diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index fae48fe4..b84cb34b 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -27,6 +27,8 @@ extern "Rust" { pub fn __virtualize(phys: u64) -> usize; pub fn __physicalize(virt: usize) -> u64; + pub fn __translate_kernel(virt: usize) -> Option; + pub fn __map_device_pages( base: PhysicalAddress, count: usize, diff --git a/lib/kernel-util/src/mem/device.rs b/lib/kernel-util/src/mem/device.rs index 7d5433ce..a39c073a 100644 --- a/lib/kernel-util/src/mem/device.rs +++ b/lib/kernel-util/src/mem/device.rs @@ -2,6 +2,7 @@ use core::{ alloc::Layout, mem::size_of, ops::{Deref, DerefMut}, + ptr::NonNull, }; use alloc::sync::Arc; @@ -66,6 +67,39 @@ impl RawDeviceMemoryMapping { core::mem::forget(self); address } + + pub fn into_raw_parts(self) -> (usize, usize, usize, usize) { + let address = self.address; + let base_address = self.base_address; + let page_count = self.page_count; + let page_size = self.page_size; + + core::mem::forget(self); + + (address, base_address, page_count, page_size) + } + + pub unsafe fn from_raw_parts( + address: usize, + base_address: usize, + page_count: usize, + page_size: usize, + ) -> Self { + Self { + address, + base_address, + page_count, + page_size, + } + } + + /// "Casts" the mapping to a specific type T and returns a [NonNull] pointer to it + pub unsafe fn as_non_null(&self) -> NonNull { + if self.page_size * self.page_count < size_of::() { + panic!(); + } + NonNull::new_unchecked(self.address as *mut T) + } } impl Drop for RawDeviceMemoryMapping { diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index bb96ffef..239707a0 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -5,11 +5,12 @@ use core::{ ops::{Deref, DerefMut}, }; +use alloc::boxed::Box; use yggdrasil_abi::error::Error; use crate::api::{self, __allocate_contiguous_pages, __free_page, __physicalize}; -use self::address::{AsPhysicalAddress, PhysicalAddress}; +use self::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; pub mod address; pub mod device; @@ -258,3 +259,11 @@ unsafe impl Sync for PageBox {} pub fn allocate_page() -> Result { unsafe { api::__allocate_page() } } + +pub fn allocate_contiguous_pages(count: usize) -> Result { + unsafe { api::__allocate_contiguous_pages(count) } +} + +pub fn translate_kernel_address(virt: usize) -> Option { + unsafe { api::__translate_kernel(virt) } +} diff --git a/lib/kernel-util/src/util/ring.rs b/lib/kernel-util/src/util/ring.rs index 1fb8c70b..009e2a80 100644 --- a/lib/kernel-util/src/util/ring.rs +++ b/lib/kernel-util/src/util/ring.rs @@ -89,12 +89,9 @@ impl RingBuffer { /// /// The caller must perform the necessary checks to avoid reading beyond the write head. #[inline] - pub unsafe fn read_single_unchecked(&mut self) -> T - where - T: Copy, - { + pub unsafe fn read_single_unchecked(&mut self) -> T { let data = self.data.as_ref().unwrap(); - let res = data[self.rd].assume_init(); + let res = data[self.rd].assume_init_read(); self.rd = (self.rd + 1) % self.capacity; res } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 6d670e85..06c28e68 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -2,6 +2,7 @@ use core::{ any::Any, fmt, mem::MaybeUninit, + net::SocketAddr, task::{Context, Poll}, }; @@ -24,8 +25,10 @@ use crate::{ channel::ChannelDescriptor, device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, + socket::PacketSocketWrapper, traits::{Read, Seek, Write}, - FdPoll, FileReadiness, PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave, SharedMemory, + FdPoll, FileReadiness, PacketSocket, PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave, + SharedMemory, Socket, }; use self::{ @@ -63,6 +66,8 @@ pub enum File { Block(BlockFile), Char(CharFile), + PacketSocket(Arc), + AnonymousPipe(PipeEnd), Poll(FdPoll), Channel(ChannelDescriptor), @@ -115,6 +120,11 @@ impl File { )) } + /// Constructs a [File] from a [PacketSocket] + pub fn from_packet_socket(socket: Arc) -> Arc { + Arc::new(Self::PacketSocket(Arc::new(PacketSocketWrapper(socket)))) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -261,6 +271,39 @@ impl File { Err(Error::InvalidOperation) } } + + /// Interprets the file as a packet-based socket + pub fn as_packet_socket(&self) -> Result<&PacketSocketWrapper, Error> { + if let Self::PacketSocket(sock) = self { + Ok(sock) + } else { + Err(Error::InvalidOperation) + } + } + + /// Sends data to a socket + pub fn send_to(&self, buffer: &[u8], recepient: Option) -> Result { + match recepient { + Some(recepient) => self.as_packet_socket()?.send(recepient, buffer), + None => todo!(), + } + } + + /// Receives data from a socket + pub fn receive_from( + &self, + buffer: &mut [u8], + remote: &mut MaybeUninit, + ) -> Result { + match self { + Self::PacketSocket(socket) => { + let (addr, len) = socket.receive(buffer)?; + remote.write(addr); + Ok(len) + } + _ => Err(Error::InvalidOperation), + } + } } impl PageProvider for File { @@ -295,6 +338,7 @@ impl Read for File { // TODO maybe allow reading messages from Channels? Self::Channel(_) => Err(Error::InvalidOperation), Self::SharedMemory(_) => Err(Error::InvalidOperation), + Self::PacketSocket(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -314,6 +358,7 @@ impl Write for File { // TODO maybe allow writing messages to Channels? Self::Channel(_) => Err(Error::InvalidOperation), Self::SharedMemory(_) => Err(Error::InvalidOperation), + Self::PacketSocket(_) => Err(Error::InvalidOperation), Self::Directory(_) => Err(Error::IsADirectory), } } @@ -366,6 +411,11 @@ impl fmt::Debug for File { Self::SharedMemory(_) => f.debug_struct("SharedMemory").finish_non_exhaustive(), Self::PtySlave(_) => f.debug_struct("PtySlave").finish_non_exhaustive(), Self::PtyMaster(_) => f.debug_struct("PtyMaster").finish_non_exhaustive(), + Self::PacketSocket(sock) => f + .debug_struct("PacketSocket") + .field("local", &sock.local_address()) + .field("remote", &sock.remote_address()) + .finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 1e63cfd9..a55e5e15 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -19,6 +19,7 @@ pub(crate) mod path; pub(crate) mod poll; pub(crate) mod pty; pub(crate) mod shared_memory; +pub(crate) mod socket; pub(crate) mod traits; pub use channel::MessagePayload; @@ -32,4 +33,5 @@ pub use node::{ pub use poll::FdPoll; pub use pty::{PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave}; pub use shared_memory::SharedMemory; +pub use socket::{ConnectionSocket, ListenerSocket, PacketSocket, Socket}; pub use traits::{FileReadiness, Read, Seek, Write}; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index 91afa247..21bfb1c5 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -292,6 +292,10 @@ where let instance = instance.unwrap().downcast_ref::().unwrap(); Ok(instance.as_write()?.lock().write_slice(pos as _, buf)) } + + fn truncate(&self, _node: &NodeRef, _new_size: u64) -> Result<(), Error> { + Ok(()) + } } // Byte read-only node diff --git a/lib/vfs/src/socket.rs b/lib/vfs/src/socket.rs new file mode 100644 index 00000000..46e3d3bb --- /dev/null +++ b/lib/vfs/src/socket.rs @@ -0,0 +1,53 @@ +use core::{ + net::SocketAddr, + ops::Deref, + task::{Context, Poll}, +}; + +use alloc::sync::Arc; +use yggdrasil_abi::error::Error; + +use crate::FileReadiness; + +/// Interface for interacting with network sockets +pub trait Socket: FileReadiness + Send { + /// Socket listen/receive address + fn local_address(&self) -> SocketAddr; + + /// Socket remote address + fn remote_address(&self) -> Option; + + /// Closes a socket + fn close(&self) -> Result<(), Error>; +} + +/// Stateless/packet-based socket interface +pub trait PacketSocket: Socket { + /// Receives a packet into provided buffer. Will return an error if packet cannot be placed + /// within the buffer. + fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error>; + + /// Sends provided data to the recepient specified by `destination` + fn send(&self, destination: SocketAddr, data: &[u8]) -> Result; +} + +/// Connection-based client socket interface +pub trait ConnectionSocket: Socket {} +/// Connection-based listener socket interface +pub trait ListenerSocket: Socket {} + +pub struct PacketSocketWrapper(pub Arc); + +impl Deref for PacketSocketWrapper { + type Target = dyn PacketSocket; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +impl Drop for PacketSocketWrapper { + fn drop(&mut self) { + self.0.close().ok(); + } +} diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 2332174f..707c9cfb 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -122,6 +122,9 @@ pub trait Architecture { /// Converts a virtual address created by [Architecture::virtualize] back to its physical form fn physicalize(address: usize) -> u64; + /// Translates a kernel virtual address into its physical page + fn translate_kernel_address(address: usize) -> Option; + // Architecture intrinsics /// Suspends CPU until an interrupt is received @@ -356,3 +359,8 @@ fn __monotonic_timestamp() -> Result { fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { ARCHITECTURE.message_interrupt_controller() } + +#[no_mangle] +fn __translate_kernel(address: usize) -> Option { + ArchitectureImpl::translate_kernel_address(address) +} diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index a2886899..7eed2df3 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -311,6 +311,23 @@ fn clone_kernel_tables(dst: &mut PageTable) { } } +pub(super) fn translate_kernel_address(address: usize) -> Option { + let l0i = address.page_index::(); + let l1i = address.page_index::(); + let l2i = address.page_index::(); + let l3i = address.page_index::(); + + match l0i { + KERNEL_L0_INDEX => match l1i { + HEAP_MAPPING_L1I => Some(PhysicalAddress::from_raw(address - HEAP_MAPPING_OFFSET)), + DEVICE_MAPPING_L1I => todo!(), + _ => todo!(), + }, + RAM_MAPPING_L0I => todo!(), + _ => None, + } +} + /// Sets up the following memory map: /// ...: KERNEL_TABLES.l0: /// * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000 : RAM_MAPPING_L1 diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 689a0597..7d168791 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -275,6 +275,11 @@ impl Architecture for X86_64 { (address - RAM_MAPPING_OFFSET) as _ } + #[inline] + fn translate_kernel_address(address: usize) -> Option { + mem::translate_kernel_address(address) + } + fn external_interrupt_controller( &'static self, ) -> &'static dyn ExternalInterruptController { @@ -394,6 +399,12 @@ impl X86_64 { Some(0x01), ygg_driver_ahci::probe, ); + ygg_driver_pci::register_vendor_driver( + "Virtio PCI Network Device", + 0x1AF4, + 0x1000, + ygg_driver_virtio_net::probe, + ); match self.boot_data.get() { &BootData::YBoot(data) => { diff --git a/src/init.rs b/src/init.rs index c0f6a092..91149743 100644 --- a/src/init.rs +++ b/src/init.rs @@ -28,6 +28,8 @@ fn setup_root() -> Result { pub fn kinit() -> Result<(), Error> { infoln!("In main"); + ygg_driver_net_core::start_network_tasks()?; + #[cfg(feature = "fb_console")] { use crate::device::display::console::update_consoles_task; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index f1d27c7d..cf8b9ff9 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -1,5 +1,5 @@ //! System function call handlers -use core::{mem::MaybeUninit, time::Duration}; +use core::{mem::MaybeUninit, net::SocketAddr, time::Duration}; use abi::{ error::Error, @@ -9,6 +9,7 @@ use abi::{ TerminalOptions, TerminalSize, }, mem::MappingSource, + net::SocketType, process::{ ExecveOptions, ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions, @@ -18,6 +19,7 @@ use abi::{ use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; +use ygg_driver_net_core::socket::UdpSocket; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ @@ -658,6 +660,45 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) } + // Networking + SyscallFunction::BindSocket => { + let listen = arg_user_ref::(args[0] as usize)?; + let ty = SocketType::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; + + run_with_io(process, |mut io| { + let socket = match ty { + SocketType::UdpPacket => UdpSocket::bind(*listen)?, + _ => todo!(), + }; + let file = File::from_packet_socket(socket); + let fd = io.files.place_file(file, true)?; + Ok(fd.0 as usize) + }) + } + SyscallFunction::ConnectSocket => todo!(), + SyscallFunction::SendTo => { + let socket_fd = RawFd::from(args[0] as u32); + let buffer = arg_buffer_ref(args[1] as usize, args[2] as usize)?; + let recepient = arg_user_ref::>(args[3] as usize)?; + + run_with_io(process, |mut io| { + let file = io.files.file(socket_fd)?; + + file.send_to(buffer, *recepient) + }) + } + SyscallFunction::ReceiveFrom => { + let socket_fd = RawFd::from(args[0] as u32); + let buffer = arg_buffer_mut(args[1] as usize, args[2] as usize)?; + let remote = arg_user_mut::>(args[3] as usize)?; + + run_with_io(process, |mut io| { + let file = io.files.file(socket_fd)?; + + file.receive_from(buffer, remote) + }) + } + SyscallFunction::Fork => unreachable!(), } } From a69cb3125bf7cc42f7e093ddead6baf8fb6fcaa9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 22 Jan 2024 14:39:06 +0200 Subject: [PATCH 153/211] net: Better netconfig mechanism + RawSocket --- Cargo.toml | 1 - driver/bus/pci/src/lib.rs | 1 + driver/fs/kernel-fs/src/devfs.rs | 84 +------- driver/net/core/Cargo.toml | 4 +- driver/net/core/src/config.rs | 150 +++++++++++++ driver/net/core/src/ethernet.rs | 52 ++--- driver/net/core/src/interface.rs | 93 +++++---- driver/net/core/src/l3/arp.rs | 44 ++-- driver/net/core/src/l3/ip.rs | 81 +------- driver/net/core/src/l3/mod.rs | 179 ++++++++++++---- driver/net/core/src/l4/icmp.rs | 50 ++--- driver/net/core/src/l4/udp.rs | 60 +++--- driver/net/core/src/lib.rs | 60 ++---- driver/net/core/src/socket.rs | 266 ++++++++++++++++++++++-- driver/net/core/src/types.rs | 225 -------------------- driver/virtio/core/src/lib.rs | 3 +- driver/virtio/core/src/queue.rs | 9 +- driver/virtio/core/src/transport/mod.rs | 4 +- driver/virtio/core/src/transport/pci.rs | 4 +- driver/virtio/net/src/lib.rs | 45 ++-- lib/kernel-util/src/mem/mod.rs | 3 +- lib/vfs/src/channel.rs | 17 +- lib/vfs/src/file/mod.rs | 18 +- lib/vfs/src/lib.rs | 2 +- lib/vfs/src/pty.rs | 21 +- lib/vfs/src/socket.rs | 24 ++- src/arch/x86_64/mem/mod.rs | 6 +- src/syscall/mod.rs | 52 +++-- tools/gentables/src/x86_64.rs | 7 +- 29 files changed, 813 insertions(+), 752 deletions(-) create mode 100644 driver/net/core/src/config.rs delete mode 100644 driver/net/core/src/types.rs diff --git a/Cargo.toml b/Cargo.toml index c69cca0e..1d135326 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ static_assertions = "1.1.0" tock-registers = "0.8.1" cfg-if = "1.0.0" git-version = "0.3.5" - log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index c76ae9ad..4aebb4b6 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -82,6 +82,7 @@ pub enum PciCapabilityId { } /// Interface used for querying PCI capabilities +#[allow(unused)] pub trait PciCapability { /// Capability ID const ID: PciCapabilityId; diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 7833f6aa..1adc7203 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -1,15 +1,9 @@ //! Device virtual file system -use core::{ - net::IpAddr, - sync::atomic::{AtomicUsize, Ordering}, -}; +use core::sync::atomic::{AtomicUsize, Ordering}; -use alloc::{format, string::String, sync::Arc}; +use alloc::{format, string::String}; use kernel_util::util::OneTimeInit; -use vfs::{ - impls::{mdir, FnValueNode, MemoryDirectory, ReadOnlyFnValueNode}, - CharDevice, Node, NodeFlags, NodeRef, -}; +use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; @@ -22,28 +16,11 @@ pub enum CharDeviceType { TtySerial, } -/// Describes the kind of a network device -#[derive(Debug)] -pub enum NetworkDeviceType { - /// Ethernet device - Ethernet, -} - -pub trait NetworkConfigInterface: Send + Sync { - fn address(&self) -> Result; - fn set_address(&self, addr: IpAddr) -> Result<(), Error>; - - fn mac(&self) -> [u8; 6]; -} - static DEVFS_ROOT: OneTimeInit = OneTimeInit::new(); -static NET_ROOT: OneTimeInit = OneTimeInit::new(); /// Sets up the device filesystem pub fn init() { - let net = MemoryDirectory::empty(); - NET_ROOT.init(net.clone()); - let root = mdir([("net", net)]); + let root = MemoryDirectory::empty(); DEVFS_ROOT.init(root); } @@ -78,59 +55,6 @@ pub fn add_named_block_device>( DEVFS_ROOT.get().add_child(name, node) } -pub fn add_network_config>(name: S, node: NodeRef) -> Result<(), Error> { - let name = name.into(); - NET_ROOT.get().add_child(name, node) -} - -pub fn add_named_network_device>( - name: S, - dev: Arc, -) -> Result<(), Error> { - let name = name.into(); - log::info!("Add network device: {}", name); - - let read_dev = dev.clone(); - let write_dev = dev.clone(); - let address = FnValueNode::new( - move || read_dev.address(), - move |value| write_dev.set_address(value), - ); - - let read_dev = dev.clone(); - let mac = ReadOnlyFnValueNode::new(move || { - let bytes = read_dev.mac(); - let mut out = String::new(); - for (i, byte) in bytes.into_iter().enumerate() { - if i != 0 { - out.push(':'); - } - out.push_str(&format!("{:02X}", byte)); - } - Ok(out) - }); - - let dev_dir = mdir([("address", address), ("mac", mac)]); - - NET_ROOT.get().add_child(name, dev_dir) -} - -pub fn add_network_device( - dev: Arc, - kind: NetworkDeviceType, -) -> Result<(), Error> { - static ETH_COUNT: AtomicUsize = AtomicUsize::new(0); - - let (count, prefix) = match kind { - NetworkDeviceType::Ethernet => (Ð_COUNT, "eth"), - }; - - let value = count.fetch_add(1, Ordering::AcqRel); - let name = format!("{}{}", prefix, value); - - add_named_network_device(name, dev) -} - pub fn add_block_device_partition>( base_name: S, index: usize, diff --git a/driver/net/core/Cargo.toml b/driver/net/core/Cargo.toml index 7c6be1f8..f8e674f9 100644 --- a/driver/net/core/Cargo.toml +++ b/driver/net/core/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["serde_kernel", "bytemuck"] } kernel-util = { path = "../../../lib/kernel-util" } vfs = { path = "../../../lib/vfs" } @@ -14,3 +14,5 @@ kernel-fs = { path = "../../fs/kernel-fs" } log = "0.4.20" bytemuck = { version = "1.14.0", features = ["derive"] } +serde_json = { version = "1.0.111", default-features = false, features = ["alloc"] } +serde = { version = "1.0.193", features = ["derive"], default-features = false } diff --git a/driver/net/core/src/config.rs b/driver/net/core/src/config.rs new file mode 100644 index 00000000..ac1b19d7 --- /dev/null +++ b/driver/net/core/src/config.rs @@ -0,0 +1,150 @@ +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use serde::Serialize; +use vfs::{ChannelDescriptor, MessagePayload}; +use yggdrasil_abi::{ + error::Error, + io::MessageDestination, + net::{ + netconfig::{InterfaceInfo, InterfaceQuery, NetConfigRequest, NetConfigResult, RouteInfo}, + IpAddr, SubnetAddr, + }, +}; + +use crate::{interface::NetworkInterface, l3::Route}; + +async fn receive_request(channel: &ChannelDescriptor) -> Result<(u32, NetConfigRequest), Error> { + loop { + let raw = channel.receive_message_async().await?; + match &raw.payload { + MessagePayload::Data(message) => { + let msg = + serde_json::from_slice(message.as_ref()).map_err(|_| Error::InvalidArgument)?; + + return Ok((raw.source, msg)); + } + MessagePayload::File(_) => (), + } + } +} + +fn send_reply( + channel: &ChannelDescriptor, + recepient: u32, + message: NetConfigResult, +) -> Result<(), Error> { + let data = serde_json::to_vec(&message).map_err(|_| Error::InvalidArgument)?; + channel.send_message( + MessagePayload::Data(data.into_boxed_slice()), + MessageDestination::Specific(recepient), + ) +} + +fn list_interfaces() -> Vec<(Box, u32)> { + let interfaces = NetworkInterface::list_ref(); + interfaces + .iter() + .map(|(&id, iface)| (iface.name.clone(), id)) + .collect() +} + +fn describe_interface(interface: &NetworkInterface) -> InterfaceInfo { + InterfaceInfo { + interface_id: interface.id, + interface_name: interface.name.clone(), + address: interface.address.read().map(Into::into), + mac: interface.mac, + } +} + +fn describe_route(route: &Route) -> RouteInfo { + // NOTE: must exist + let interface = NetworkInterface::get(route.interface).unwrap(); + RouteInfo { + interface_name: interface.name.clone(), + interface_id: route.interface, + subnet: route.subnet, + gateway: route.gateway.map(Into::into), + } +} + +fn query_interface(query: InterfaceQuery) -> Option> { + match query { + InterfaceQuery::ById(id) => NetworkInterface::get(id).ok(), + InterfaceQuery::ByName(name) => { + let interfaces = NetworkInterface::list_ref(); + interfaces.iter().find_map(|(_, iface)| { + if iface.name == name { + Some(iface.clone()) + } else { + None + } + }) + } + } +} + +fn add_route( + query: InterfaceQuery, + gateway: Option, + subnet: SubnetAddr, +) -> Result<(), &'static str> { + let interface = query_interface(query).ok_or("No such interface")?; + let route = Route { + interface: interface.id, + gateway, + subnet, + }; + Route::insert(route).map_err(|_| "Could not insert route")?; + Ok(()) +} + +pub async fn network_config_service() -> Result<(), Error> { + let channel = ChannelDescriptor::open("@kernel-netconf", true); + + loop { + let (sender_id, request) = receive_request(&channel).await?; + + match request { + NetConfigRequest::ListRoutes => { + let routes = Route::list_ref(); + let route_info: Vec<_> = routes.iter().map(describe_route).collect(); + send_reply(&channel, sender_id, NetConfigResult::Ok(route_info))?; + } + NetConfigRequest::ListInterfaces => { + let interfaces = list_interfaces(); + send_reply(&channel, sender_id, NetConfigResult::Ok(interfaces))?; + } + NetConfigRequest::DescribeRoutes(_query) => todo!(), + NetConfigRequest::DescribeInterface(query) => { + let result = match query_interface(query) { + Some(interface) => NetConfigResult::Ok(describe_interface(&interface)), + None => NetConfigResult::err("No such interface"), + }; + send_reply(&channel, sender_id, result)?; + } + NetConfigRequest::AddRoute { + interface, + gateway, + subnet, + } => { + let result = match add_route(interface, gateway, subnet) { + Ok(()) => NetConfigResult::Ok(()), + Err(error) => NetConfigResult::err(error), + }; + send_reply(&channel, sender_id, result)?; + } + NetConfigRequest::SetNetworkAddress { interface, address } => { + let result = match query_interface(interface) { + Some(interface) => { + interface.set_address(address); + NetConfigResult::Ok(()) + } + None => NetConfigResult::err("No such interface"), + }; + + send_reply(&channel, sender_id, result)?; + } + NetConfigRequest::ClearNetworkAddress(_interface) => todo!(), + } + } +} diff --git a/driver/net/core/src/ethernet.rs b/driver/net/core/src/ethernet.rs index c7653dab..a9db10ce 100644 --- a/driver/net/core/src/ethernet.rs +++ b/driver/net/core/src/ethernet.rs @@ -1,15 +1,20 @@ -use core::{fmt, mem::size_of}; +use core::mem::size_of; -use bytemuck::{Pod, Zeroable}; +use alloc::sync::Arc; +use bytemuck::Pod; use kernel_util::mem::PageBox; -use yggdrasil_abi::error::Error; - -use crate::{ - interface::{self, NetworkInterface}, - l3, - types::{MacAddress, NetValue, Value}, +use yggdrasil_abi::{ + error::Error, + net::{ + protocols::{EtherType, EthernetFrame}, + types::NetValueImpl, + MacAddress, + }, }; +use crate::{interface::NetworkInterface, l3, socket::RawSocket}; + +#[derive(Clone)] pub struct L2Packet { pub interface_id: u32, @@ -19,7 +24,7 @@ pub struct L2Packet { pub l2_offset: usize, pub l3_offset: usize, - pub data: PageBox<[u8]>, + pub data: Arc>, } impl L2Packet { @@ -34,33 +39,6 @@ impl L2Packet { } } -#[derive(Clone, Copy, Debug, Pod, Zeroable)] -#[repr(C)] -pub struct EthernetFrame { - pub destination_mac: MacAddress, - pub source_mac: MacAddress, - pub ethertype: Value, -} - -#[derive(Clone, Copy, PartialEq, Eq, Debug, Pod, Zeroable)] -#[repr(transparent)] -pub struct EtherType(u16); - -impl EtherType { - pub const ARP: Self = Self(0x0806); - pub const IPV4: Self = Self(0x0800); -} - -impl NetValue for EtherType { - fn to_network_order(self) -> Value { - Value(Self(self.0.to_be())) - } - - fn from_network_order(value: Value) -> Self { - Self(u16::from_be(value.0 .0)) - } -} - pub fn send_l2( interface: &NetworkInterface, source_mac: MacAddress, @@ -87,6 +65,8 @@ pub fn handle(packet: L2Packet) { let frame = packet.ethernet_frame(); let ty = EtherType::from_network_order(frame.ethertype); + RawSocket::packet_received(packet.clone()); + match ty { EtherType::ARP => l3::arp::handle_packet(packet), EtherType::IPV4 => l3::ip::handle_v4_packet(packet), diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs index 4a843119..97a86850 100644 --- a/driver/net/core/src/interface.rs +++ b/driver/net/core/src/interface.rs @@ -1,24 +1,20 @@ use core::{ mem::size_of, - net::{IpAddr, Ipv4Addr}, sync::atomic::{AtomicU32, Ordering}, }; -use alloc::{collections::BTreeMap, sync::Arc, vec}; -use bytemuck::Pod; -use kernel_fs::devfs::{self, NetworkConfigInterface, NetworkDeviceType}; +use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; // TODO: link state management? use kernel_util::{ mem::PageBox, - sync::{mutex::Mutex, spin_rwlock::IrqSafeRwLock}, + sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, +}; +use yggdrasil_abi::{ + error::Error, + net::{protocols::EthernetFrame, IpAddr, MacAddress}, }; -use yggdrasil_abi::error::Error; -use crate::{ - ethernet::EthernetFrame, - l3::{self, arp::ARP_TABLE, IpFrame, Route}, - types::{MacAddress, SubnetAddress, SubnetV4Address}, -}; +use crate::l3::{arp::ARP_TABLE, Route}; pub trait NetworkDevice: Sync { fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error>; @@ -28,6 +24,7 @@ pub trait NetworkDevice: Sync { } pub struct NetworkInterface { + pub(crate) name: Box, pub(crate) device: &'static dyn NetworkDevice, pub(crate) mac: MacAddress, @@ -35,39 +32,6 @@ pub struct NetworkInterface { pub(crate) id: u32, } -impl NetworkConfigInterface for NetworkInterface { - fn address(&self) -> Result { - if let Some(address) = self.address.read().as_ref() { - Ok(*address) - } else { - Err(Error::InvalidOperation) - } - } - - fn set_address(&self, addr: IpAddr) -> Result<(), Error> { - // Remove old ARP entry, if exists - ARP_TABLE.write().remove_ip(self.id, addr); - *self.address.write() = Some(addr); - ARP_TABLE.write().insert_ip(self.id, self.mac, addr, true); - match addr { - IpAddr::V4(v4) => { - l3::add_route(Route { - subnet: SubnetAddress::V4(SubnetV4Address::from_address_mask(v4, 24)), - interface: self.id, - // TODO manual configuration for this - gateway: Some(IpAddr::V4(Ipv4Addr::new(11, 0, 0, 1))), - }); - } - IpAddr::V6(_) => todo!(), - } - Ok(()) - } - - fn mac(&self) -> [u8; 6] { - self.mac.into() - } -} - static ETH_INTERFACES: IrqSafeRwLock>> = IrqSafeRwLock::new(BTreeMap::new()); static ETH_INTERFACE_ID: AtomicU32 = AtomicU32::new(1); @@ -81,6 +45,44 @@ impl NetworkInterface { .ok_or(Error::DoesNotExist) } + pub fn query_by_name(name: &str) -> Result, Error> { + ETH_INTERFACES + .read() + .iter() + .find_map(|(_, iface)| { + if iface.name.as_ref() == name { + Some(iface.clone()) + } else { + None + } + }) + .ok_or(Error::DoesNotExist) + } + + pub fn list_ref() -> IrqSafeRwLockReadGuard<'static, BTreeMap>> { + ETH_INTERFACES.read() + } + + pub fn set_address(&self, address: IpAddr) { + // Flush routes associated with the interface + { + let mut routes = Route::list_mut(); + routes.retain(|route| route.interface != self.id); + } + + let mut addr = self.address.write(); + // Flush owned ARP entries related to the old address + if let Some(address) = *addr { + ARP_TABLE.write().remove_ip(self.id, address); + } + + addr.replace(address); + + ARP_TABLE + .write() + .insert_ip(self.id, self.mac, address, true); + } + pub fn send_l2(&self, l2_frame: &EthernetFrame, l2_data: &[u8]) -> Result<(), Error> { let l2_offset = self.device.packet_prefix_size(); let l2_data_offset = l2_offset + size_of::(); @@ -98,8 +100,10 @@ pub fn register_interface(dev: &'static dyn NetworkDevice) -> u32 { let mac = dev.read_hardware_address(); let id = ETH_INTERFACE_ID.fetch_add(1, Ordering::SeqCst); + let name = format!("eth{}", id).into_boxed_str(); let iface = NetworkInterface { + name, device: dev, mac, address: IrqSafeRwLock::new(None), @@ -109,7 +113,6 @@ pub fn register_interface(dev: &'static dyn NetworkDevice) -> u32 { let interface = Arc::new(iface); - devfs::add_network_device(interface.clone(), NetworkDeviceType::Ethernet).unwrap(); ETH_INTERFACES.write().insert(id, interface); id diff --git a/driver/net/core/src/l3/arp.rs b/driver/net/core/src/l3/arp.rs index b89a1656..d208b28c 100644 --- a/driver/net/core/src/l3/arp.rs +++ b/driver/net/core/src/l3/arp.rs @@ -1,35 +1,17 @@ -use core::{ - borrow::Borrow, - mem::size_of, - net::{IpAddr, Ipv4Addr}, +use core::mem::size_of; + +use alloc::collections::BTreeMap; +use kernel_util::sync::spin_rwlock::IrqSafeRwLock; +use yggdrasil_abi::{ + error::Error, + net::{ + protocols::{ArpFrame, EtherType}, + types::NetValueImpl, + IpAddr, Ipv4Addr, MacAddress, + }, }; -use alloc::{collections::BTreeMap, sync::Arc}; -use bytemuck::{Pod, Zeroable}; -use kernel_util::{mem::PageBox, sync::spin_rwlock::IrqSafeRwLock}; -use yggdrasil_abi::error::Error; - -use crate::{ - ethernet::{self, EtherType, EthernetFrame}, - interface::NetworkInterface, - types::{MacAddress, NetValue, Value}, - L2Packet, -}; - -#[derive(Clone, Copy, Debug, Pod, Zeroable)] -#[repr(C, packed)] -struct ArpFrame { - pub hardware_type: Value, - pub protocol: Value, - pub hardware_size: u8, - pub protocol_size: u8, - pub opcode: Value, - pub sender_mac: MacAddress, - pub sender_ip: Value, - pub target_mac: MacAddress, - // TODO handle IPv6 - pub target_ip: Value, -} +use crate::{ethernet, interface::NetworkInterface, L2Packet}; pub struct ArpTable { entries_v4: BTreeMap<(u32, Ipv4Addr), (MacAddress, bool)>, @@ -129,6 +111,8 @@ pub fn handle_packet(packet: L2Packet) { } }; + log::debug!("ARP: {} -> {}", sender_address, target_address); + let mut table = ARP_TABLE.write(); table.insert_v4(packet.interface_id, arp.sender_mac, sender_address, false); diff --git a/driver/net/core/src/l3/ip.rs b/driver/net/core/src/l3/ip.rs index 88945bb9..2a75b640 100644 --- a/driver/net/core/src/l3/ip.rs +++ b/driver/net/core/src/l3/ip.rs @@ -1,71 +1,15 @@ -use core::{ - fmt, - mem::size_of, - net::{IpAddr, Ipv4Addr}, +use core::mem::size_of; + +use yggdrasil_abi::net::{ + protocols::{IpProtocol, Ipv4Frame, UdpFrame}, + types::NetValueImpl, + IpAddr, Ipv4Addr, }; -use alloc::sync::Arc; -use bytemuck::{Pod, Zeroable}; -use yggdrasil_abi::error::Error; - -use crate::{ - ethernet::EthernetFrame, - interface::NetworkInterface, - l3, - l4::udp, - types::{InetChecksum, NetValue, Value}, - L2Packet, L3Packet, ACCEPT_QUEUE, -}; +use crate::{interface::NetworkInterface, L2Packet, L3Packet, ACCEPT_QUEUE}; use super::IpFrame; -#[derive(Clone, Copy, Debug, Pod, Zeroable)] -#[repr(C)] -pub struct Ipv4Frame { - pub version_length: u8, - pub dscp_flags: u8, - pub total_length: Value, - pub id: Value, - pub flags_frag: Value, - pub ttl: u8, - pub protocol: Protocol, - pub header_checksum: Value, - pub source_address: Value, - pub destination_address: Value, -} - -#[derive(Clone, Copy, PartialEq, Eq, Pod, Zeroable)] -#[repr(transparent)] -pub struct Protocol(pub u8); - -impl Protocol { - pub const ICMP: Self = Self(1); - pub const UDP: Self = Self(17); -} - -impl fmt::Debug for Protocol { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - Self::ICMP => f.write_str("ICMP"), - Self::UDP => f.write_str("UDP"), - _ => f.write_str(""), - } - } -} - -impl Ipv4Frame { - fn header_length(&self) -> usize { - core::cmp::min( - (self.version_length & 0xF) << 2, - size_of::() as u8, - ) as usize - } - - fn total_length(&self) -> usize { - u16::from_network_order(self.total_length) as usize - } -} - impl IpFrame for Ipv4Frame { fn destination_ip(&self) -> IpAddr { IpAddr::V4(Ipv4Addr::from(u32::from_network_order( @@ -78,9 +22,7 @@ impl IpFrame for Ipv4Frame { } fn data_length(&self) -> usize { - self.total_length() - .checked_sub(self.header_length()) - .unwrap_or(0) + self.total_length().saturating_sub(self.header_length()) } } @@ -104,16 +46,15 @@ pub fn handle_v4_packet(packet: L2Packet) { if is_input { // Extract ports from L4 proto let (source_port, destination_port) = match l3_frame.protocol { - Protocol::UDP => { + IpProtocol::UDP => { // TODO check size - let l4_frame: &udp::UdpFrame = - bytemuck::from_bytes(&l3_data[..size_of::()]); + let l4_frame: &UdpFrame = bytemuck::from_bytes(&l3_data[..size_of::()]); ( Some(u16::from_network_order(l4_frame.source_port)), Some(u16::from_network_order(l4_frame.destination_port)), ) } - Protocol::ICMP => (None, None), + IpProtocol::ICMP => (None, None), _ => (None, None), }; diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 321b93b4..0eecb384 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -1,24 +1,21 @@ -use core::{ - fmt, - mem::size_of, - net::{IpAddr, Ipv4Addr}, - str::FromStr, -}; +use core::{fmt, mem::size_of}; use alloc::{sync::Arc, vec::Vec}; use bytemuck::Pod; -use kernel_util::{mem::PageBox, sync::spin_rwlock::IrqSafeRwLock}; -use yggdrasil_abi::error::Error; - -use crate::{ - ethernet::{EtherType, EthernetFrame}, - interface::NetworkInterface, - l3::ip::Ipv4Frame, - l4, - types::{InetChecksum, NetValue, SubnetAddress}, +use kernel_util::{ + mem::PageBox, + sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard}, +}; +use yggdrasil_abi::{ + error::Error, + net::{ + protocols::{EtherType, EthernetFrame, InetChecksum, IpProtocol, Ipv4Frame}, + types::NetValueImpl, + IpAddr, Ipv4Addr, MacAddress, SubnetAddr, + }, }; -use self::ip::Protocol; +use crate::{interface::NetworkInterface, l4}; pub mod arp; pub mod ip; @@ -26,7 +23,7 @@ pub mod ip; pub struct L3Packet { pub interface_id: u32, - pub protocol: Protocol, + pub protocol: IpProtocol, pub source_address: IpAddr, pub destination_address: IpAddr, @@ -39,7 +36,7 @@ pub struct L3Packet { pub l4_offset: usize, pub data_length: usize, - pub data: PageBox<[u8]>, + pub data: Arc>, } pub trait IpFrame: Pod { @@ -50,7 +47,7 @@ pub trait IpFrame: Pod { // TODO use range map for this? pub struct Route { - pub subnet: SubnetAddress, + pub subnet: SubnetAddr, pub interface: u32, pub gateway: Option, } @@ -63,30 +60,33 @@ impl L3Packet { } } -pub fn lookup_route(address: IpAddr) -> Option<(u32, Option)> { - let routes = ROUTES.read(); - for route in routes.iter() { - if route.subnet.contains(&address) { - return Some((route.interface, route.gateway)); +impl Route { + pub fn list_mut() -> IrqSafeRwLockWriteGuard<'static, Vec> { + ROUTES.write() + } + + pub fn list_ref() -> IrqSafeRwLockReadGuard<'static, Vec> { + ROUTES.read() + } + + pub fn lookup(address: IpAddr) -> Option<(u32, Option)> { + let routes = ROUTES.read(); + for route in routes.iter() { + if route.subnet.contains(&address) { + return Some((route.interface, route.gateway)); + } } + None } - None -} -pub fn list_routes(mut f: F) { - let routes = ROUTES.read(); - for route in routes.iter() { - f(route); + pub fn insert(route: Self) -> Result<(), Error> { + // TODO check for conflicts + log::debug!("Add route: {}", route); + ROUTES.write().push(route); + Ok(()) } } -pub fn add_route(route: Route) -> Result<(), Error> { - // TODO check for conflicts - log::debug!("Add route: {}", route); - ROUTES.write().push(route); - Ok(()) -} - impl fmt::Display for Route { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{} ", self.subnet)?; @@ -99,35 +99,126 @@ impl fmt::Display for Route { pub fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { match l3_packet.protocol { - Protocol::UDP => l4::udp::handle(l3_packet), - Protocol::ICMP => l4::icmp::handle(l3_packet), - _ => todo!(), + IpProtocol::UDP => l4::udp::handle(l3_packet), + IpProtocol::ICMP => l4::icmp::handle(l3_packet), + _ => Ok(()), } } pub fn send_l4_ip( destination_ip: IpAddr, - protocol: ip::Protocol, + protocol: IpProtocol, l4_frame: &L4, l4_data: &[u8], ) -> Result<(), Error> { send_l4_ip_opt(destination_ip, protocol, l4_frame, &[], l4_data) } +pub fn send_l4_ip_broadcast( + v6: bool, + protocol: IpProtocol, + l4_frame: &L4, + l4_data: &[u8], +) -> Result<(), Error> { + send_l4_ip_broadcast_opt(v6, protocol, l4_frame, &[], l4_data) +} + +pub fn send_l4_ip_broadcast_opt( + v6: bool, + protocol: IpProtocol, + l4_frame: &L4, + l4_options: &[u8], + l4_data: &[u8], +) -> Result<(), Error> { + let destination = match v6 { + true => todo!(), + false => IpAddr::V4(Ipv4Addr::BROADCAST), + }; + // TODO not sure how broadcasts work lol + for (_, interface) in NetworkInterface::list_ref().iter() { + let source = interface.address.read().unwrap_or_else(|| match v6 { + true => todo!(), + false => IpAddr::V4(Ipv4Addr::UNSPECIFIED), + }); + + if source.is_ipv6() && !v6 { + continue; + } + + let IpAddr::V4(source_v4) = source else { + todo!(); + }; + let IpAddr::V4(destination_v4) = destination else { + todo!(); + }; + + let destination_mac = MacAddress::BROADCAST; + + let l2_offset = interface.device.packet_prefix_size(); + let l3_offset = l2_offset + size_of::(); + let l4_offset = l3_offset + size_of::(); + let l4_data_offset = l4_offset + size_of::(); + + let mut packet = PageBox::new_slice(0, l4_data_offset + l4_data.len())?; + + let l2_frame: &mut EthernetFrame = + bytemuck::from_bytes_mut(&mut packet[l2_offset..l3_offset]); + + l2_frame.source_mac = interface.mac; + l2_frame.destination_mac = destination_mac; + l2_frame.ethertype = EtherType::IPV4.to_network_order(); + + let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); + + l3_frame.source_address = u32::to_network_order(u32::from(source_v4)); + l3_frame.destination_address = u32::to_network_order(u32::from(destination_v4)); + l3_frame.protocol = protocol; + l3_frame.version_length = 0x45; + l3_frame.dscp_flags = 0; + l3_frame.total_length = u16::to_network_order( + (size_of::() + size_of::() + l4_data.len()) + .try_into() + .unwrap(), + ); + // Disable fragmentation + l3_frame.flags_frag = u16::to_network_order(0x4000); + l3_frame.id = u16::to_network_order(0); + l3_frame.header_checksum = u16::to_network_order(0); + l3_frame.ttl = 64; + + let l3_frame_bytes = &packet[l3_offset..l4_offset]; + let mut ip_checksum = InetChecksum::new(); + ip_checksum.add_bytes(l3_frame_bytes, true); + let ip_checksum = ip_checksum.finish(); + + let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); + l3_frame.header_checksum = u16::to_network_order(ip_checksum); + + packet[l4_offset..l4_data_offset].copy_from_slice(bytemuck::bytes_of(l4_frame)); + packet[l4_data_offset..l4_data_offset + l4_options.len()].copy_from_slice(l4_options); + packet[l4_data_offset + l4_options.len()..].copy_from_slice(l4_data); + + if let Err(error) = interface.device.transmit(packet) { + log::warn!("Could not transmit on {}: {:?}", interface.name, error); + } + } + + Ok(()) +} + pub fn send_l4_ip_opt( destination_ip: IpAddr, - protocol: ip::Protocol, + protocol: IpProtocol, l4_frame: &L4, l4_options: &[u8], l4_data: &[u8], ) -> Result<(), Error> { let IpAddr::V4(destination_v4) = destination_ip else { - log::debug!("Destination: {}", destination_ip); todo!(); }; // Lookup route to destination - let (iface_id, gateway) = lookup_route(destination_ip).unwrap(); + let (iface_id, gateway) = Route::lookup(destination_ip).unwrap(); let iface = NetworkInterface::get(iface_id).unwrap(); let Some(IpAddr::V4(gateway_v4)) = gateway else { diff --git a/driver/net/core/src/l4/icmp.rs b/driver/net/core/src/l4/icmp.rs index d4a0c95c..db02cf6a 100644 --- a/driver/net/core/src/l4/icmp.rs +++ b/driver/net/core/src/l4/icmp.rs @@ -1,25 +1,15 @@ -use core::{ - mem::size_of, - net::{IpAddr, Ipv4Addr}, +use core::mem::size_of; + +use yggdrasil_abi::{ + error::Error, + net::{ + protocols::{IcmpV4Frame, InetChecksum, IpProtocol}, + types::NetValueImpl, + IpAddr, Ipv4Addr, + }, }; -use bytemuck::{Pod, Zeroable}; -use yggdrasil_abi::error::Error; - -use crate::{ - l3::{self, ip::Protocol}, - types::{InetChecksum, NetValue, Value}, - L3Packet, -}; - -#[derive(Clone, Copy, Debug, Pod, Zeroable)] -#[repr(C)] -pub struct IcmpV4Frame { - ty: u8, - code: u8, - checksum: Value, - rest: Value, -} +use crate::{l3, L3Packet}; fn send_v4_reply( destination_ip: Ipv4Addr, @@ -46,7 +36,7 @@ fn send_v4_reply( l3::send_l4_ip( IpAddr::V4(destination_ip), - Protocol::ICMP, + IpProtocol::ICMP, &reply_frame, icmp_data, ) @@ -75,7 +65,7 @@ fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), Error> icmp_frame.ty, icmp_frame.code ); - return Ok(()); + Ok(()) } } } @@ -86,19 +76,3 @@ pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { IpAddr::V6(_) => todo!(), } } - -// fn send_icmp_v4_reply( -// l3_frame: &Ipv4Frame, -// icmp_frame: &IcmpV4Frame, -// icmp_data: &[u8], -// ) -> Result<(), Error> { -// -// l3::send_l4_ip(l3_frame.source_ip(), Protocol::ICMP, &l4_frame, icmp_data) -// } -// -// pub fn handle_icmp_v4_packet( -// interface: Arc, -// l3_frame: &Ipv4Frame, -// l3_data: &[u8], -// ) { -// } diff --git a/driver/net/core/src/l4/udp.rs b/driver/net/core/src/l4/udp.rs index 66b2beeb..f07307f9 100644 --- a/driver/net/core/src/l4/udp.rs +++ b/driver/net/core/src/l4/udp.rs @@ -1,34 +1,15 @@ -use core::{ - mem::size_of, - net::{IpAddr, SocketAddr}, +use core::mem::size_of; + +use yggdrasil_abi::{ + error::Error, + net::{ + protocols::{IpProtocol, UdpFrame}, + types::NetValueImpl, + IpAddr, SocketAddr, + }, }; -use bytemuck::{Pod, Zeroable}; -use yggdrasil_abi::error::Error; - -use crate::{ - l3::{self, ip::Protocol, IpFrame}, - socket::{self, UdpSocket}, - types::{NetValue, Value}, - L3Packet, -}; - -#[derive(Clone, Copy, Pod, Zeroable)] -#[repr(C)] -pub struct UdpFrame { - pub source_port: Value, - pub destination_port: Value, - pub length: Value, - pub checksum: Value, -} - -impl UdpFrame { - fn data_length(&self) -> usize { - (u16::from_network_order(self.length) as usize) - .checked_sub(0) - .unwrap_or(0) - } -} +use crate::{l3, socket::UdpSocket, L3Packet}; pub fn send( source_port: u16, @@ -44,7 +25,24 @@ pub fn send( checksum: 0u16.to_network_order(), }; - l3::send_l4_ip(destination_ip, Protocol::UDP, &udp_frame, data) + l3::send_l4_ip(destination_ip, IpProtocol::UDP, &udp_frame, data) +} + +pub fn send_broadcast( + v6: bool, + source_port: u16, + destination_port: u16, + data: &[u8], +) -> Result<(), Error> { + let length: u16 = (data.len() + size_of::()).try_into().unwrap(); + let udp_frame = UdpFrame { + source_port: source_port.to_network_order(), + destination_port: destination_port.to_network_order(), + length: length.to_network_order(), + checksum: 0u16.to_network_order(), + }; + + l3::send_l4_ip_broadcast(v6, IpProtocol::UDP, &udp_frame, data) } pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { @@ -72,7 +70,7 @@ pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { ); if let Some(socket) = UdpSocket::get(&destination) { - socket.packet_received(source, udp_data); + socket.packet_received(source, udp_data).ok(); } Ok(()) diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index bce65a9a..f77e9432 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -1,37 +1,17 @@ #![feature(map_try_insert)] +#![allow(clippy::type_complexity)] #![no_std] extern crate alloc; -use core::{ - fmt, - future::Future, - mem::size_of, - net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, - pin::Pin, - str::FromStr, - sync::atomic::{AtomicU32, Ordering}, - task::{Context, Poll}, -}; +use core::mem::size_of; -use alloc::{collections::BTreeMap, format, string::String, vec::Vec}; -use ethernet::{EtherType, EthernetFrame, L2Packet}; -use interface::{NetworkDevice, NetworkInterface}; -use kernel_fs::devfs; -use kernel_util::{ - mem::PageBox, - runtime::{self, QueueWaker}, - sync::{mutex::Mutex, spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, - util::ring::RingBuffer, -}; -use l3::{ip::Protocol, L3Packet, Route}; +use alloc::sync::Arc; +use ethernet::L2Packet; +use kernel_util::{mem::PageBox, runtime}; +use l3::L3Packet; use queue::Queue; -use socket::UdpSocket; -use types::{MacAddress, NetValue, SubnetAddress, Value}; -use vfs::{impls::FnValueNode, PacketSocket}; -use yggdrasil_abi::error::Error; - -use crate::types::SubnetV4Address; +use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; pub mod ethernet; pub mod l3; @@ -39,9 +19,9 @@ pub mod l4; pub mod socket; +pub mod config; pub mod interface; pub mod queue; -pub mod types; pub use interface::register_interface; @@ -72,26 +52,14 @@ pub fn receive_packet(packet: Packet) -> Result<(), Error> { } pub fn start_network_tasks() -> Result<(), Error> { - devfs::add_network_config( - "routes", - FnValueNode::new( - || { - let mut output = String::new(); - l3::list_routes(|route| { - output.push_str(&format!("{}\n", route)); - }); - - Ok(output) - }, - |_value| Err(Error::NotImplemented), - ), - ); - runtime::spawn(l2_packet_handler_worker())?; - for _ in 0..1 { + + for _ in 0..4 { runtime::spawn(l3_accept_worker())?; } + runtime::spawn(config::network_config_service())?; + Ok(()) } @@ -112,7 +80,7 @@ async fn l2_packet_handler_worker() { l2_offset: packet.offset, l3_offset: packet.offset + size_of::(), - data: packet.buffer, + data: Arc::new(packet.buffer), }; ethernet::handle(l2_packet); @@ -124,7 +92,7 @@ async fn l3_accept_worker() { let l3_packet = ACCEPT_QUEUE.wait().await; log::debug!( - "INPUT {:?} {}:{:?} -> {}:{:?}: ACCEPT", + "INPUT {} {}:{:?} -> {}:{:?}: ACCEPT", l3_packet.protocol, l3_packet.source_address, l3_packet.source_port, diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index fe851462..a9fa01a2 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -1,35 +1,48 @@ use core::{ future::Future, - net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, pin::Pin, - sync::atomic::{AtomicBool, Ordering}, + sync::atomic::{AtomicBool, AtomicU32, Ordering}, task::{Context, Poll}, }; use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use kernel_util::{ block, + mem::PageBox, runtime::QueueWaker, sync::{ mutex::{Mutex, MutexGuard}, - spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, + spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, LockMethod, }, util::ring::RingBuffer, }; use vfs::{FileReadiness, PacketSocket, Socket}; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{ + error::Error, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketInterfaceQuery, SocketOption}, +}; -use crate::{l4, queue::Queue}; +use crate::{ethernet::L2Packet, interface::NetworkInterface, l4}; pub struct UdpSocket { local: SocketAddr, remote: Option, + + broadcast: AtomicBool, + // TODO just place packets here for one less copy? receive_queue: Mutex)>>, receive_notify: QueueWaker, } +pub struct RawSocket { + id: u32, + bound: IrqSafeSpinlock>, + receive_queue: Mutex>, + receive_notify: QueueWaker, +} + pub struct SocketTable { inner: BTreeMap>, } @@ -44,7 +57,7 @@ impl SocketTable { pub fn insert(&mut self, address: SocketAddr, socket: Arc) -> Result<(), Error> { match self.inner.try_insert(address, socket) { Ok(_) => Ok(()), - Err(_) => return Err(Error::AlreadyExists), + Err(_) => Err(Error::AlreadyExists), } } @@ -65,7 +78,7 @@ impl SocketTable { } match local { - SocketAddr::V4(v4) => { + SocketAddr::V4(_v4) => { let unspec_v4 = SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, local.port()); self.inner.get(&SocketAddr::V4(unspec_v4)).cloned() } @@ -75,6 +88,11 @@ impl SocketTable { } static UDP_SOCKETS: IrqSafeRwLock> = IrqSafeRwLock::new(SocketTable::new()); +static RAW_SOCKET_ID: AtomicU32 = AtomicU32::new(0); +static RAW_SOCKETS: IrqSafeRwLock>> = + IrqSafeRwLock::new(BTreeMap::new()); +static BOUND_RAW_SOCKETS: IrqSafeRwLock>> = + IrqSafeRwLock::new(BTreeMap::new()); impl UdpSocket { pub fn bind(address: SocketAddr) -> Result, Error> { @@ -88,18 +106,19 @@ impl UdpSocket { let socket = Arc::new(Self { local: address, remote: None, + broadcast: AtomicBool::new(false), receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), receive_notify: QueueWaker::new(), }); - sockets.insert(address, socket.clone()); + sockets.insert(address, socket.clone()).unwrap(); log::debug!("UDP socket opened: {}", address); Ok(socket) } - pub fn connect(address: SocketAddr) -> Result, Error> { + pub fn connect(&self, _address: SocketAddr) -> Result<(), Error> { todo!() } @@ -112,7 +131,7 @@ impl UdpSocket { cx: &mut Context<'_>, ) -> Poll)>>, Error>> { self.receive_notify.register(cx.waker()); - let mut lock = self.receive_queue.lock()?; + let lock = self.receive_queue.lock()?; if lock.is_readable() { self.receive_notify.remove(cx.waker()); Poll::Ready(Ok(lock)) @@ -121,9 +140,7 @@ impl UdpSocket { } } - pub fn receive_raw<'a>( - &'a self, - ) -> impl Future), Error>> + 'a { + pub fn receive_raw(&self) -> impl Future), Error>> + '_ { struct F<'f> { socket: &'f UdpSocket, } @@ -131,7 +148,7 @@ impl UdpSocket { impl<'f> Future for F<'f> { type Output = Result<(SocketAddr, Vec), Error>; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { match self.socket.poll_receive(cx)? { Poll::Ready(mut lock) => { let (source, data) = unsafe { lock.read_single_unchecked() }; @@ -160,14 +177,27 @@ impl FileReadiness for UdpSocket { } impl PacketSocket for UdpSocket { - fn send(&self, destination: SocketAddr, data: &[u8]) -> Result { + fn send(&self, destination: Option, data: &[u8]) -> Result { + let Some(destination) = destination else { + // TODO can still send without setting address if "connected" + return Err(Error::InvalidArgument); + }; // TODO check that destnation family matches self family - l4::udp::send( - self.local.port(), - destination.ip(), - destination.port(), - data, - )?; + match (self.broadcast.load(Ordering::Relaxed), destination.ip()) { + // SendTo in broadcast? + (true, IpAddr::V4(Ipv4Addr::BROADCAST)) => { + l4::udp::send_broadcast(false, self.local.port(), destination.port(), data)?; + } + (true, _) => todo!(), + (false, _) => { + l4::udp::send( + self.local.port(), + destination.ip(), + destination.port(), + data, + )?; + } + } Ok(data.len()) } @@ -195,4 +225,198 @@ impl Socket for UdpSocket { log::debug!("UDP socket closed: {}", self.local); UDP_SOCKETS.write().remove(self.local) } + + fn set_option(&self, option: &SocketOption) -> Result<(), Error> { + match option { + &SocketOption::Broadcast(broadcast) => { + log::debug!("{} broadcast: {}", self.local, broadcast); + self.broadcast.store(broadcast, Ordering::Relaxed); + Ok(()) + } + _ => Err(Error::InvalidOperation), + } + } + + fn get_option(&self, option: &mut SocketOption) -> Result<(), Error> { + match option { + SocketOption::Broadcast(broadcast) => { + *broadcast = self.broadcast.load(Ordering::Relaxed); + Ok(()) + } + _ => Err(Error::InvalidOperation), + } + } +} + +impl RawSocket { + pub fn bind() -> Result, Error> { + let id = RAW_SOCKET_ID.fetch_add(1, Ordering::SeqCst); + let socket = Self { + id, + bound: IrqSafeSpinlock::new(None), + receive_queue: Mutex::new(RingBuffer::with_capacity(128)), + receive_notify: QueueWaker::new(), + }; + let socket = Arc::new(socket); + RAW_SOCKETS.write().insert(id, socket.clone()); + + Ok(socket) + } + + fn bound_packet_received(&self, packet: L2Packet) { + self.receive_queue.lock().unwrap().write(packet); + self.receive_notify.wake_one(); + } + + fn poll_receive( + &self, + cx: &mut Context<'_>, + ) -> Poll>, Error>> { + self.receive_notify.register(cx.waker()); + let lock = self.receive_queue.lock()?; + if lock.is_readable() { + self.receive_notify.remove(cx.waker()); + Poll::Ready(Ok(lock)) + } else { + Poll::Pending + } + } + + pub fn packet_received(packet: L2Packet) { + let bound_sockets = BOUND_RAW_SOCKETS.read(); + let raw_sockets = RAW_SOCKETS.read(); + + if let Some(ids) = bound_sockets.get(&packet.interface_id) { + for id in ids { + let socket = raw_sockets.get(id).unwrap(); + socket.bound_packet_received(packet.clone()); + } + } + } + + pub fn receive_raw(&self) -> impl Future> + '_ { + struct F<'f> { + socket: &'f RawSocket, + } + + impl<'f> Future for F<'f> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.socket.poll_receive(cx)? { + Poll::Ready(mut lock) => { + let packet = unsafe { lock.read_single_unchecked() }; + Poll::Ready(Ok(packet)) + } + Poll::Pending => Poll::Pending, + } + } + } + + F { socket: self } + } +} + +impl FileReadiness for RawSocket { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + self.poll_receive(cx).map_ok(|_| ()) + } +} + +impl Socket for RawSocket { + fn get_option(&self, option: &mut SocketOption) -> Result<(), Error> { + match option { + SocketOption::BoundHardwareAddress(mac) => { + let bound = self.bound.lock().ok_or(Error::DoesNotExist)?; + let interface = NetworkInterface::get(bound).unwrap(); + *mac = interface.mac; + Ok(()) + } + _ => Err(Error::InvalidOperation), + } + } + + fn set_option(&self, option: &SocketOption) -> Result<(), Error> { + match option { + SocketOption::BindInterface(query) => { + let mut bound = self.bound.lock(); + if bound.is_some() { + return Err(Error::AlreadyExists); + } + + let mut bound_sockets = BOUND_RAW_SOCKETS.write(); + + let interface = match *query { + SocketInterfaceQuery::ById(id) => NetworkInterface::get(id), + SocketInterfaceQuery::ByName(name) => NetworkInterface::query_by_name(name), + }?; + + let list = bound_sockets.entry(interface.id).or_default(); + bound.replace(interface.id); + list.push(self.id); + + Ok(()) + } + SocketOption::UnbindInterface => todo!(), + _ => Err(Error::InvalidOperation), + } + } + + fn close(&self) -> Result<(), Error> { + let bound = self.bound.lock().take(); + + if let Some(bound) = bound { + let mut bound_sockets = BOUND_RAW_SOCKETS.write(); + let mut clear = false; + + if let Some(list) = bound_sockets.get_mut(&bound) { + list.retain(|&item| item != self.id); + clear = list.is_empty(); + } + + if clear { + bound_sockets.remove(&bound); + } + } + + RAW_SOCKETS.write().remove(&self.id).unwrap(); + + Ok(()) + } + + fn local_address(&self) -> SocketAddr { + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)) + } + + fn remote_address(&self) -> Option { + None + } +} + +impl PacketSocket for RawSocket { + fn send(&self, _destination: Option, data: &[u8]) -> Result { + let bound = self.bound.lock().ok_or(Error::InvalidOperation)?; + let interface = NetworkInterface::get(bound).unwrap(); + let l2_offset = interface.device.packet_prefix_size(); + if data.len() > 4096 - l2_offset { + return Err(Error::InvalidArgument); + } + let mut packet = PageBox::new_slice(0, 4096)?; + packet[l2_offset..l2_offset + data.len()].copy_from_slice(data); + interface.device.transmit(packet)?; + Ok(data.len()) + } + + fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error> { + let data = block!(self.receive_raw().await)??; + let len = 4096 - data.l2_offset; + if buffer.len() < len { + todo!() + } + buffer[..len].copy_from_slice(&data.data[data.l2_offset..]); + Ok(( + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), + len, + )) + } } diff --git a/driver/net/core/src/types.rs b/driver/net/core/src/types.rs deleted file mode 100644 index fa439d54..00000000 --- a/driver/net/core/src/types.rs +++ /dev/null @@ -1,225 +0,0 @@ -use core::{ - fmt, - net::{IpAddr, Ipv4Addr}, - str::FromStr, -}; - -use bytemuck::{Pod, Zeroable}; -use yggdrasil_abi::error::Error; - -#[macro_export] -macro_rules! wrap_value { - ($vis:vis struct $name:ident($ty_vis:vis $ty:ty)) => { - #[derive(Clone, Copy, Debug, bytemuck::Pod, bytemuck::Zeroable)] - $vis struct $name($ty_vis $ty); - - impl $crate::types::NetValue for $name { - fn from_network_order(value: $crate::types::Value) -> Self { - todo!() - } - - fn to_network_order(self) -> $crate::types::Value { - todo!() - } - } - }; -} - -pub trait NetValue: Copy + Eq + Pod + Zeroable { - fn from_network_order(value: Value) -> Self; - fn to_network_order(self) -> Value; -} - -#[derive(Clone, Copy, PartialEq, Eq, Pod, Zeroable)] -#[repr(transparent)] -pub struct Value(pub T); - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Pod, Zeroable)] -#[repr(transparent)] -pub struct MacAddress([u8; 6]); - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub struct SubnetV4Address { - pub root: Ipv4Addr, - pub mask_bits: u8, - pub mask: u32, -} - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -pub enum SubnetAddress { - V4(SubnetV4Address), -} - -impl fmt::Debug for Value { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let value = T::from_network_order(*self); - fmt::Debug::fmt(&value, f) - } -} - -impl NetValue for u16 { - fn from_network_order(value: Value) -> Self { - Self::from_be(value.0) - } - - fn to_network_order(self) -> Value { - Value(self.to_be()) - } -} - -impl NetValue for u32 { - fn from_network_order(value: Value) -> Self { - Self::from_be(value.0) - } - - fn to_network_order(self) -> Value { - Value(self.to_be()) - } -} - -impl fmt::Display for MacAddress { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use fmt::Write; - - for (i, &v) in self.0.iter().enumerate() { - if i != 0 { - write!(f, ":")?; - } - write!(f, "{:02X}", v)?; - } - - Ok(()) - } -} - -impl From for [u8; 6] { - fn from(value: MacAddress) -> Self { - value.0 - } -} - -impl From<[u8; 6]> for MacAddress { - fn from(value: [u8; 6]) -> Self { - Self(value) - } -} - -impl SubnetV4Address { - pub const UNSPECIFIED: Self = Self { - root: Ipv4Addr::new(0, 0, 0, 0), - mask: 0, - mask_bits: 0, - }; - - pub fn from_address_mask(root: Ipv4Addr, mask_bits: u8) -> Self { - let root = u32::from(root); - let mask = Self::make_mask(mask_bits); - // Clear masked bits - let root = root & mask; - - Self { - root: Ipv4Addr::from(root), - mask_bits, - mask, - } - } - - pub fn contains(&self, address: &Ipv4Addr) -> bool { - let root = u32::from(self.root); - let address = u32::from(*address); - - (address & self.mask) == root - } - - fn make_mask(bits: u8) -> u32 { - ((1 << bits) - 1) << (32 - bits) - } -} - -impl fmt::Display for SubnetV4Address { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}/{}", self.root, self.mask_bits) - } -} - -impl FromStr for SubnetV4Address { - type Err = Error; - - fn from_str(s: &str) -> Result { - let (addr, mask) = s.split_once('/').ok_or(Error::InvalidArgument)?; - let addr = Ipv4Addr::from_str(addr).map_err(|_| Error::InvalidArgument)?; - let mask = u8::from_str(mask).map_err(|_| Error::InvalidArgument)?; - - Ok(Self::from_address_mask(addr, mask)) - } -} - -impl SubnetAddress { - pub const UNSPECIFIED_V4: Self = Self::V4(SubnetV4Address::UNSPECIFIED); - - pub fn contains(&self, address: &IpAddr) -> bool { - match (self, address) { - (Self::V4(subnet), IpAddr::V4(address)) => subnet.contains(address), - _ => false, - } - } - - pub fn is_v4(&self) -> bool { - true - } -} - -impl From for SubnetAddress { - fn from(value: SubnetV4Address) -> Self { - Self::V4(value) - } -} - -impl FromStr for SubnetAddress { - type Err = Error; - - fn from_str(s: &str) -> Result { - // TODO v6 - let v4 = SubnetV4Address::from_str(s)?; - Ok(Self::from(v4)) - } -} - -impl fmt::Display for SubnetAddress { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::V4(v4) => fmt::Display::fmt(v4, f), - } - } -} - -pub struct InetChecksum { - state: u32, -} - -impl InetChecksum { - pub fn new() -> Self { - Self { state: 0 } - } - - pub fn add_bytes(&mut self, bytes: &[u8], reorder: bool) { - let len = bytes.len(); - for i in 0..len / 2 { - let word = if reorder { - ((bytes[i * 2] as u16) << 8) | (bytes[i * 2 + 1] as u16) - } else { - (bytes[i * 2] as u16) | ((bytes[i * 2 + 1] as u16) << 8) - }; - self.state = self.state.wrapping_add(word as u32); - } - if len % 2 != 0 { - self.state += self.state.wrapping_add(bytes[len - 1] as u32); - } - } - - pub fn finish(self) -> u16 { - let sum = (self.state >> 16) + (self.state & 0xFFFF); - - (!(sum & 0xFFFF)) as u16 - } -} diff --git a/driver/virtio/core/src/lib.rs b/driver/virtio/core/src/lib.rs index e0505ccc..85b7491e 100644 --- a/driver/virtio/core/src/lib.rs +++ b/driver/virtio/core/src/lib.rs @@ -8,9 +8,8 @@ pub mod transport; use bitflags::bitflags; use tock_registers::{ - interfaces::{Readable, Writeable}, register_structs, - registers::{ReadOnly, ReadWrite, WriteOnly}, + registers::{ReadOnly, ReadWrite}, }; register_structs! { diff --git a/driver/virtio/core/src/queue.rs b/driver/virtio/core/src/queue.rs index 4fd27108..ac3db8c5 100644 --- a/driver/virtio/core/src/queue.rs +++ b/driver/virtio/core/src/queue.rs @@ -8,7 +8,6 @@ use core::{ sync::atomic::{fence, Ordering}, }; -use alloc::boxed::Box; use kernel_util::mem::{ address::{AsPhysicalAddress, IntoRaw}, PageBox, @@ -215,7 +214,7 @@ impl VirtQueue { let desc = &mut self.descriptor_table[usize::from(self.free_head)]; let next = (self.free_head + 1) % self.capacity as u16; - let desc = desc.write(Descriptor { + desc.write(Descriptor { address: item.as_physical_address().into_raw(), len: item.len().try_into().unwrap(), // TODO MAGIC @@ -232,7 +231,7 @@ impl VirtQueue { let desc = &mut self.descriptor_table[usize::from(self.free_head)]; let next = (self.free_head + 1) % self.capacity as u16; - let desc = desc.write(Descriptor { + desc.write(Descriptor { address: item.as_physical_address().into_raw(), len: item.len().try_into().unwrap(), // TODO @@ -251,7 +250,7 @@ impl VirtQueue { last_desc.flags &= !(1 << 0); } - self.used.used_count += (input.len() + output.len()); + self.used.used_count += input.len() + output.len(); head } @@ -307,7 +306,7 @@ impl VirtQueue { return Err(Error::WrongToken); } - let freed_count = self.free_descriptor_chain(token); + self.free_descriptor_chain(token); self.last_used_idx = self.last_used_idx.wrapping_add(1); diff --git a/driver/virtio/core/src/transport/mod.rs b/driver/virtio/core/src/transport/mod.rs index b4db67ab..4d761147 100644 --- a/driver/virtio/core/src/transport/mod.rs +++ b/driver/virtio/core/src/transport/mod.rs @@ -6,7 +6,7 @@ use kernel_util::mem::{ }; use tock_registers::{ interfaces::{Readable, Writeable}, - registers::{ReadWrite, WriteOnly}, + registers::WriteOnly, }; use crate::{CommonConfiguration, DeviceStatus}; @@ -75,7 +75,7 @@ pub trait Transport { cfg.queue_enable.set(1); } - fn unset_queue(&mut self, queue: u16) { + fn unset_queue(&mut self, _queue: u16) { todo!() } diff --git a/driver/virtio/core/src/transport/pci.rs b/driver/virtio/core/src/transport/pci.rs index aac330d1..eca3d1e7 100644 --- a/driver/virtio/core/src/transport/pci.rs +++ b/driver/virtio/core/src/transport/pci.rs @@ -5,8 +5,8 @@ use kernel_util::mem::{ use tock_registers::registers::WriteOnly; use ygg_driver_pci::{ capability::{ - MsiXCapability, VirtioCapabilityData, VirtioCommonConfigCapability, - VirtioDeviceConfigCapability, VirtioNotifyConfigCapability, + VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability, + VirtioNotifyConfigCapability, }, PciCommandRegister, PciConfigurationSpace, }; diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs index 9b4b5d6f..b7a05c60 100644 --- a/driver/virtio/net/src/lib.rs +++ b/driver/virtio/net/src/lib.rs @@ -4,53 +4,37 @@ extern crate alloc; -use core::{ - future::Future, - mem::size_of, - pin::Pin, - ptr::NonNull, - task::{Context, Poll}, -}; +use core::mem::size_of; use alloc::{boxed::Box, collections::BTreeMap}; -use bitflags::Flags; use bytemuck::{Pod, Zeroable}; use device_api::{ - interrupt::{InterruptAffinity, InterruptHandler, MsiHandler}, + interrupt::{InterruptAffinity, MsiHandler}, Device, }; use kernel_util::{ - block, - mem::{ - self, - address::{FromRaw, IntoRaw, PhysicalAddress}, - device::{DeviceMemoryIo, RawDeviceMemoryMapping}, - PageBox, - }, + mem::PageBox, message_interrupt_controller, - runtime::{self, QueueWaker}, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, - util::{ring::RingBuffer, OneTimeInit}, + util::OneTimeInit, }; -use ygg_driver_net_core::{interface::NetworkDevice, types::MacAddress, Packet}; +use ygg_driver_net_core::{interface::NetworkDevice, Packet}; use ygg_driver_pci::{ - capability::{ - MsiXCapability, MsiXVectorTable, VirtioCapabilityData, VirtioCommonConfigCapability, - VirtioDeviceConfigCapability, VirtioNotifyConfigCapability, - }, - PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, + capability::{MsiXCapability, MsiXVectorTable}, + PciConfigurationSpace, PciDeviceInfo, }; use ygg_driver_virtio_core::{ queue::VirtQueue, transport::{pci::PciTransport, Transport}, DeviceStatus, }; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{error::Error, net::MacAddress}; struct Queues { receive: IrqSafeSpinlock, transmit: IrqSafeSpinlock, + #[allow(unused)] configuration_vector: usize, receive_vector: usize, } @@ -63,7 +47,6 @@ pub struct VirtioNet { mac: IrqSafeRwLock, pending_packets: IrqSafeRwLock>>, - pending_tx_packets: IrqSafeRwLock>>, vector_table: IrqSafeRwLock>, } @@ -82,7 +65,8 @@ struct VirtioPacketHeader { impl Queues { pub fn try_receive(&self, _index: usize) -> Option<(u16, IrqSafeSpinlockGuard)> { let mut queue = self.receive.lock(); - let (token, _) = unsafe { queue.pop_last_used() }?; + // TODO use len for packet size hint + let (token, _len) = unsafe { queue.pop_last_used() }?; Some((token, queue)) } } @@ -90,7 +74,7 @@ impl Queues { impl VirtioNet { const PACKET_SIZE: usize = 4096; - pub fn new(mut transport: T, vector_table: MsiXVectorTable<'static>) -> Self { + pub fn new(transport: T, vector_table: MsiXVectorTable<'static>) -> Self { // Read MAC from device config let device_cfg = transport .device_cfg() @@ -107,7 +91,6 @@ impl VirtioNet { mac: IrqSafeRwLock::new(mac), pending_packets: IrqSafeRwLock::new(BTreeMap::new()), - pending_tx_packets: IrqSafeRwLock::new(BTreeMap::new()), vector_table: IrqSafeRwLock::new(vector_table), } } @@ -161,7 +144,7 @@ impl VirtioNet { status |= DeviceStatus::DRIVER; transport.write_device_status(status); - let device_features = transport.read_device_features(); + let _device_features = transport.read_device_features(); // TODO blah blah blah @@ -232,7 +215,7 @@ impl NetworkDevice for VirtioNet { let mut tx = queues.transmit.lock(); let mut transport = self.transport.lock(); packet[..size_of::()].fill(0); - let len = tx + let _len = tx .add_notify_wait_pop(&[], &[&packet], &mut *transport) .unwrap(); diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 239707a0..d68e80aa 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -5,12 +5,11 @@ use core::{ ops::{Deref, DerefMut}, }; -use alloc::boxed::Box; use yggdrasil_abi::error::Error; use crate::api::{self, __allocate_contiguous_pages, __free_page, __physicalize}; -use self::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; +use self::address::{AsPhysicalAddress, PhysicalAddress}; pub mod address; pub mod device; diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index 51f4ff95..e83859f4 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -19,6 +19,7 @@ use yggdrasil_abi::{error::Error, io::MessageDestination}; use crate::{FileReadiness, FileRef}; +/// Describes a channel over which messages can be sent to [Subscription]s pub struct Channel { last_id: AtomicU32, subscriptions: Mutex>>, @@ -32,16 +33,21 @@ pub enum MessagePayload { Data(Box<[u8]>), } +/// Describes a message sent over a channel pub struct Message { + /// Channel descriptor ID from which the message came pub source: u32, + /// Data of the message pub payload: MessagePayload, } +/// Describes a single subscription so some [Channel] pub struct Subscription { queue: Mutex>>, notify: AtomicWaker, } +/// Describes a pair of a [Channel] descriptor plus an optional [Subscription] pub struct ChannelDescriptor { id: u32, tx: Arc, @@ -49,6 +55,7 @@ pub struct ChannelDescriptor { } impl ChannelDescriptor { + /// Opens a channel descriptor, optionally creating a subscription to it pub fn open(name: &str, subscribe: bool) -> ChannelDescriptor { let tx = Channel::get_or_create(name.into()); // NOTE The first one to open the channel is guaranteed to get an ID of 0 @@ -62,6 +69,7 @@ impl ChannelDescriptor { Self { tx, rx, id } } + /// Receives a message from the subscription pub fn receive_message(&self) -> Result, Error> { let Some(rx) = self.rx.as_ref() else { return Err(Error::InvalidOperation); @@ -70,6 +78,13 @@ impl ChannelDescriptor { rx.receive_message_inner() } + /// Asynchronously receives a message from the subscription + pub async fn receive_message_async(&self) -> Result, Error> { + let rx = self.rx.as_ref().ok_or(Error::InvalidOperation)?; + rx.receive_message_async().await + } + + /// Sends a message to the channel pub fn send_message( &self, payload: MessagePayload, @@ -112,7 +127,7 @@ impl Channel { }) } - pub fn get_or_create(name: String) -> Arc { + fn get_or_create(name: String) -> Arc { let mut channels = CHANNELS.lock(); channels.entry(name).or_insert_with(Self::new).clone() diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 06c28e68..b2a22034 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -2,7 +2,6 @@ use core::{ any::Any, fmt, mem::MaybeUninit, - net::SocketAddr, task::{Context, Poll}, }; @@ -19,6 +18,7 @@ use yggdrasil_abi::{ io::{ DeviceRequest, DirectoryEntry, OpenOptions, RawFd, SeekFrom, TerminalOptions, TerminalSize, }, + net::SocketAddr, }; use crate::{ @@ -238,8 +238,9 @@ impl File { Self::Poll(ch) => ch.poll_read(cx), Self::PtyMaster(f) => f.poll_read(cx), Self::PtySlave(f) => f.poll_read(cx), + Self::PacketSocket(sock) => sock.poll_read(cx), // Polling not implemented, return ready immediately (XXX ?) - _ => Poll::Ready(Ok(())), + _ => Poll::Ready(Err(Error::NotImplemented)), } } @@ -272,6 +273,14 @@ impl File { } } + /// Interprets the file as a socket + pub fn as_socket(&self) -> Result<&dyn Socket, Error> { + match self { + Self::PacketSocket(socket) => Ok(socket.0.as_ref()), + _ => Err(Error::InvalidOperation), + } + } + /// Interprets the file as a packet-based socket pub fn as_packet_socket(&self) -> Result<&PacketSocketWrapper, Error> { if let Self::PacketSocket(sock) = self { @@ -283,10 +292,7 @@ impl File { /// Sends data to a socket pub fn send_to(&self, buffer: &[u8], recepient: Option) -> Result { - match recepient { - Some(recepient) => self.as_packet_socket()?.send(recepient, buffer), - None => todo!(), - } + self.as_packet_socket()?.send(recepient, buffer) } /// Receives data from a socket diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index a55e5e15..03239420 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -22,7 +22,7 @@ pub(crate) mod shared_memory; pub(crate) mod socket; pub(crate) mod traits; -pub use channel::MessagePayload; +pub use channel::{Channel, ChannelDescriptor, Message, MessagePayload, Subscription}; pub use device::CharDevice; pub use file::{DirectoryOpenPosition, File, FileRef, FileSet, InstanceData}; pub use ioctx::{Action, IoContext}; diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 2f0ab550..3b78a3d6 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -20,8 +20,8 @@ use kernel_util::{ use yggdrasil_abi::{ error::Error, io::{ - DeviceRequest, TerminalControlCharacters, TerminalInputOptions, TerminalLineOptions, - TerminalOptions, TerminalOutputOptions, TerminalSize, + DeviceRequest, TerminalInputOptions, TerminalLineOptions, TerminalOptions, + TerminalOutputOptions, TerminalSize, }, process::Signal, }; @@ -140,7 +140,14 @@ impl PtyMasterToSlaveHalf { } pub fn poll(&self, cx: &mut Context<'_>) -> Poll> { - todo!() + self.notify.register(cx.waker()); + + if self.try_read().is_some() { + self.notify.remove(cx.waker()); + Poll::Ready(Ok(())) + } else { + Poll::Pending + } } pub fn flush(&self) { @@ -198,7 +205,7 @@ impl PtySlaveToMasterHalf { }) } - pub fn handle_input(&self, byte: u8, config: &TerminalOutputOptions) { + pub fn handle_input(&self, byte: u8, _config: &TerminalOutputOptions) { let mut lock = self.ring.lock(); // if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { @@ -302,10 +309,8 @@ impl PseudoTerminal { self.slave_to_master .handle_input(byte + 0x40, &config.output); } - } else { - if config.line.contains(TerminalLineOptions::ECHO) { - self.slave_to_master.handle_input(byte, &config.output); - } + } else if config.line.contains(TerminalLineOptions::ECHO) { + self.slave_to_master.handle_input(byte, &config.output); } if byte == config.chars.interrupt { diff --git a/lib/vfs/src/socket.rs b/lib/vfs/src/socket.rs index 46e3d3bb..dcec6aa1 100644 --- a/lib/vfs/src/socket.rs +++ b/lib/vfs/src/socket.rs @@ -1,15 +1,15 @@ -use core::{ - net::SocketAddr, - ops::Deref, - task::{Context, Poll}, -}; +use core::ops::Deref; use alloc::sync::Arc; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{ + error::Error, + net::{SocketAddr, SocketOption}, +}; use crate::FileReadiness; /// Interface for interacting with network sockets +#[allow(unused)] pub trait Socket: FileReadiness + Send { /// Socket listen/receive address fn local_address(&self) -> SocketAddr; @@ -19,6 +19,16 @@ pub trait Socket: FileReadiness + Send { /// Closes a socket fn close(&self) -> Result<(), Error>; + + /// Updates a socket option + fn set_option(&self, option: &SocketOption) -> Result<(), Error> { + Err(Error::InvalidOperation) + } + + /// Gets a socket option + fn get_option(&self, option: &mut SocketOption) -> Result<(), Error> { + Err(Error::InvalidOperation) + } } /// Stateless/packet-based socket interface @@ -28,7 +38,7 @@ pub trait PacketSocket: Socket { fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error>; /// Sends provided data to the recepient specified by `destination` - fn send(&self, destination: SocketAddr, data: &[u8]) -> Result; + fn send(&self, destination: Option, data: &[u8]) -> Result; } /// Connection-based client socket interface diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 7eed2df3..e131408e 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -28,7 +28,7 @@ use crate::{ use self::table::{PageEntry, PageTable, L0, L1, L2, L3}; const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; -const KERNEL_PHYS_BASE: usize = 0x400000; +const KERNEL_PHYS_BASE: usize = 0x200000; // Mapped at compile time const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE; @@ -314,8 +314,8 @@ fn clone_kernel_tables(dst: &mut PageTable) { pub(super) fn translate_kernel_address(address: usize) -> Option { let l0i = address.page_index::(); let l1i = address.page_index::(); - let l2i = address.page_index::(); - let l3i = address.page_index::(); + // let l2i = address.page_index::(); + // let l3i = address.page_index::(); match l0i { KERNEL_L0_INDEX => match l1i { diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index cf8b9ff9..a76bade6 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -1,5 +1,5 @@ //! System function call handlers -use core::{mem::MaybeUninit, net::SocketAddr, time::Duration}; +use core::{mem::MaybeUninit, time::Duration}; use abi::{ error::Error, @@ -9,7 +9,7 @@ use abi::{ TerminalOptions, TerminalSize, }, mem::MappingSource, - net::SocketType, + net::{SocketOption, SocketType}, process::{ ExecveOptions, ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, ThreadSpawnOptions, @@ -18,8 +18,8 @@ use abi::{ }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; -use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; -use ygg_driver_net_core::socket::UdpSocket; +use vfs::{File, IoContext, MessagePayload, NodeRef, PacketSocket, Read, Seek, Write}; +use ygg_driver_net_core::socket::{RawSocket, UdpSocket}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ @@ -662,12 +662,13 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Networking SyscallFunction::BindSocket => { - let listen = arg_user_ref::(args[0] as usize)?; + let listen = arg_user_ref::(args[0] as usize)?; let ty = SocketType::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; run_with_io(process, |mut io| { - let socket = match ty { - SocketType::UdpPacket => UdpSocket::bind(*listen)?, + let socket: Arc = match ty { + SocketType::UdpPacket => UdpSocket::bind((*listen).into())?, + SocketType::RawPacket => RawSocket::bind()?, _ => todo!(), }; let file = File::from_packet_socket(socket); @@ -679,23 +680,48 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::SendTo => { let socket_fd = RawFd::from(args[0] as u32); let buffer = arg_buffer_ref(args[1] as usize, args[2] as usize)?; - let recepient = arg_user_ref::>(args[3] as usize)?; + let recepient = arg_user_ref::>(args[3] as usize)?; - run_with_io(process, |mut io| { + run_with_io(process, |io| { let file = io.files.file(socket_fd)?; - file.send_to(buffer, *recepient) + file.send_to(buffer, recepient.map(Into::into)) }) } SyscallFunction::ReceiveFrom => { let socket_fd = RawFd::from(args[0] as u32); let buffer = arg_buffer_mut(args[1] as usize, args[2] as usize)?; - let remote = arg_user_mut::>(args[3] as usize)?; + let remote_result = + arg_user_mut::>(args[3] as usize)?; - run_with_io(process, |mut io| { + run_with_io(process, |io| { let file = io.files.file(socket_fd)?; + let mut remote = MaybeUninit::uninit(); + let len = file.receive_from(buffer, &mut remote)?; + remote_result.write(unsafe { remote.assume_init() }.into()); + Ok(len) + }) + } + SyscallFunction::GetSocketOption => { + let socket_fd = RawFd::from(args[0] as u32); + let option = arg_user_mut::(args[1] as usize)?; - file.receive_from(buffer, remote) + run_with_io(process, |io| { + let file = io.files.file(socket_fd)?; + let socket = file.as_socket()?; + socket.get_option(option)?; + Ok(0) + }) + } + SyscallFunction::SetSocketOption => { + let socket_fd = RawFd::from(args[0] as u32); + let option = arg_user_ref::(args[1] as usize)?; + + run_with_io(process, |io| { + let file = io.files.file(socket_fd)?; + let socket = file.as_socket()?; + socket.set_option(option)?; + Ok(0) }) } diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs index 87aa5c45..c1a5f7d3 100644 --- a/tools/gentables/src/x86_64.rs +++ b/tools/gentables/src/x86_64.rs @@ -166,13 +166,18 @@ impl X8664Builder { let entry = page | (PageFlags::PRESENT | flags).bits(); - let l2i = (address >> 21) as usize & 0x1FF - l2i_offset; + let l1i = (address >> 30) as usize & 0x1FF; + let l2i = ((address >> 21) as usize & 0x1FF) - l2i_offset; let l3i = (address >> 12) as usize & 0x1FF; let l3 = &mut tables.kernel_l3s[l2i]; if l3.data[l3i] != 0 { if l3.data[l3i] != entry { + eprintln!( + "{:#x} is already mapped to {:#x}, tried to map to {:#x}", + address, l3.data[l3i], page + ); todo!(); } else { continue; From b5f5c2c77e8550ef6977b9624d7ed27e355b32b9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 22 Jan 2024 17:14:13 +0200 Subject: [PATCH 154/211] bus/pci: use volatile properly --- driver/bus/pci/Cargo.toml | 1 + driver/bus/pci/src/capability.rs | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index 8ef36ab4..d411e98f 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -13,5 +13,6 @@ kernel-util = { path = "../../../lib/kernel-util" } log = "0.4.20" bitflags = "2.3.3" +tock-registers = "0.8.1" acpi = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" } diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index 387e070d..6e99c2e9 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -6,6 +6,10 @@ use kernel_util::mem::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIoMut, }; +use tock_registers::{ + interfaces::{Readable, Writeable}, + registers::{ReadWrite, WriteOnly}, +}; use yggdrasil_abi::error::Error; use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; @@ -57,11 +61,11 @@ pub struct VirtioNotifyConfigCapability; #[repr(C)] pub struct MsiXEntry { /// Address to which the value is written on interrupt - pub address: u64, + pub address: WriteOnly, /// Value which is written to trigger an interrupt - pub data: u32, + pub data: WriteOnly, /// Vector control word - pub control: u32, + pub control: ReadWrite, } pub struct MsiXVectorTable<'a> { @@ -292,8 +296,8 @@ impl MsiXVectorTable<'_> { for (i, info) in range.iter().enumerate() { let index = i + start; - self.vectors[index].address = info.address as _; - self.vectors[index].data = info.value; + self.vectors[index].address.set(info.address as _); + self.vectors[index].data.set(info.value); self.vectors[index].set_masked(false); } @@ -305,9 +309,9 @@ impl MsiXEntry { /// If set, prevents the MSI-X interrupt from being delivered fn set_masked(&mut self, masked: bool) { if masked { - self.control |= 1; + self.control.set(self.control.get() | 1); } else { - self.control &= !1; + self.control.set(self.control.get() & !1); } } } From 5ebb1409c37c0fcc2bf37003b07af6b28ce1a064 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 22 Jan 2024 23:59:42 +0200 Subject: [PATCH 155/211] net: implement async ARP resolution --- driver/net/core/src/interface.rs | 8 +- driver/net/core/src/l3/arp.rs | 210 ++++++++++++++++++++++----- driver/net/core/src/l3/mod.rs | 235 ++++++++++--------------------- driver/net/core/src/l4/icmp.rs | 12 +- driver/net/core/src/l4/mod.rs | 1 + driver/net/core/src/l4/tcp.rs | 0 driver/net/core/src/l4/udp.rs | 37 ++--- driver/net/core/src/lib.rs | 54 +++++-- driver/net/core/src/socket.rs | 21 +-- driver/virtio/core/src/queue.rs | 6 + 10 files changed, 339 insertions(+), 245 deletions(-) create mode 100644 driver/net/core/src/l4/tcp.rs diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs index 97a86850..32d42ea3 100644 --- a/driver/net/core/src/interface.rs +++ b/driver/net/core/src/interface.rs @@ -14,7 +14,7 @@ use yggdrasil_abi::{ net::{protocols::EthernetFrame, IpAddr, MacAddress}, }; -use crate::l3::{arp::ARP_TABLE, Route}; +use crate::l3::{arp::ArpTable, Route}; pub trait NetworkDevice: Sync { fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error>; @@ -73,14 +73,12 @@ impl NetworkInterface { let mut addr = self.address.write(); // Flush owned ARP entries related to the old address if let Some(address) = *addr { - ARP_TABLE.write().remove_ip(self.id, address); + ArpTable::flush_address(self.id, address); } addr.replace(address); - ARP_TABLE - .write() - .insert_ip(self.id, self.mac, address, true); + ArpTable::insert_address(self.id, self.mac, address, true); } pub fn send_l2(&self, l2_frame: &EthernetFrame, l2_data: &[u8]) -> Result<(), Error> { diff --git a/driver/net/core/src/l3/arp.rs b/driver/net/core/src/l3/arp.rs index d208b28c..56399ab1 100644 --- a/driver/net/core/src/l3/arp.rs +++ b/driver/net/core/src/l3/arp.rs @@ -1,7 +1,16 @@ -use core::mem::size_of; +use core::{ + future::Future, + mem::size_of, + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; -use alloc::collections::BTreeMap; -use kernel_util::sync::spin_rwlock::IrqSafeRwLock; +use alloc::{boxed::Box, collections::BTreeMap}; +use kernel_util::{ + runtime::{self, QueueWaker}, + sync::spin_rwlock::IrqSafeRwLock, +}; use yggdrasil_abi::{ error::Error, net::{ @@ -13,59 +22,189 @@ use yggdrasil_abi::{ use crate::{ethernet, interface::NetworkInterface, L2Packet}; +struct Inner { + entries: BTreeMap<(u32, A), (MacAddress, bool)>, + reverse: BTreeMap<(u32, MacAddress), (A, bool)>, +} + pub struct ArpTable { - entries_v4: BTreeMap<(u32, Ipv4Addr), (MacAddress, bool)>, - reverse_v4: BTreeMap<(u32, MacAddress), (Ipv4Addr, bool)>, + v4: IrqSafeRwLock>, + notify: QueueWaker, +} + +impl Inner { + const fn new() -> Self { + Self { + entries: BTreeMap::new(), + reverse: BTreeMap::new(), + } + } + + fn query_mac(&self, interface: u32, address: A) -> Option<(MacAddress, bool)> { + self.entries.get(&(interface, address)).copied() + } + + // fn query_address(&self, interface: u32, mac: MacAddress) -> Option<(A, bool)> { + // self.reverse.get(&(interface, mac)).copied() + // } + + fn insert(&mut self, interface: u32, mac: MacAddress, address: A, owned: bool) -> bool { + let new = self + .entries + .insert((interface, address), (mac, owned)) + .is_none(); + self.reverse.insert((interface, mac), (address, owned)); + new + } + + fn flush(&mut self, interface: u32, address: A) -> bool { + if let Some((mac, _)) = self.entries.remove(&(interface, address)) { + self.reverse.remove(&(interface, mac)); + true + } else { + false + } + } } impl ArpTable { pub const fn new() -> Self { Self { - reverse_v4: BTreeMap::new(), - entries_v4: BTreeMap::new(), + v4: IrqSafeRwLock::new(Inner::new()), + notify: QueueWaker::new(), } } - // TODO multiple IPs per MAC? - pub fn insert_v4(&mut self, interface: u32, mac: MacAddress, ip: Ipv4Addr, own: bool) { - if !self.reverse_v4.contains_key(&(interface, mac)) { - log::debug!("Insert {} <-> {}", mac, ip); - } - - self.reverse_v4.insert((interface, mac), (ip, own)); - self.entries_v4.insert((interface, ip), (mac, own)); + pub fn lookup_cache_v4(interface: u32, address: Ipv4Addr) -> Option<(MacAddress, bool)> { + ARP_TABLE.v4.read().query_mac(interface, address) } - pub fn insert_ip(&mut self, interface: u32, mac: MacAddress, ip: IpAddr, own: bool) { - match ip { - IpAddr::V4(v4) => self.insert_v4(interface, mac, v4, own), + pub fn lookup_cache(interface: u32, address: IpAddr) -> Option { + let (address, _) = match address { + IpAddr::V4(address) => Self::lookup_cache_v4(interface, address), + IpAddr::V6(_) => todo!(), + }?; + Some(address) + } + + pub fn flush_address_v4(interface: u32, address: Ipv4Addr) -> bool { + ARP_TABLE.v4.write().flush(interface, address) + } + + pub fn flush_address(interface: u32, address: IpAddr) -> bool { + match address { + IpAddr::V4(address) => Self::flush_address_v4(interface, address), IpAddr::V6(_) => todo!(), } } - pub fn remove_ip(&mut self, interface: u32, ip: IpAddr) { - match ip { - IpAddr::V4(v4) => { - let entry = self.entries_v4.remove(&(interface, v4)); - if let Some((mac, _)) = entry { - self.reverse_v4.remove(&(interface, mac)).unwrap(); + pub fn insert_address_v4(interface: u32, mac: MacAddress, address: Ipv4Addr, owned: bool) { + ARP_TABLE.v4.write().insert(interface, mac, address, owned); + } + + pub fn insert_address(interface: u32, mac: MacAddress, address: IpAddr, owned: bool) { + match address { + IpAddr::V4(address) => Self::insert_address_v4(interface, mac, address, owned), + IpAddr::V6(_) => todo!(), + } + ARP_TABLE.notify.wake_all(); + } + + fn poll_address( + interface: u32, + address: IpAddr, + timeout: Duration, + ) -> impl Future> { + struct F> { + interface: u32, + address: IpAddr, + timeout: Pin>, + } + + impl> Future for F { + type Output = Option; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.timeout.as_mut().poll(cx).is_ready() { + ARP_TABLE.notify.remove(cx.waker()); + return Poll::Ready(None); + } + ARP_TABLE.notify.register(cx.waker()); + if let Some(mac) = ArpTable::lookup_cache(self.interface, self.address) { + ARP_TABLE.notify.remove(cx.waker()); + Poll::Ready(Some(mac)) + } else { + Poll::Pending } } - IpAddr::V6(_) => todo!(), } - } - pub fn get_v4(&self, interface: u32, ip: Ipv4Addr) -> Option<(MacAddress, bool)> { - self.entries_v4.get(&(interface, ip)).cloned() + F { + interface, + address, + timeout: Box::pin(runtime::sleep(timeout)), + } } } -pub(crate) static ARP_TABLE: IrqSafeRwLock = IrqSafeRwLock::new(ArpTable::new()); +static ARP_TABLE: ArpTable = ArpTable::new(); -pub fn lookup(interface: u32, ip: IpAddr) -> Option { - // TODO send a request if MAC is not yet in the table - match ip { - IpAddr::V4(v4) => Some(ARP_TABLE.read().get_v4(interface, v4)?.0), +pub async fn lookup(interface: u32, ip: IpAddr, perform_query: bool) -> Result { + if let Some(mac) = ArpTable::lookup_cache(interface, ip) { + return Ok(mac); + } + if !perform_query { + return Err(Error::HostUnreachable); + } + query(interface, ip, 5, Duration::from_millis(200)).await +} + +async fn query( + interface: u32, + ip: IpAddr, + retries: usize, + retry_timeout: Duration, +) -> Result { + let interface = NetworkInterface::get(interface)?; + for _ in 0..retries { + send_request(&interface, ip)?; + if let Some(mac) = ArpTable::poll_address(interface.id, ip, retry_timeout).await { + return Ok(mac); + } + } + Err(Error::HostUnreachable) +} + +fn send_request_v4(interface: &NetworkInterface, query_address: Ipv4Addr) -> Result<(), Error> { + let request = ArpFrame { + protocol: EtherType::IPV4.to_network_order(), + protocol_size: 4, + hardware_type: u16::to_network_order(1), + hardware_size: 6, + opcode: 1u16.to_network_order(), + + sender_mac: interface.mac, + // TODO maybe would be nice to specify + sender_ip: 0u32.to_network_order(), + + target_ip: u32::to_network_order(query_address.into()), + target_mac: MacAddress::UNSPECIFIED, + }; + + ethernet::send_l2( + interface, + interface.mac, + MacAddress::BROADCAST, + EtherType::ARP, + &request, + ) +} + +fn send_request(interface: &NetworkInterface, query_address: IpAddr) -> Result<(), Error> { + log::debug!("Querying address of {}", query_address); + + match query_address { + IpAddr::V4(address) => send_request_v4(interface, address), IpAddr::V6(_) => todo!(), } } @@ -113,12 +252,11 @@ pub fn handle_packet(packet: L2Packet) { log::debug!("ARP: {} -> {}", sender_address, target_address); - let mut table = ARP_TABLE.write(); - table.insert_v4(packet.interface_id, arp.sender_mac, sender_address, false); + ArpTable::insert_address_v4(packet.interface_id, arp.sender_mac, sender_address, false); if opcode == 1 { // Don't answer with non-owned addresses - if let Some((mac, true)) = table.get_v4(packet.interface_id, target_address) { + if let Some((mac, true)) = ArpTable::lookup_cache_v4(packet.interface_id, target_address) { // Reply with own address send_reply(packet.interface_id, arp, mac).ok(); } diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 0eecb384..397a2aca 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -1,7 +1,7 @@ use core::{fmt, mem::size_of}; use alloc::{sync::Arc, vec::Vec}; -use bytemuck::Pod; +use bytemuck::{Pod, Zeroable}; use kernel_util::{ mem::PageBox, sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard}, @@ -11,11 +11,11 @@ use yggdrasil_abi::{ net::{ protocols::{EtherType, EthernetFrame, InetChecksum, IpProtocol, Ipv4Frame}, types::NetValueImpl, - IpAddr, Ipv4Addr, MacAddress, SubnetAddr, + IpAddr, Ipv4Addr, SubnetAddr, }, }; -use crate::{interface::NetworkInterface, l4}; +use crate::{interface::NetworkInterface, l4, PacketBuilder}; pub mod arp; pub mod ip; @@ -97,182 +97,93 @@ impl fmt::Display for Route { } } -pub fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { +pub async fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { match l3_packet.protocol { IpProtocol::UDP => l4::udp::handle(l3_packet), - IpProtocol::ICMP => l4::icmp::handle(l3_packet), + IpProtocol::ICMP => l4::icmp::handle(l3_packet).await, _ => Ok(()), } } -pub fn send_l4_ip( +fn resolve_l3_route( destination_ip: IpAddr, - protocol: IpProtocol, - l4_frame: &L4, - l4_data: &[u8], -) -> Result<(), Error> { - send_l4_ip_opt(destination_ip, protocol, l4_frame, &[], l4_data) +) -> Result<(Arc, IpAddr, IpAddr), Error> { + // Find the route itself + let (interface_id, gateway) = Route::lookup(destination_ip).ok_or(Error::NetworkUnreachable)?; + let interface = NetworkInterface::get(interface_id)?; + // Route exists, but has no gateway (TODO: assign subnets to interfaces) + let gateway = gateway.ok_or(Error::NetworkUnreachable)?; + // Route exists, but network has no address assigned (TODO: how?) + let source_address = interface.address.read().ok_or(Error::NetworkUnreachable)?; + + Ok((interface, source_address, gateway)) } -pub fn send_l4_ip_broadcast( - v6: bool, +fn make_ipv4_frame( protocol: IpProtocol, - l4_frame: &L4, - l4_data: &[u8], -) -> Result<(), Error> { - send_l4_ip_broadcast_opt(v6, protocol, l4_frame, &[], l4_data) -} + source_ip: Ipv4Addr, + destination_ip: Ipv4Addr, + payload_length: usize, + ttl: u8, +) -> Result { + let total_length = (payload_length + size_of::()) + .try_into() + .map_err(|_| Error::InvalidArgument)?; -pub fn send_l4_ip_broadcast_opt( - v6: bool, - protocol: IpProtocol, - l4_frame: &L4, - l4_options: &[u8], - l4_data: &[u8], -) -> Result<(), Error> { - let destination = match v6 { - true => todo!(), - false => IpAddr::V4(Ipv4Addr::BROADCAST), - }; - // TODO not sure how broadcasts work lol - for (_, interface) in NetworkInterface::list_ref().iter() { - let source = interface.address.read().unwrap_or_else(|| match v6 { - true => todo!(), - false => IpAddr::V4(Ipv4Addr::UNSPECIFIED), - }); - - if source.is_ipv6() && !v6 { - continue; - } - - let IpAddr::V4(source_v4) = source else { - todo!(); - }; - let IpAddr::V4(destination_v4) = destination else { - todo!(); - }; - - let destination_mac = MacAddress::BROADCAST; - - let l2_offset = interface.device.packet_prefix_size(); - let l3_offset = l2_offset + size_of::(); - let l4_offset = l3_offset + size_of::(); - let l4_data_offset = l4_offset + size_of::(); - - let mut packet = PageBox::new_slice(0, l4_data_offset + l4_data.len())?; - - let l2_frame: &mut EthernetFrame = - bytemuck::from_bytes_mut(&mut packet[l2_offset..l3_offset]); - - l2_frame.source_mac = interface.mac; - l2_frame.destination_mac = destination_mac; - l2_frame.ethertype = EtherType::IPV4.to_network_order(); - - let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); - - l3_frame.source_address = u32::to_network_order(u32::from(source_v4)); - l3_frame.destination_address = u32::to_network_order(u32::from(destination_v4)); - l3_frame.protocol = protocol; - l3_frame.version_length = 0x45; - l3_frame.dscp_flags = 0; - l3_frame.total_length = u16::to_network_order( - (size_of::() + size_of::() + l4_data.len()) - .try_into() - .unwrap(), - ); - // Disable fragmentation - l3_frame.flags_frag = u16::to_network_order(0x4000); - l3_frame.id = u16::to_network_order(0); - l3_frame.header_checksum = u16::to_network_order(0); - l3_frame.ttl = 64; - - let l3_frame_bytes = &packet[l3_offset..l4_offset]; - let mut ip_checksum = InetChecksum::new(); - ip_checksum.add_bytes(l3_frame_bytes, true); - let ip_checksum = ip_checksum.finish(); - - let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); - l3_frame.header_checksum = u16::to_network_order(ip_checksum); - - packet[l4_offset..l4_data_offset].copy_from_slice(bytemuck::bytes_of(l4_frame)); - packet[l4_data_offset..l4_data_offset + l4_options.len()].copy_from_slice(l4_options); - packet[l4_data_offset + l4_options.len()..].copy_from_slice(l4_data); - - if let Err(error) = interface.device.transmit(packet) { - log::warn!("Could not transmit on {}: {:?}", interface.name, error); - } - } - - Ok(()) -} - -pub fn send_l4_ip_opt( - destination_ip: IpAddr, - protocol: IpProtocol, - l4_frame: &L4, - l4_options: &[u8], - l4_data: &[u8], -) -> Result<(), Error> { - let IpAddr::V4(destination_v4) = destination_ip else { - todo!(); + let mut l3_frame = Ipv4Frame { + source_address: u32::to_network_order(source_ip.into()), + destination_address: u32::to_network_order(destination_ip.into()), + protocol, + version_length: 0x45, + total_length: u16::to_network_order(total_length), + flags_frag: u16::to_network_order(0x4000), + id: u16::to_network_order(0), + ttl, + ..Ipv4Frame::zeroed() }; - // Lookup route to destination - let (iface_id, gateway) = Route::lookup(destination_ip).unwrap(); - let iface = NetworkInterface::get(iface_id).unwrap(); - - let Some(IpAddr::V4(gateway_v4)) = gateway else { - todo!(); - }; - let Some(IpAddr::V4(source_v4)) = *iface.address.read() else { - todo!(); - }; - - // Lookup gateway MAC - let gateway_mac = arp::lookup(iface.id, IpAddr::V4(gateway_v4)).unwrap(); - - let l2_offset = iface.device.packet_prefix_size(); - let l3_offset = l2_offset + size_of::(); - let l4_offset = l3_offset + size_of::(); - let l4_data_offset = l4_offset + size_of::(); - - let mut packet = PageBox::new_slice(0, l4_data_offset + l4_data.len())?; - - let l2_frame: &mut EthernetFrame = bytemuck::from_bytes_mut(&mut packet[l2_offset..l3_offset]); - - l2_frame.source_mac = iface.mac; - l2_frame.destination_mac = gateway_mac; - l2_frame.ethertype = EtherType::IPV4.to_network_order(); - - let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); - - l3_frame.source_address = u32::to_network_order(u32::from(source_v4)); - l3_frame.destination_address = u32::to_network_order(u32::from(destination_v4)); - l3_frame.protocol = protocol; - l3_frame.version_length = 0x45; - l3_frame.dscp_flags = 0; - l3_frame.total_length = u16::to_network_order( - (size_of::() + size_of::() + l4_data.len()) - .try_into() - .unwrap(), - ); - // Disable fragmentation - l3_frame.flags_frag = u16::to_network_order(0x4000); - l3_frame.id = u16::to_network_order(0); - l3_frame.header_checksum = u16::to_network_order(0); - l3_frame.ttl = 64; - - let l3_frame_bytes = &packet[l3_offset..l4_offset]; + let l3_frame_bytes = bytemuck::bytes_of(&l3_frame); let mut ip_checksum = InetChecksum::new(); ip_checksum.add_bytes(l3_frame_bytes, true); let ip_checksum = ip_checksum.finish(); - let l3_frame: &mut Ipv4Frame = bytemuck::from_bytes_mut(&mut packet[l3_offset..l4_offset]); l3_frame.header_checksum = u16::to_network_order(ip_checksum); - packet[l4_offset..l4_data_offset].copy_from_slice(bytemuck::bytes_of(l4_frame)); - packet[l4_data_offset..l4_data_offset + l4_options.len()].copy_from_slice(l4_options); - packet[l4_data_offset + l4_options.len()..].copy_from_slice(l4_data); - - iface.device.transmit(packet) + Ok(l3_frame) +} + +pub async fn send_l4_ip( + destination_ip: IpAddr, + protocol: IpProtocol, + ttl: u8, + l4_frame: &L4, + l4_data: &[u8], +) -> Result<(), Error> { + let (interface, source_ip, gateway_ip) = resolve_l3_route(destination_ip)?; + let gateway_mac = arp::lookup(interface.id, gateway_ip, true).await?; + + // TODO what if source_ip/gateway_ip/destination_ip are a mix of IPv4/IPv6? + let source_ip = source_ip.into_ipv4().ok_or(Error::NotImplemented)?; + let destination_ip = destination_ip.into_ipv4().ok_or(Error::NotImplemented)?; + + let l3_frame = make_ipv4_frame( + protocol, + source_ip, + destination_ip, + size_of::() + l4_data.len(), + ttl, + )?; + + let mut builder = PacketBuilder::new(interface.device.packet_prefix_size())?; + builder.push(&EthernetFrame { + source_mac: interface.mac, + destination_mac: gateway_mac, + ethertype: EtherType::IPV4.to_network_order(), + })?; + builder.push(&l3_frame)?; + builder.push(l4_frame)?; + builder.push_bytes(l4_data)?; + + let (packet, _len) = builder.finish(); + interface.device.transmit(packet) } diff --git a/driver/net/core/src/l4/icmp.rs b/driver/net/core/src/l4/icmp.rs index db02cf6a..754d6c93 100644 --- a/driver/net/core/src/l4/icmp.rs +++ b/driver/net/core/src/l4/icmp.rs @@ -11,7 +11,7 @@ use yggdrasil_abi::{ use crate::{l3, L3Packet}; -fn send_v4_reply( +async fn send_v4_reply( destination_ip: Ipv4Addr, icmp_frame: &IcmpV4Frame, icmp_data: &[u8], @@ -37,12 +37,14 @@ fn send_v4_reply( l3::send_l4_ip( IpAddr::V4(destination_ip), IpProtocol::ICMP, + 255, &reply_frame, icmp_data, ) + .await } -fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), Error> { +async fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), Error> { if l3_packet.data_length < size_of::() { log::debug!("Truncated ICMPv4 packet"); return Err(Error::MissingData); @@ -58,7 +60,7 @@ fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), Error> let icmp_data = &l3_data[size_of::()..l3_packet.data_length]; match (icmp_frame.ty, icmp_frame.code) { - (8, 0) => send_v4_reply(source_address, icmp_frame, icmp_data), + (8, 0) => send_v4_reply(source_address, icmp_frame, icmp_data).await, _ => { log::debug!( "Ignoring unknown ICMPv4 type:code: {}:{}", @@ -70,9 +72,9 @@ fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), Error> } } -pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { +pub async fn handle(l3_packet: L3Packet) -> Result<(), Error> { match l3_packet.source_address { - IpAddr::V4(v4) => handle_v4(v4, l3_packet), + IpAddr::V4(v4) => handle_v4(v4, l3_packet).await, IpAddr::V6(_) => todo!(), } } diff --git a/driver/net/core/src/l4/mod.rs b/driver/net/core/src/l4/mod.rs index 2b647e5c..decb9436 100644 --- a/driver/net/core/src/l4/mod.rs +++ b/driver/net/core/src/l4/mod.rs @@ -1,2 +1,3 @@ pub mod icmp; +pub mod tcp; pub mod udp; diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs new file mode 100644 index 00000000..e69de29b diff --git a/driver/net/core/src/l4/udp.rs b/driver/net/core/src/l4/udp.rs index f07307f9..8e51918f 100644 --- a/driver/net/core/src/l4/udp.rs +++ b/driver/net/core/src/l4/udp.rs @@ -11,10 +11,11 @@ use yggdrasil_abi::{ use crate::{l3, socket::UdpSocket, L3Packet}; -pub fn send( +pub async fn send( source_port: u16, destination_ip: IpAddr, destination_port: u16, + ttl: u8, data: &[u8], ) -> Result<(), Error> { let length: u16 = (data.len() + size_of::()).try_into().unwrap(); @@ -25,25 +26,25 @@ pub fn send( checksum: 0u16.to_network_order(), }; - l3::send_l4_ip(destination_ip, IpProtocol::UDP, &udp_frame, data) + l3::send_l4_ip(destination_ip, IpProtocol::UDP, ttl, &udp_frame, data).await } -pub fn send_broadcast( - v6: bool, - source_port: u16, - destination_port: u16, - data: &[u8], -) -> Result<(), Error> { - let length: u16 = (data.len() + size_of::()).try_into().unwrap(); - let udp_frame = UdpFrame { - source_port: source_port.to_network_order(), - destination_port: destination_port.to_network_order(), - length: length.to_network_order(), - checksum: 0u16.to_network_order(), - }; - - l3::send_l4_ip_broadcast(v6, IpProtocol::UDP, &udp_frame, data) -} +// pub fn send_broadcast( +// v6: bool, +// source_port: u16, +// destination_port: u16, +// data: &[u8], +// ) -> Result<(), Error> { +// let length: u16 = (data.len() + size_of::()).try_into().unwrap(); +// let udp_frame = UdpFrame { +// source_port: source_port.to_network_order(), +// destination_port: destination_port.to_network_order(), +// length: length.to_network_order(), +// checksum: 0u16.to_network_order(), +// }; +// +// l3::send_l4_ip_broadcast(v6, IpProtocol::UDP, &udp_frame, data) +// } pub fn handle(l3_packet: L3Packet) -> Result<(), Error> { if l3_packet.data_length < size_of::() { diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index f77e9432..044c6424 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -7,6 +7,7 @@ extern crate alloc; use core::mem::size_of; use alloc::sync::Arc; +use bytemuck::Pod; use ethernet::L2Packet; use kernel_util::{mem::PageBox, runtime}; use l3::L3Packet; @@ -32,6 +33,41 @@ pub struct Packet { iface: u32, } +pub struct PacketBuilder { + data: PageBox<[u8]>, + pos: usize, + len: usize, +} + +impl PacketBuilder { + pub fn new(l2_offset: usize) -> Result { + let data = PageBox::new_slice(0, 4096)?; + Ok(Self { + data, + pos: l2_offset, + len: l2_offset, + }) + } + + #[inline] + pub fn push(&mut self, value: &T) -> Result<(), Error> { + self.push_bytes(bytemuck::bytes_of(value)) + } + + pub fn push_bytes(&mut self, bytes: &[u8]) -> Result<(), Error> { + if self.pos + bytes.len() > self.data.len() { + return Err(Error::OutOfMemory); + } + self.data[self.pos..self.pos + bytes.len()].copy_from_slice(bytes); + self.pos += bytes.len(); + Ok(()) + } + + pub fn finish(self) -> (PageBox<[u8]>, usize) { + (self.data, self.len) + } +} + impl Packet { #[inline] pub fn new(buffer: PageBox<[u8]>, offset: usize, iface: u32) -> Self { @@ -91,16 +127,16 @@ async fn l3_accept_worker() { loop { let l3_packet = ACCEPT_QUEUE.wait().await; - log::debug!( - "INPUT {} {}:{:?} -> {}:{:?}: ACCEPT", - l3_packet.protocol, - l3_packet.source_address, - l3_packet.source_port, - l3_packet.destination_address, - l3_packet.destination_port - ); + // log::debug!( + // "INPUT {} {}:{:?} -> {}:{:?}: ACCEPT", + // l3_packet.protocol, + // l3_packet.source_address, + // l3_packet.source_port, + // l3_packet.destination_address, + // l3_packet.destination_port + // ); - if let Err(error) = l3::handle_accepted(l3_packet) { + if let Err(error) = l3::handle_accepted(l3_packet).await { log::error!("L3 handle error: {:?}", error); } } diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index a9fa01a2..d4e162b4 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -20,7 +20,7 @@ use kernel_util::{ use vfs::{FileReadiness, PacketSocket, Socket}; use yggdrasil_abi::{ error::Error, - net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketInterfaceQuery, SocketOption}, + net::{Ipv4Addr, SocketAddr, SocketAddrV4, SocketInterfaceQuery, SocketOption}, }; use crate::{ethernet::L2Packet, interface::NetworkInterface, l4}; @@ -185,17 +185,18 @@ impl PacketSocket for UdpSocket { // TODO check that destnation family matches self family match (self.broadcast.load(Ordering::Relaxed), destination.ip()) { // SendTo in broadcast? - (true, IpAddr::V4(Ipv4Addr::BROADCAST)) => { - l4::udp::send_broadcast(false, self.local.port(), destination.port(), data)?; - } (true, _) => todo!(), (false, _) => { - l4::udp::send( - self.local.port(), - destination.ip(), - destination.port(), - data, - )?; + block!( + l4::udp::send( + self.local.port(), + destination.ip(), + destination.port(), + 64, + data, + ) + .await + )??; } } diff --git a/driver/virtio/core/src/queue.rs b/driver/virtio/core/src/queue.rs index ac3db8c5..4043b556 100644 --- a/driver/virtio/core/src/queue.rs +++ b/driver/virtio/core/src/queue.rs @@ -252,6 +252,8 @@ impl VirtQueue { self.used.used_count += input.len() + output.len(); + fence(Ordering::SeqCst); + head } @@ -269,6 +271,8 @@ impl VirtQueue { core::hint::spin_loop(); } + fence(Ordering::SeqCst); + unsafe { self.pop_used(token) } } @@ -308,6 +312,8 @@ impl VirtQueue { self.free_descriptor_chain(token); + fence(Ordering::SeqCst); + self.last_used_idx = self.last_used_idx.wrapping_add(1); Ok(len) From f5ba40eea6ab9cf3e1775d6a1413823327f3bbda Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 23 Jan 2024 00:04:20 +0200 Subject: [PATCH 156/211] net: don't transmit huge packets from raw socket --- driver/net/core/src/socket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index d4e162b4..17c4e629 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -402,7 +402,7 @@ impl PacketSocket for RawSocket { if data.len() > 4096 - l2_offset { return Err(Error::InvalidArgument); } - let mut packet = PageBox::new_slice(0, 4096)?; + let mut packet = PageBox::new_slice(0, l2_offset + data.len())?; packet[l2_offset..l2_offset + data.len()].copy_from_slice(data); interface.device.transmit(packet)?; Ok(data.len()) From 05cb6a2994082929040b3cf15963e259b4f2c84b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 24 Jan 2024 17:48:09 +0200 Subject: [PATCH 157/211] net: simple TCP listener/socket implementation --- driver/net/core/src/l3/ip.rs | 10 +- driver/net/core/src/l3/mod.rs | 1 + driver/net/core/src/l4/tcp.rs | 595 ++++++++++++++++++++++++++++++++++ driver/net/core/src/socket.rs | 401 ++++++++++++++++++++++- lib/vfs/src/file/mod.rs | 73 ++++- lib/vfs/src/socket.rs | 43 ++- src/syscall/mod.rs | 33 +- 7 files changed, 1125 insertions(+), 31 deletions(-) diff --git a/driver/net/core/src/l3/ip.rs b/driver/net/core/src/l3/ip.rs index 2a75b640..c0ae9b3d 100644 --- a/driver/net/core/src/l3/ip.rs +++ b/driver/net/core/src/l3/ip.rs @@ -1,7 +1,7 @@ use core::mem::size_of; use yggdrasil_abi::net::{ - protocols::{IpProtocol, Ipv4Frame, UdpFrame}, + protocols::{IpProtocol, Ipv4Frame, TcpFrame, UdpFrame}, types::NetValueImpl, IpAddr, Ipv4Addr, }; @@ -54,6 +54,14 @@ pub fn handle_v4_packet(packet: L2Packet) { Some(u16::from_network_order(l4_frame.destination_port)), ) } + IpProtocol::TCP => { + // TODO check size + let l4_frame: &TcpFrame = bytemuck::from_bytes(&l3_data[..size_of::()]); + ( + Some(u16::from_network_order(l4_frame.source_port)), + Some(u16::from_network_order(l4_frame.destination_port)), + ) + } IpProtocol::ICMP => (None, None), _ => (None, None), }; diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 397a2aca..89c4e011 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -101,6 +101,7 @@ pub async fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { match l3_packet.protocol { IpProtocol::UDP => l4::udp::handle(l3_packet), IpProtocol::ICMP => l4::icmp::handle(l3_packet).await, + IpProtocol::TCP => l4::tcp::handle(l3_packet).await, _ => Ok(()), } } diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index e69de29b..e6ad274d 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -0,0 +1,595 @@ +use core::{ + mem::size_of, + task::{Context, Poll}, +}; + +use alloc::{vec, vec::Vec}; +use bytemuck::Zeroable; +use kernel_util::runtime::QueueWaker; +use yggdrasil_abi::{ + error::Error, + net::{ + protocols::{InetChecksum, IpProtocol, TcpFlags, TcpFrame, TcpV4PseudoHeader}, + types::NetValueImpl, + SocketAddr, + }, +}; + +use crate::{ + l3::{self, L3Packet}, + socket::{TcpListener, TcpSocket}, +}; + +#[derive(PartialEq, Debug)] +pub enum TcpConnectionState { + SynSent, + SynReceived, + Established, + FinWait1, + FinWait2, + Closed, +} + +pub enum TcpSocketBehavior { + None, + Accept, + Remove, +} + +struct SocketBuffer { + data: Vec, + wr: usize, + rd: usize, +} + +pub struct TcpConnection { + state: TcpConnectionState, + + local: SocketAddr, + remote: SocketAddr, + + rx_window_size: u16, + + // Rx half + // RCV.WND = rx_buffer.capacity() + // RCV.NXT = rx_window_start + rx_window.len() + // TODO RCV.UP + rx_buffer: SocketBuffer, + + // Relative RX sequence number of window start + rx_window_start: usize, + + // Tx half + // SND.UNA = tx_window_start + tx_sent_unacknowledged + // SND.WND = tx_buffer.capacity() + // SND.NXT = tx_window_start IF tx_sent_unacknowledged == 0 + tx_buffer: Vec, + + tx_window_start: usize, + tx_sent_unacknowledged: usize, + + // IRS + initial_rx_seq: u32, + initial_tx_seq: u32, + + rx_notify: QueueWaker, + tx_notify: QueueWaker, +} + +#[allow(unused)] +struct TcpPacket { + local: SocketAddr, + remote: SocketAddr, + seq: u32, + ack: u32, + window_size: u16, + flags: TcpFlags, +} + +impl SocketBuffer { + pub fn with_capacity(capacity: usize) -> Self { + Self { + data: vec![0; capacity], + wr: 0, + rd: 0, + } + } + + pub fn len(&self) -> usize { + if self.wr >= self.rd { + self.wr - self.rd + } else { + self.wr + self.capacity() - self.rd + } + } + + pub fn capacity(&self) -> usize { + self.data.len() + } + + pub fn can_read(&self) -> bool { + self.rd != self.wr + } + + pub fn write(&mut self, data: &[u8]) { + for &byte in data { + self.putc(byte); + } + } + + pub fn putc(&mut self, data: u8) { + if (self.wr + 1) % self.capacity() == self.rd { + self.rd = (self.rd + 1) % self.capacity(); + } + self.data[self.wr] = data; + self.wr = (self.wr + 1) % self.capacity(); + } + + pub fn read(&mut self, buffer: &mut [u8]) -> usize { + let mut amount = 0; + while amount != buffer.len() { + if self.rd == self.wr { + break; + } + buffer[amount] = self.data[self.rd]; + self.rd = (self.rd + 1) % self.capacity(); + amount += 1; + } + amount + } +} + +impl TcpConnection { + pub fn new( + local: SocketAddr, + remote: SocketAddr, + window_size: usize, + tx_seq: u32, + rx_seq: u32, + state: TcpConnectionState, + ) -> Self { + debug_assert!( + state == TcpConnectionState::SynSent || state == TcpConnectionState::SynReceived + ); + debug_assert!(window_size < u16::MAX as usize); + Self { + state, + + local, + remote, + + rx_buffer: SocketBuffer::with_capacity(window_size), + rx_window_start: 1, + + tx_buffer: Vec::with_capacity(window_size), + tx_window_start: 1, + tx_sent_unacknowledged: 0, + + rx_window_size: window_size as u16, + + initial_rx_seq: rx_seq, + initial_tx_seq: tx_seq, + + rx_notify: QueueWaker::new(), + tx_notify: QueueWaker::new(), + } + } + + fn ack_number(&self) -> u32 { + (self.initial_rx_seq as usize + self.rx_window_start + self.rx_buffer.len()) as u32 + } + + fn seq_number(&self) -> u32 { + (self.initial_tx_seq as usize + self.tx_window_start) as u32 + } + + pub fn is_closing(&self) -> bool { + self.state == TcpConnectionState::FinWait1 + || self.state == TcpConnectionState::FinWait2 + || self.state == TcpConnectionState::Closed + } + + pub fn is_closed(&self) -> bool { + self.state == TcpConnectionState::Closed + } + + pub fn read_nonblocking(&mut self, buffer: &mut [u8]) -> Result { + let amount = self.rx_buffer.read(buffer); + if amount == 0 && self.state == TcpConnectionState::Established { + // TODO error + return Err(Error::DoesNotExist); + } + self.rx_window_start += amount; + Ok(amount) + } + + pub(crate) fn poll_receive(&self, cx: &mut Context<'_>) -> Poll> { + self.rx_notify.register(cx.waker()); + if self.rx_buffer.can_read() { + self.rx_notify.remove(cx.waker()); + Poll::Ready(Ok(())) + } else if self.state != TcpConnectionState::Established { + self.rx_notify.remove(cx.waker()); + // TODO error + Poll::Ready(Err(Error::DoesNotExist)) + } else { + Poll::Pending + } + } + + pub(crate) fn poll_send(&self, cx: &mut Context<'_>) -> Poll> { + self.tx_notify.register(cx.waker()); + if self.state == TcpConnectionState::Closed { + self.tx_notify.remove(cx.waker()); + // TODO error + Poll::Ready(Err(Error::DoesNotExist)) + } else if self.tx_sent_unacknowledged == 0 { + self.tx_notify.remove(cx.waker()); + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } + + pub(crate) fn poll_acknowledge(&self, cx: &mut Context<'_>) -> Poll<()> { + self.tx_notify.register(cx.waker()); + if self.tx_sent_unacknowledged == 0 { + self.tx_notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + + pub(crate) fn poll_finish(&self, cx: &mut Context<'_>) -> Poll<()> { + self.tx_notify.register(cx.waker()); + if self.state == TcpConnectionState::FinWait2 || self.state == TcpConnectionState::Closed { + self.tx_notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + + pub(crate) async fn transmit(&mut self, data: &[u8]) -> Result<(), Error> { + assert_eq!(self.tx_sent_unacknowledged, 0); + assert_eq!(self.tx_buffer.len(), 0); + + self.tx_buffer.extend_from_slice(data); + self.tx_sent_unacknowledged = data.len(); + + send( + self.local, + self.remote, + self.seq_number(), + self.ack_number(), + self.rx_window_size, + TcpFlags::ACK, + data, + ) + .await + } + + pub(crate) async fn finish(&mut self) -> Result<(), Error> { + assert_eq!(self.tx_sent_unacknowledged, 0); + assert_eq!(self.tx_buffer.len(), 0); + + log::debug!("Finish connection {} <-> {}", self.local, self.remote); + send( + self.local, + self.remote, + self.seq_number(), + self.ack_number(), + self.rx_window_size, + TcpFlags::FIN | TcpFlags::ACK, + &[], + ) + .await?; + + self.state = TcpConnectionState::FinWait1; + + Ok(()) + } + + pub(crate) fn notify_all(&self) { + self.rx_notify.wake_all(); + self.tx_notify.wake_all(); + } + + async fn handle_packet( + &mut self, + packet: TcpPacket, + data: &[u8], + ) -> Result { + // TODO what if window_size changes? + + match self.state { + TcpConnectionState::FinWait1 => { + // Check if connection close initiated locally got ACKed by remote + // TODO check ack/seq + if packet.flags == TcpFlags::FIN | TcpFlags::ACK { + self.state = TcpConnectionState::Closed; + send( + self.local, + self.remote, + self.seq_number() + 1, + self.ack_number() + 1, + self.rx_window_size, + TcpFlags::ACK, + &[], + ) + .await?; + + // Socket fully closed, remove from table + return Ok(TcpSocketBehavior::Remove); + } + if packet.flags == TcpFlags::ACK { + self.state = TcpConnectionState::FinWait2; + self.tx_notify.wake_all(); + } + return Ok(TcpSocketBehavior::None); + } + TcpConnectionState::FinWait2 => { + if packet.flags == TcpFlags::FIN | TcpFlags::ACK { + self.state = TcpConnectionState::Closed; + send( + self.local, + self.remote, + self.seq_number() + 1, + self.ack_number() + 1, + self.rx_window_size, + TcpFlags::ACK, + &[], + ) + .await?; + + // Socket fully closed, remove from table + return Ok(TcpSocketBehavior::Remove); + } + + return Ok(TcpSocketBehavior::None); + } + TcpConnectionState::Closed => { + log::warn!("Packet received on closed connection"); + return Ok(TcpSocketBehavior::None); + } + TcpConnectionState::SynReceived => { + // TODO check ack/seq + // Handshake continuation expected (ACK) + if packet.flags == TcpFlags::ACK { + self.state = TcpConnectionState::Established; + return Ok(TcpSocketBehavior::Accept); + } + + return Err(Error::InvalidArgument); + } + TcpConnectionState::SynSent => todo!(), + TcpConnectionState::Established => (), + } + + let mut reply_flags = TcpFlags::empty(); + + if self.tx_sent_unacknowledged != 0 { + let tx_acknowledge_end = packet.ack.wrapping_sub(self.initial_tx_seq) as usize; + + if tx_acknowledge_end == self.tx_window_start + self.tx_sent_unacknowledged { + self.tx_window_start += self.tx_sent_unacknowledged; + self.tx_sent_unacknowledged = 0; + self.tx_buffer.clear(); + self.tx_notify.wake_one(); + } + } + + if data.len() != 0 { + // Local side + let rx_window_start = self.rx_window_start + self.rx_buffer.len(); + let rx_window_end = self.rx_window_start + self.rx_buffer.capacity(); + // Remote side + let rx_segment_start = packet.seq.wrapping_sub(self.initial_rx_seq) as usize; + let rx_segment_end = rx_segment_start + data.len(); + + if rx_segment_end > rx_window_end || rx_segment_start < rx_window_start { + log::warn!( + "Segment {:?} outside of Rx window {:?}", + rx_segment_start..rx_segment_end, + rx_window_start..rx_window_end + ); + return Ok(TcpSocketBehavior::None); + } + if rx_segment_start != rx_window_start { + log::warn!( + "Out-of-order segment {:?} received, will not accept into {:?}", + rx_segment_start..rx_segment_end, + rx_window_start..rx_window_end + ); + return Ok(TcpSocketBehavior::None); + } + + // segment_start == window_start + let amount = data.len().min(rx_window_end - rx_window_start); + + if amount != 0 { + log::trace!( + "Received segment: {:?}, current window {:?}", + rx_segment_start..rx_segment_end, + rx_window_start..rx_window_end + ); + + self.rx_buffer.write(&data[..amount]); + self.rx_notify.wake_one(); + reply_flags |= TcpFlags::ACK; + } + } + + let mut ack_number = self.ack_number(); + let mut behavior = TcpSocketBehavior::None; + + if packet.flags.contains(TcpFlags::FIN) { + // TODO wait for ACK? + reply_flags |= TcpFlags::FIN | TcpFlags::ACK; + ack_number = ack_number.wrapping_add(1); + self.state = TcpConnectionState::Closed; + log::trace!( + "TCP connection FIN requested by remote: {} <-> {}", + self.local, + self.remote + ); + + behavior = TcpSocketBehavior::Remove; + } + + if reply_flags != TcpFlags::empty() { + send( + packet.local, + packet.remote, + self.seq_number(), + ack_number, + self.rx_window_size, + reply_flags, + &[], + ) + .await?; + } + + Ok(behavior) + } +} + +async fn send( + local: SocketAddr, + remote: SocketAddr, + seq: u32, + ack: u32, + window_size: u16, + flags: TcpFlags, + data: &[u8], +) -> Result<(), Error> { + // TODO TCPv6 + let local = local.into_ipv4().unwrap(); + let remote = remote.into_ipv4().unwrap(); + + let tcp_length = size_of::() + data.len(); + + let mut frame = TcpFrame { + source_port: local.port().to_network_order(), + destination_port: remote.port().to_network_order(), + sequence_number: seq.to_network_order(), + acknowledge_number: ack.to_network_order(), + data_offset: 5 << 4, + window_size: window_size.to_network_order(), + flags, + ..TcpFrame::zeroed() + }; + let pseudo_header = TcpV4PseudoHeader { + source_address: u32::from(local.ip()).to_network_order(), + destination_address: u32::from(remote.ip()).to_network_order(), + _zero: 0, + protocol: IpProtocol::TCP, + tcp_length: (tcp_length as u16).to_network_order(), + }; + + let mut checksum = InetChecksum::new(); + checksum.add_value(&pseudo_header, true); + checksum.add_value(&frame, true); + checksum.add_bytes(data, true); + let checksum = checksum.finish(); + + frame.checksum = checksum.to_network_order(); + + l3::send_l4_ip(remote.ip().into(), IpProtocol::TCP, 64, &frame, data).await +} + +pub async fn handle(packet: L3Packet) -> Result<(), Error> { + if packet.data_length < size_of::() { + log::warn!("Truncated TCP packet"); + return Ok(()); + } + + let l3_data = packet.l3_data(); + + let tcp_frame: &TcpFrame = bytemuck::from_bytes(&l3_data[..size_of::()]); + let tcp_data_offset = tcp_frame.data_offset(); + let tcp_data = &l3_data[tcp_data_offset..packet.data_length]; + + let remote = SocketAddr::new( + packet.source_address, + u16::from_network_order(tcp_frame.source_port), + ); + let local = SocketAddr::new( + packet.destination_address, + u16::from_network_order(tcp_frame.destination_port), + ); + + let seq = u32::from_network_order(tcp_frame.sequence_number); + let ack = u32::from_network_order(tcp_frame.acknowledge_number); + + // TODO validate checksum + + match tcp_frame.flags { + TcpFlags::SYN => { + if let Some(listener) = TcpListener::get(local) { + let window_size = u16::from_network_order(tcp_frame.window_size); + let tx_seq = 12345; + + // Create a socket and insert it into the table + TcpSocket::accept_remote( + listener.clone(), + local, + remote, + window_size as usize, + tx_seq, + seq, + )?; + + // Send SYN+ACK + send( + local, + remote, + tx_seq, + seq.wrapping_add(1), + window_size, + TcpFlags::SYN | TcpFlags::ACK, + &[], + ) + .await + } else { + // RST+ACK + let window_size = u16::from_network_order(tcp_frame.window_size); + send( + local, + remote, + 0, + seq.wrapping_add(1), + window_size, + TcpFlags::RST | TcpFlags::ACK, + &[], + ) + .await + } + } + _ => { + let packet = TcpPacket { + local, + remote, + window_size: u16::from_network_order(tcp_frame.window_size), + flags: tcp_frame.flags, + ack, + seq, + }; + + let socket = TcpSocket::get(local, remote).ok_or(Error::DoesNotExist)?; + let mut connection = socket.connection().write(); + match connection.handle_packet(packet, tcp_data).await? { + TcpSocketBehavior::None => (), + TcpSocketBehavior::Accept => { + socket.accept(); + } + TcpSocketBehavior::Remove => { + drop(connection); + socket.remove_socket()?; + } + } + Ok(()) + } + } +} diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 17c4e629..585efb39 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -1,5 +1,5 @@ use core::{ - future::Future, + future::{poll_fn, Future}, pin::Pin, sync::atomic::{AtomicBool, AtomicU32, Ordering}, task::{Context, Poll}, @@ -12,18 +12,25 @@ use kernel_util::{ runtime::QueueWaker, sync::{ mutex::{Mutex, MutexGuard}, - spin_rwlock::IrqSafeRwLock, - IrqSafeSpinlock, LockMethod, + spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, + IrqSafeSpinlock, IrqSafeSpinlockGuard, LockMethod, }, util::ring::RingBuffer, }; -use vfs::{FileReadiness, PacketSocket, Socket}; +use vfs::{ConnectionSocket, FileReadiness, ListenerSocket, PacketSocket, Socket}; use yggdrasil_abi::{ error::Error, - net::{Ipv4Addr, SocketAddr, SocketAddrV4, SocketInterfaceQuery, SocketOption}, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4, SocketInterfaceQuery, SocketOption}, }; -use crate::{ethernet::L2Packet, interface::NetworkInterface, l4}; +use crate::{ + ethernet::L2Packet, + interface::NetworkInterface, + l4::{ + self, + tcp::{TcpConnection, TcpConnectionState}, + }, +}; pub struct UdpSocket { local: SocketAddr, @@ -36,6 +43,23 @@ pub struct UdpSocket { receive_notify: QueueWaker, } +pub struct TcpSocket { + local: SocketAddr, + remote: SocketAddr, + // Listener which accepted the socket + listener: Option>, + connection: IrqSafeRwLock, +} + +pub struct TcpListener { + accept: SocketAddr, + + // Currently active sockets + sockets: IrqSafeRwLock>>, + pending_accept: IrqSafeSpinlock>>, + accept_notify: QueueWaker, +} + pub struct RawSocket { id: u32, bound: IrqSafeSpinlock>, @@ -47,6 +71,43 @@ pub struct SocketTable { inner: BTreeMap>, } +pub struct TwoWaySocketTable { + inner: BTreeMap<(SocketAddr, SocketAddr), Arc>, +} + +impl TwoWaySocketTable { + pub const fn new() -> Self { + Self { + inner: BTreeMap::new(), + } + } + + pub fn try_insert_with Arc>( + &mut self, + local: SocketAddr, + remote: SocketAddr, + with: F, + ) -> Result, Error> { + if self.inner.contains_key(&(local, remote)) { + return Err(Error::AlreadyExists); + } + let socket = with(); + self.inner.insert((local, remote), socket.clone()); + Ok(socket) + } + + pub fn remove(&mut self, local: SocketAddr, remote: SocketAddr) -> Result<(), Error> { + match self.inner.remove(&(local, remote)) { + Some(_) => Ok(()), + None => Err(Error::DoesNotExist), + } + } + + pub fn get(&self, local: SocketAddr, remote: SocketAddr) -> Option> { + self.inner.get(&(local, remote)).cloned() + } +} + impl SocketTable { pub const fn new() -> Self { Self { @@ -54,6 +115,19 @@ impl SocketTable { } } + pub fn try_insert_with Arc>( + &mut self, + address: SocketAddr, + with: F, + ) -> Result, Error> { + if self.inner.contains_key(&address) { + return Err(Error::AlreadyExists); + } + let socket = with(); + self.inner.insert(address, socket.clone()); + Ok(socket) + } + pub fn insert(&mut self, address: SocketAddr, socket: Arc) -> Result<(), Error> { match self.inner.try_insert(address, socket) { Ok(_) => Ok(()), @@ -88,11 +162,15 @@ impl SocketTable { } static UDP_SOCKETS: IrqSafeRwLock> = IrqSafeRwLock::new(SocketTable::new()); +static TCP_SOCKETS: IrqSafeRwLock> = + IrqSafeRwLock::new(TwoWaySocketTable::new()); static RAW_SOCKET_ID: AtomicU32 = AtomicU32::new(0); static RAW_SOCKETS: IrqSafeRwLock>> = IrqSafeRwLock::new(BTreeMap::new()); static BOUND_RAW_SOCKETS: IrqSafeRwLock>> = IrqSafeRwLock::new(BTreeMap::new()); +static TCP_LISTENERS: IrqSafeRwLock> = + IrqSafeRwLock::new(SocketTable::new()); impl UdpSocket { pub fn bind(address: SocketAddr) -> Result, Error> { @@ -396,6 +474,7 @@ impl Socket for RawSocket { impl PacketSocket for RawSocket { fn send(&self, _destination: Option, data: &[u8]) -> Result { + // TODO cap by MTU? let bound = self.bound.lock().ok_or(Error::InvalidOperation)?; let interface = NetworkInterface::get(bound).unwrap(); let l2_offset = interface.device.packet_prefix_size(); @@ -421,3 +500,313 @@ impl PacketSocket for RawSocket { )) } } + +impl TcpSocket { + pub async fn connect(_remote: IpAddr) -> Result, Error> { + todo!() + } + + pub fn accept_remote( + listener: Arc, + local: SocketAddr, + remote: SocketAddr, + remote_window_size: usize, + tx_seq: u32, + rx_seq: u32, + ) -> Result, Error> { + let mut sockets = TCP_SOCKETS.write(); + sockets.try_insert_with(local, remote, move || { + let connection = TcpConnection::new( + local, + remote, + remote_window_size, + tx_seq, + rx_seq, + TcpConnectionState::SynReceived, + ); + + log::debug!("Accepted TCP socket {} -> {}", local, remote); + + let socket = Self { + local, + remote, + listener: Some(listener), + connection: IrqSafeRwLock::new(connection), + }; + + Arc::new(socket) + }) + } + + pub fn connection(&self) -> &IrqSafeRwLock { + &self.connection + } + + pub(crate) fn accept(self: &Arc) { + if let Some(listener) = self.listener.as_ref() { + listener.accept_socket(self.clone()); + } + } + + pub fn get(local: SocketAddr, remote: SocketAddr) -> Option> { + TCP_SOCKETS.read().get(local, remote) + } + + pub fn receive_async<'a>( + &'a self, + buffer: &'a mut [u8], + ) -> impl Future> + 'a { + // TODO timeout here + // TODO don't throw ConnectionReset immediately + struct F<'f> { + socket: &'f TcpSocket, + buffer: &'f mut [u8], + } + + impl<'f> Future for F<'f> { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.socket.poll_receive(cx) { + Poll::Ready(Ok(mut lock)) => Poll::Ready(lock.read_nonblocking(self.buffer)), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + } + } + + F { + socket: self, + buffer, + } + } + + async fn send_segment_async(&self, data: &[u8]) -> Result<(), Error> { + // TODO timeout here + { + let mut connection = poll_fn(|cx| { + let connection = self.connection.write(); + match connection.poll_send(cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(connection)), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + }) + .await?; + + connection.transmit(data).await?; + } + + poll_fn(|cx| { + let connection = self.connection.read(); + connection.poll_acknowledge(cx) + }) + .await; + + Ok(()) + } + + pub async fn send_async(&self, data: &[u8]) -> Result { + let mut pos = 0; + let mut rem = data.len(); + while rem != 0 { + // TODO check MTU + let amount = rem.min(512); + self.send_segment_async(&data[pos..pos + amount]).await?; + pos += amount; + rem -= amount; + } + Ok(pos) + } + + pub async fn close_async(&self, remove_from_listener: bool) -> Result<(), Error> { + // TODO timeout here + // Already closing + if self.connection.read().is_closing() { + return Ok(()); + } + + // Wait for all sent data to be acknowledged + { + let mut connection = poll_fn(|cx| { + let connection = self.connection.write(); + match connection.poll_send(cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(connection)), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + }) + .await?; + + connection.finish().await?; + } + + log::debug!( + "TCP socket closed (FinWait2/Closed): {} <-> {}", + self.local, + self.remote + ); + + // Wait for connection to get closed + poll_fn(|cx| { + let connection = self.connection.read(); + connection.poll_finish(cx) + }) + .await; + + if remove_from_listener { + if let Some(listener) = self.listener.as_ref() { + listener.remove_socket(self.remote); + }; + } + + Ok(()) + } + + pub(crate) fn remove_socket(&self) -> Result<(), Error> { + log::debug!( + "TCP socket closed and removed: {} <-> {}", + self.local, + self.remote + ); + let connection = self.connection.read(); + debug_assert!(connection.is_closed()); + TCP_SOCKETS.write().remove(self.local, self.remote)?; + connection.notify_all(); + Ok(()) + } + + fn poll_receive( + &self, + cx: &mut Context<'_>, + ) -> Poll, Error>> { + let lock = self.connection.write(); + match lock.poll_receive(cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(lock)), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + } +} + +impl Socket for TcpSocket { + fn local_address(&self) -> SocketAddr { + self.local + } + + fn remote_address(&self) -> Option { + Some(self.remote) + } + + fn close(&self) -> Result<(), Error> { + block!(self.close_async(true).await)? + } +} + +impl FileReadiness for TcpSocket { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + self.poll_receive(cx).map_ok(|_| ()) + } +} + +impl ConnectionSocket for TcpSocket { + fn receive(&self, buffer: &mut [u8]) -> Result { + block!(self.receive_async(buffer).await)? + } + + fn send(&self, data: &[u8]) -> Result { + block!(self.send_async(data).await)? + } +} + +impl TcpListener { + pub fn bind(accept: SocketAddr) -> Result, Error> { + TCP_LISTENERS.write().try_insert_with(accept, || { + let listener = TcpListener { + accept, + sockets: IrqSafeRwLock::new(BTreeMap::new()), + pending_accept: IrqSafeSpinlock::new(Vec::new()), + accept_notify: QueueWaker::new(), + }; + + log::debug!("TCP Listener opened: {}", accept); + + Arc::new(listener) + }) + } + + pub fn get(local: SocketAddr) -> Option> { + TCP_LISTENERS.read().get(&local) + } + + pub fn accept_async(&self) -> impl Future, Error>> + '_ { + struct F<'f> { + listener: &'f TcpListener, + } + + impl<'f> Future for F<'f> { + type Output = Result, Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.listener.poll_accept(cx) { + Poll::Ready(mut lock) => Poll::Ready(Ok(lock.pop().unwrap())), + Poll::Pending => Poll::Pending, + } + } + } + + F { listener: self } + } + + fn accept_socket(&self, socket: Arc) { + log::debug!("{}: accept {}", self.accept, socket.remote); + self.sockets.write().insert(socket.remote, socket.clone()); + self.pending_accept.lock().push(socket); + self.accept_notify.wake_one(); + } + + fn remove_socket(&self, remote: SocketAddr) { + log::debug!("Remove client {}", remote); + self.sockets.write().remove(&remote); + } + + fn poll_accept(&self, cx: &mut Context<'_>) -> Poll>>> { + let lock = self.pending_accept.lock(); + self.accept_notify.register(cx.waker()); + if !lock.is_empty() { + self.accept_notify.remove(cx.waker()); + Poll::Ready(lock) + } else { + Poll::Pending + } + } +} + +impl Socket for TcpListener { + fn local_address(&self) -> SocketAddr { + self.accept + } + + fn remote_address(&self) -> Option { + None + } + + fn close(&self) -> Result<(), Error> { + // TODO if clients not closed already, send RST? + TCP_LISTENERS.write().remove(self.accept) + } +} + +impl FileReadiness for TcpListener { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + self.poll_accept(cx).map(|_| Ok(())) + } +} + +impl ListenerSocket for TcpListener { + fn accept(&self) -> Result<(SocketAddr, Arc), Error> { + let socket = block!(self.accept_async().await)??; + let remote = socket.remote; + Ok((remote, socket)) + } +} diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index b2a22034..36fc7b24 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -25,10 +25,10 @@ use crate::{ channel::ChannelDescriptor, device::{BlockDeviceWrapper, CharDeviceWrapper}, node::NodeRef, - socket::PacketSocketWrapper, + socket::{ConnectionSocketWrapper, ListenerSocketWrapper, PacketSocketWrapper}, traits::{Read, Seek, Write}, - FdPoll, FileReadiness, PacketSocket, PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave, - SharedMemory, Socket, + ConnectionSocket, FdPoll, FileReadiness, ListenerSocket, PacketSocket, PseudoTerminal, + PseudoTerminalMaster, PseudoTerminalSlave, SharedMemory, Socket, }; use self::{ @@ -67,6 +67,8 @@ pub enum File { Char(CharFile), PacketSocket(Arc), + ListenerSocket(Arc), + StreamSocket(Arc), AnonymousPipe(PipeEnd), Poll(FdPoll), @@ -125,6 +127,20 @@ impl File { Arc::new(Self::PacketSocket(Arc::new(PacketSocketWrapper(socket)))) } + /// Constructs a [File] from a [ListenerSocket] + pub fn from_listener_socket(socket: Arc) -> Arc { + Arc::new(Self::ListenerSocket(Arc::new(ListenerSocketWrapper( + socket, + )))) + } + + /// Constructs a [File] from a [ConnectionSocket] + pub fn from_stream_socket(socket: Arc) -> Arc { + Arc::new(Self::StreamSocket(Arc::new(ConnectionSocketWrapper( + socket, + )))) + } + pub(crate) fn directory(node: NodeRef, position: DirectoryOpenPosition) -> Arc { let position = IrqSafeSpinlock::new(position.into()); Arc::new(Self::Directory(DirectoryFile { node, position })) @@ -281,18 +297,13 @@ impl File { } } - /// Interprets the file as a packet-based socket - pub fn as_packet_socket(&self) -> Result<&PacketSocketWrapper, Error> { - if let Self::PacketSocket(sock) = self { - Ok(sock) - } else { - Err(Error::InvalidOperation) - } - } - /// Sends data to a socket pub fn send_to(&self, buffer: &[u8], recepient: Option) -> Result { - self.as_packet_socket()?.send(recepient, buffer) + match (self, recepient) { + (Self::PacketSocket(socket), recepient) => socket.send(recepient, buffer), + (Self::StreamSocket(socket), None) => socket.send(buffer), + (_, _) => todo!(), + } } /// Receives data from a socket @@ -307,6 +318,23 @@ impl File { remote.write(addr); Ok(len) } + Self::StreamSocket(socket) => { + // Always the same + remote.write(socket.remote_address().unwrap()); + socket.receive(buffer) + } + _ => Err(Error::InvalidOperation), + } + } + + /// Waits for incoming connection to be accepted by the listener + pub fn accept(&self, remote: &mut MaybeUninit) -> Result { + match self { + Self::ListenerSocket(socket) => { + let (address, incoming) = socket.accept()?; + remote.write(address); + Ok(File::from_stream_socket(incoming)) + } _ => Err(Error::InvalidOperation), } } @@ -344,7 +372,10 @@ impl Read for File { // TODO maybe allow reading messages from Channels? Self::Channel(_) => Err(Error::InvalidOperation), Self::SharedMemory(_) => Err(Error::InvalidOperation), - Self::PacketSocket(_) => Err(Error::InvalidOperation), + // TODO maybe allow reading messages from Packet/Stream sockets? + Self::PacketSocket(_) | Self::ListenerSocket(_) | Self::StreamSocket(_) => { + Err(Error::InvalidOperation) + } Self::Directory(_) => Err(Error::IsADirectory), } } @@ -364,7 +395,10 @@ impl Write for File { // TODO maybe allow writing messages to Channels? Self::Channel(_) => Err(Error::InvalidOperation), Self::SharedMemory(_) => Err(Error::InvalidOperation), - Self::PacketSocket(_) => Err(Error::InvalidOperation), + // TODO maybe allow writing messages to Packet/Stream sockets? + Self::PacketSocket(_) | Self::ListenerSocket(_) | Self::StreamSocket(_) => { + Err(Error::InvalidOperation) + } Self::Directory(_) => Err(Error::IsADirectory), } } @@ -422,6 +456,15 @@ impl fmt::Debug for File { .field("local", &sock.local_address()) .field("remote", &sock.remote_address()) .finish_non_exhaustive(), + Self::StreamSocket(sock) => f + .debug_struct("StreamSocket") + .field("local", &sock.local_address()) + .field("remote", &sock.remote_address()) + .finish_non_exhaustive(), + Self::ListenerSocket(sock) => f + .debug_struct("ListenerSocket") + .field("local", &sock.local_address()) + .finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/socket.rs b/lib/vfs/src/socket.rs index dcec6aa1..86c72412 100644 --- a/lib/vfs/src/socket.rs +++ b/lib/vfs/src/socket.rs @@ -42,11 +42,22 @@ pub trait PacketSocket: Socket { } /// Connection-based client socket interface -pub trait ConnectionSocket: Socket {} +pub trait ConnectionSocket: Socket { + /// Receives data into provided buffer + fn receive(&self, buffer: &mut [u8]) -> Result; + + /// Transmits data + fn send(&self, data: &[u8]) -> Result; +} /// Connection-based listener socket interface -pub trait ListenerSocket: Socket {} +pub trait ListenerSocket: Socket { + /// Blocks the execution until an incoming connection is accepted + fn accept(&self) -> Result<(SocketAddr, Arc), Error>; +} pub struct PacketSocketWrapper(pub Arc); +pub struct ListenerSocketWrapper(pub Arc); +pub struct ConnectionSocketWrapper(pub Arc); impl Deref for PacketSocketWrapper { type Target = dyn PacketSocket; @@ -61,3 +72,31 @@ impl Drop for PacketSocketWrapper { self.0.close().ok(); } } + +impl Deref for ListenerSocketWrapper { + type Target = dyn ListenerSocket; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +impl Drop for ListenerSocketWrapper { + fn drop(&mut self) { + self.0.close().ok(); + } +} + +impl Deref for ConnectionSocketWrapper { + type Target = dyn ConnectionSocket; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +impl Drop for ConnectionSocketWrapper { + fn drop(&mut self) { + self.0.close().ok(); + } +} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index a76bade6..bcc40db0 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -18,8 +18,8 @@ use abi::{ }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; -use vfs::{File, IoContext, MessagePayload, NodeRef, PacketSocket, Read, Seek, Write}; -use ygg_driver_net_core::socket::{RawSocket, UdpSocket}; +use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; +use ygg_driver_net_core::socket::{RawSocket, TcpListener, UdpSocket}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ @@ -666,12 +666,15 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let ty = SocketType::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; run_with_io(process, |mut io| { - let socket: Arc = match ty { - SocketType::UdpPacket => UdpSocket::bind((*listen).into())?, - SocketType::RawPacket => RawSocket::bind()?, - _ => todo!(), + let file = match ty { + SocketType::UdpPacket => { + File::from_packet_socket(UdpSocket::bind((*listen).into())?) + } + SocketType::RawPacket => File::from_packet_socket(RawSocket::bind()?), + SocketType::TcpStream => { + File::from_listener_socket(TcpListener::bind((*listen).into())?) + } }; - let file = File::from_packet_socket(socket); let fd = io.files.place_file(file, true)?; Ok(fd.0 as usize) }) @@ -724,6 +727,22 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) }) } + SyscallFunction::Accept => { + let socket_fd = RawFd::from(args[0] as u32); + let remote_result = + arg_user_mut::>(args[1] as usize)?; + + run_with_io(process, |mut io| { + let file = io.files.file(socket_fd)?; + let mut remote = MaybeUninit::uninit(); + let accepted_file = file.accept(&mut remote)?; + let accepted_fd = io.files.place_file(accepted_file, true)?; + unsafe { + remote_result.write(remote.assume_init().into()); + } + Ok(accepted_fd.0 as usize) + }) + } SyscallFunction::Fork => unreachable!(), } From 7945710b88e2b8fb5c6f844ef24d30413feed833 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 25 Jan 2024 12:12:07 +0200 Subject: [PATCH 158/211] util: less unnecessary wakeups for sleep()s --- lib/kernel-util/src/lib.rs | 3 +- lib/kernel-util/src/runtime/mod.rs | 45 +-------- lib/kernel-util/src/runtime/timer.rs | 142 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 44 deletions(-) create mode 100644 lib/kernel-util/src/runtime/timer.rs diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 7197417c..575b2641 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -11,7 +11,8 @@ allocator_api, maybe_uninit_uninit_array, const_maybe_uninit_uninit_array, - new_uninit + new_uninit, + inline_const )] use device_api::interrupt::MessageInterruptController; diff --git a/lib/kernel-util/src/runtime/mod.rs b/lib/kernel-util/src/runtime/mod.rs index d9e93dd0..3778f2d2 100644 --- a/lib/kernel-util/src/runtime/mod.rs +++ b/lib/kernel-util/src/runtime/mod.rs @@ -1,54 +1,13 @@ -use core::{ - pin::Pin, - task::{Context, Poll}, - time::Duration, -}; - #[macro_use] mod macros; mod executor; mod task; mod task_queue; +mod timer; mod waker; pub use executor::{run_to_completion, spawn, spawn_async_worker}; -use futures_util::Future; pub use task_queue::init_task_queue; +pub use timer::{run_with_timeout, sleep, tick, FutureTimeout}; pub use waker::QueueWaker; - -use crate::api; - -static SLEEP_WAKER: QueueWaker = QueueWaker::new(); - -/// Suspends the task until given duration passes -pub fn sleep(duration: Duration) -> impl Future { - struct SleepFuture { - deadline: Duration, - } - - impl Future for SleepFuture { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - SLEEP_WAKER.register(cx.waker()); - let now = unsafe { api::__monotonic_timestamp() }.unwrap(); - if now >= self.deadline { - SLEEP_WAKER.remove(cx.waker()); - Poll::Ready(()) - } else { - Poll::Pending - } - } - } - - let now = unsafe { api::__monotonic_timestamp() }.unwrap(); - let deadline = now + duration; - - SleepFuture { deadline } -} - -/// Updates the runtime's time -pub fn tick(_now: Duration) { - SLEEP_WAKER.wake_all(); -} diff --git a/lib/kernel-util/src/runtime/timer.rs b/lib/kernel-util/src/runtime/timer.rs new file mode 100644 index 00000000..05f23544 --- /dev/null +++ b/lib/kernel-util/src/runtime/timer.rs @@ -0,0 +1,142 @@ +use core::{ + pin::Pin, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use alloc::vec::Vec; +use futures_util::{future::BoxFuture, Future, FutureExt}; + +use crate::{api, sync::IrqSafeSpinlock}; + +// 1..32ms, tick every 1ms +static SHORT_TERM_SLEEPS: IrqSafeSpinlock> = + IrqSafeSpinlock::new(TimerWheel::new(0, 1)); +// 32ms..288ms, tick every 8ms +static LONG_TERM_SLEEPS: IrqSafeSpinlock> = + IrqSafeSpinlock::new(TimerWheel::new(4, 8)); + +// A simple timer wheel inspired by +// https://www.snellman.net/blog/archive/2016-07-27-ratas-hierarchical-timer-wheel/ +// The wheel consists of a ring with slots. When some task wants to sleep for N ticks, +// it adds itself to a list N steps after current tick (modulo STEPS). The whell also contains +// a sort of interval to "scale" ticks for longer-term sleeps. +struct TimerWheel { + // TODO maybe use something better than Vec + ring: [Vec; STEPS], + tick: u64, + real_tick: u64, + base: u64, + interval: u64, +} + +impl TimerWheel { + pub const fn new(base: u64, interval: u64) -> Self { + Self { + ring: [const { Vec::new() }; STEPS], + tick: 0, + real_tick: 0, + base, + interval, + } + } + + pub fn tick(&mut self) { + self.real_tick += 1; + if self.real_tick == self.interval { + self.tick = self.tick.wrapping_add(1); + for waker in self.ring[(self.tick % (STEPS as u64)) as usize].drain(..) { + waker.wake(); + } + self.real_tick = 0; + } + } + + pub fn wake_after(&mut self, ticks: u64, waker: &Waker) { + debug_assert!((ticks / self.interval) - self.base <= STEPS as u64,); + let ticks = ticks.max(1); + self.ring[((self.tick + ticks / self.interval - self.base) % (STEPS as u64)) as usize] + .push(waker.clone()); + } +} + +fn register_timeout(duration: Duration, waker: &Waker) { + let nticks = duration.as_millis().min(288) as u64; + + if nticks < 32 { + SHORT_TERM_SLEEPS.lock().wake_after(nticks, waker); + } else { + LONG_TERM_SLEEPS.lock().wake_after(nticks, waker); + } +} + +pub enum FutureTimeout { + Ok(T), + Timeout, +} + +/// Suspends the task until given duration passes +pub fn sleep(duration: Duration) -> impl Future { + struct SleepFuture { + deadline: Duration, + } + + impl Future for SleepFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + match self.deadline.checked_sub(now) { + // Pending + Some(duration) if !duration.is_zero() => { + register_timeout(duration, cx.waker()); + Poll::Pending + } + // Ran out + _ => Poll::Ready(()), + } + } + } + + let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + let deadline = now + duration; + + SleepFuture { deadline } +} + +pub fn run_with_timeout<'a, T: 'a, F: Future + Send + 'a>( + duration: Duration, + fut: F, +) -> impl Future> + 'a { + struct TimeoutFuture<'f, T> { + fut: BoxFuture<'f, T>, + sleep_fut: BoxFuture<'f, ()>, + } + + impl<'f, T> Future for TimeoutFuture<'f, T> { + type Output = FutureTimeout; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let (timeout, result) = (self.sleep_fut.as_mut().poll(cx), self.fut.as_mut().poll(cx)); + + if let Poll::Ready(result) = result { + Poll::Ready(FutureTimeout::Ok(result)) + } else if timeout.is_ready() { + Poll::Ready(FutureTimeout::Timeout) + } else { + Poll::Pending + } + } + } + + TimeoutFuture { + fut: fut.boxed(), + sleep_fut: sleep(duration).boxed(), + } +} + +/// Updates the runtime's time +pub fn tick(_now: Duration) { + SHORT_TERM_SLEEPS.lock().tick(); + LONG_TERM_SLEEPS.lock().tick(); +} From f8b81c92da50b0b3d48fb4014fc8f197adf851bc Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 25 Jan 2024 13:10:01 +0200 Subject: [PATCH 159/211] net/tcp: implement TCP locally initiated connections --- driver/net/core/src/l4/tcp.rs | 86 +++++++++++++++++++--- driver/net/core/src/socket.rs | 135 +++++++++++++++++++++++++--------- lib/kernel-util/src/lib.rs | 9 ++- lib/vfs/src/file/mod.rs | 2 + src/syscall/mod.rs | 26 ++++++- 5 files changed, 212 insertions(+), 46 deletions(-) diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index e6ad274d..1dc65c10 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -149,7 +149,9 @@ impl TcpConnection { state: TcpConnectionState, ) -> Self { debug_assert!( - state == TcpConnectionState::SynSent || state == TcpConnectionState::SynReceived + state == TcpConnectionState::SynSent + || state == TcpConnectionState::Closed + || state == TcpConnectionState::SynReceived ); debug_assert!(window_size < u16::MAX as usize); Self { @@ -195,9 +197,9 @@ impl TcpConnection { pub fn read_nonblocking(&mut self, buffer: &mut [u8]) -> Result { let amount = self.rx_buffer.read(buffer); - if amount == 0 && self.state == TcpConnectionState::Established { - // TODO error - return Err(Error::DoesNotExist); + if amount == 0 && self.state != TcpConnectionState::Established { + // TODO ConnectionAborted? + return Err(Error::ConnectionReset); } self.rx_window_start += amount; Ok(amount) @@ -210,8 +212,7 @@ impl TcpConnection { Poll::Ready(Ok(())) } else if self.state != TcpConnectionState::Established { self.rx_notify.remove(cx.waker()); - // TODO error - Poll::Ready(Err(Error::DoesNotExist)) + Poll::Ready(Err(Error::ConnectionReset)) } else { Poll::Pending } @@ -221,8 +222,7 @@ impl TcpConnection { self.tx_notify.register(cx.waker()); if self.state == TcpConnectionState::Closed { self.tx_notify.remove(cx.waker()); - // TODO error - Poll::Ready(Err(Error::DoesNotExist)) + Poll::Ready(Err(Error::ConnectionReset)) } else if self.tx_sent_unacknowledged == 0 { self.tx_notify.remove(cx.waker()); Poll::Ready(Ok(())) @@ -251,6 +251,15 @@ impl TcpConnection { } } + pub(crate) fn poll_established(&self, cx: &mut Context<'_>) -> Poll> { + self.rx_notify.register(cx.waker()); + match self.state { + TcpConnectionState::Established => Poll::Ready(Ok(())), + TcpConnectionState::Closed => Poll::Ready(Err(Error::ConnectionRefused)), + _ => Poll::Pending, + } + } + pub(crate) async fn transmit(&mut self, data: &[u8]) -> Result<(), Error> { assert_eq!(self.tx_sent_unacknowledged, 0); assert_eq!(self.tx_buffer.len(), 0); @@ -296,6 +305,24 @@ impl TcpConnection { self.tx_notify.wake_all(); } + pub(crate) async fn send_syn(&mut self) -> Result<(), Error> { + assert!( + self.state == TcpConnectionState::SynSent || self.state == TcpConnectionState::Closed + ); + log::debug!("Send SYN {} -> {}", self.local, self.remote); + self.state = TcpConnectionState::SynSent; + send( + self.local, + self.remote, + self.initial_tx_seq, + 0, + self.rx_window_size, + TcpFlags::SYN, + &[], + ) + .await + } + async fn handle_packet( &mut self, packet: TcpPacket, @@ -304,6 +331,48 @@ impl TcpConnection { // TODO what if window_size changes? match self.state { + TcpConnectionState::SynSent => { + if packet.flags == TcpFlags::SYN | TcpFlags::ACK { + if packet.ack != self.initial_tx_seq.wrapping_add(1) { + log::warn!( + "Expected ACK {}, got {}", + self.initial_tx_seq.wrapping_add(1), + packet.ack + ); + return Ok(TcpSocketBehavior::None); + } + + log::debug!( + "TCP {} -> {} got ACKed, established", + self.local, + self.remote + ); + self.initial_rx_seq = packet.seq; + + // ACK the SYN+ACK + send( + self.local, + self.remote, + self.initial_tx_seq.wrapping_add(1), + self.initial_rx_seq.wrapping_add(1), + self.rx_window_size, + TcpFlags::ACK, + &[], + ) + .await?; + + self.state = TcpConnectionState::Established; + self.rx_notify.wake_all(); + } else if packet.flags == TcpFlags::RST | TcpFlags::ACK { + log::debug!("TCP {} -> {} got RSTd, closing", self.local, self.remote); + self.state = TcpConnectionState::Closed; + self.rx_notify.wake_all(); + return Ok(TcpSocketBehavior::Remove); + } + + // TODO try re-sending SYN? + return Ok(TcpSocketBehavior::None); + } TcpConnectionState::FinWait1 => { // Check if connection close initiated locally got ACKed by remote // TODO check ack/seq @@ -363,7 +432,6 @@ impl TcpConnection { return Err(Error::InvalidArgument); } - TcpConnectionState::SynSent => todo!(), TcpConnectionState::Established => (), } diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 585efb39..ac094da3 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -3,13 +3,15 @@ use core::{ pin::Pin, sync::atomic::{AtomicBool, AtomicU32, Ordering}, task::{Context, Poll}, + time::Duration, }; use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use kernel_util::{ block, mem::PageBox, - runtime::QueueWaker, + monotonic_timestamp, + runtime::{run_with_timeout, FutureTimeout, QueueWaker}, sync::{ mutex::{Mutex, MutexGuard}, spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, @@ -26,6 +28,7 @@ use yggdrasil_abi::{ use crate::{ ethernet::L2Packet, interface::NetworkInterface, + l3::Route, l4::{ self, tcp::{TcpConnection, TcpConnectionState}, @@ -82,20 +85,38 @@ impl TwoWaySocketTable { } } - pub fn try_insert_with Arc>( + pub fn try_insert_with Result, Error>>( &mut self, local: SocketAddr, remote: SocketAddr, with: F, ) -> Result, Error> { if self.inner.contains_key(&(local, remote)) { - return Err(Error::AlreadyExists); + return Err(Error::AddrInUse); } - let socket = with(); + let socket = with()?; self.inner.insert((local, remote), socket.clone()); Ok(socket) } + pub fn try_insert_with_ephemeral_port Result, Error>>( + &mut self, + local: IpAddr, + remote: SocketAddr, + mut with: F, + ) -> Result, Error> { + for port in 32768..u16::MAX - 1 { + let local = SocketAddr::new(local, port); + + match self.try_insert_with(local, remote, || with(port)) { + Ok(socket) => return Ok(socket), + Err(Error::AddrInUse) => continue, + Err(error) => return Err(error), + } + } + Err(Error::AddrInUse) + } + pub fn remove(&mut self, local: SocketAddr, remote: SocketAddr) -> Result<(), Error> { match self.inner.remove(&(local, remote)) { Some(_) => Ok(()), @@ -115,7 +136,7 @@ impl SocketTable { } } - pub fn try_insert_with Arc>( + pub fn try_insert_with Result, Error>>( &mut self, address: SocketAddr, with: F, @@ -123,18 +144,11 @@ impl SocketTable { if self.inner.contains_key(&address) { return Err(Error::AlreadyExists); } - let socket = with(); + let socket = with()?; self.inner.insert(address, socket.clone()); Ok(socket) } - pub fn insert(&mut self, address: SocketAddr, socket: Arc) -> Result<(), Error> { - match self.inner.try_insert(address, socket) { - Ok(_) => Ok(()), - Err(_) => Err(Error::AlreadyExists), - } - } - pub fn remove(&mut self, local: SocketAddr) -> Result<(), Error> { match self.inner.remove(&local) { Some(_) => Ok(()), @@ -175,25 +189,19 @@ static TCP_LISTENERS: IrqSafeRwLock> = impl UdpSocket { pub fn bind(address: SocketAddr) -> Result, Error> { let mut sockets = UDP_SOCKETS.write(); + sockets.try_insert_with(address, move || { + let socket = Arc::new(Self { + local: address, + remote: None, + broadcast: AtomicBool::new(false), + receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), + receive_notify: QueueWaker::new(), + }); - if sockets.get(&address).is_some() { - // TODO use network-specific error - return Err(Error::AlreadyExists); - } + log::debug!("UDP socket opened: {}", address); - let socket = Arc::new(Self { - local: address, - remote: None, - broadcast: AtomicBool::new(false), - receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), - receive_notify: QueueWaker::new(), - }); - - sockets.insert(address, socket.clone()).unwrap(); - - log::debug!("UDP socket opened: {}", address); - - Ok(socket) + Ok(socket) + }) } pub fn connect(&self, _address: SocketAddr) -> Result<(), Error> { @@ -502,8 +510,67 @@ impl PacketSocket for RawSocket { } impl TcpSocket { - pub async fn connect(_remote: IpAddr) -> Result, Error> { - todo!() + pub fn connect(remote: SocketAddr) -> Result<(SocketAddr, Arc), Error> { + block!(Self::connect_async(remote).await)? + } + + async fn connect_async(remote: SocketAddr) -> Result<(SocketAddr, Arc), Error> { + // Lookup route to remote + let (interface_id, _) = Route::lookup(remote.ip()).ok_or(Error::HostUnreachable)?; + let interface = NetworkInterface::get(interface_id)?; + let local_ip = interface.address.read().ok_or(Error::NetworkUnreachable)?; + + let socket = { + let mut sockets = TCP_SOCKETS.write(); + sockets.try_insert_with_ephemeral_port(local_ip, remote, |port| { + let t = monotonic_timestamp()?; + let tx_seq = t.as_micros() as u32; + let local = SocketAddr::new(local_ip, port); + let connection = + TcpConnection::new(local, remote, 16384, tx_seq, 0, TcpConnectionState::Closed); + + let socket = Self { + local, + remote, + listener: None, + connection: IrqSafeRwLock::new(connection), + }; + + Ok(Arc::new(socket)) + })? + }; + + let mut t = 200; + for _ in 0..5 { + let timeout = Duration::from_millis(t); + log::debug!("Try SYN with timeout={:?}", timeout); + match socket.try_connect(timeout).await { + Ok(()) => return Ok((socket.local, socket)), + Err(Error::TimedOut) => (), + Err(error) => return Err(error), + } + t *= 2; + } + + // Couldn't establish + Err(Error::TimedOut) + } + + async fn try_connect(&self, timeout: Duration) -> Result<(), Error> { + { + let mut connection = self.connection.write(); + connection.send_syn().await?; + } + + let fut = poll_fn(|cx| { + let connection = self.connection.read(); + connection.poll_established(cx) + }); + + match run_with_timeout(timeout, fut).await { + FutureTimeout::Ok(value) => value, + FutureTimeout::Timeout => Err(Error::TimedOut), + } } pub fn accept_remote( @@ -534,7 +601,7 @@ impl TcpSocket { connection: IrqSafeRwLock::new(connection), }; - Arc::new(socket) + Ok(Arc::new(socket)) }) } @@ -731,7 +798,7 @@ impl TcpListener { log::debug!("TCP Listener opened: {}", accept); - Arc::new(listener) + Ok(Arc::new(listener)) }) } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 575b2641..98403a48 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -15,8 +15,10 @@ inline_const )] +use core::time::Duration; + use device_api::interrupt::MessageInterruptController; -use yggdrasil_abi::process::Signal; +use yggdrasil_abi::{error::Error, process::Signal}; extern crate alloc; @@ -53,3 +55,8 @@ pub struct AlignedTo { pub align: [Align; 0], pub bytes: Bytes, } + +#[inline] +pub fn monotonic_timestamp() -> Result { + unsafe { api::__monotonic_timestamp() } +} diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 36fc7b24..d7e9c01d 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -255,6 +255,8 @@ impl File { Self::PtyMaster(f) => f.poll_read(cx), Self::PtySlave(f) => f.poll_read(cx), Self::PacketSocket(sock) => sock.poll_read(cx), + Self::StreamSocket(sock) => sock.poll_read(cx), + Self::ListenerSocket(sock) => sock.poll_read(cx), // Polling not implemented, return ready immediately (XXX ?) _ => Poll::Ready(Err(Error::NotImplemented)), } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index bcc40db0..c66c1061 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -19,7 +19,7 @@ use abi::{ use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; -use ygg_driver_net_core::socket::{RawSocket, TcpListener, UdpSocket}; +use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; use crate::{ @@ -679,7 +679,29 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(fd.0 as usize) }) } - SyscallFunction::ConnectSocket => todo!(), + SyscallFunction::ConnectSocket => { + let socket_fd = RawFd::from(args[0] as u32); + let remote = arg_user_ref::(args[1] as usize)?; + let ty = SocketType::try_from(args[2] as u32).map_err(|_| Error::InvalidArgument)?; + let local_result = + arg_user_mut::>(args[3] as usize)?; + + // TODO connect on existing sockets? + assert_eq!(socket_fd, RawFd::NONE); + + run_with_io(process, |mut io| { + let (local, file) = match ty { + SocketType::TcpStream => { + let (local, socket) = TcpSocket::connect((*remote).into())?; + (local, File::from_stream_socket(socket)) + } + _ => return Err(Error::InvalidArgument), + }; + let fd = io.files.place_file(file, true)?; + local_result.write(local.into()); + Ok(fd.0 as usize) + }) + } SyscallFunction::SendTo => { let socket_fd = RawFd::from(args[0] as u32); let buffer = arg_buffer_ref(args[1] as usize, args[2] as usize)?; From d49eeac8bfa1f7e0f04dbc5cc99b89e49a365513 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 26 Jan 2024 16:31:11 +0200 Subject: [PATCH 160/211] mem/phys: fix incorrect reservation of page bitmap --- src/mem/phys/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 3f760516..fe6fa775 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -155,7 +155,7 @@ pub unsafe fn init_from_iter + Clone>( ARCHITECTURE.map_physical_memory(it.clone(), phys_start, phys_end)?; let total_count = (phys_end - phys_start) / 0x1000; - let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / BITMAP_WORD_SIZE; + let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / (BITMAP_WORD_SIZE / 8); let page_bitmap_page_count = (page_bitmap_size + 0xFFF) / 0x1000; let page_bitmap_phys_base = find_contiguous_region(it.clone(), page_bitmap_page_count).unwrap(); From 01240031301072485fc8c720ede6d31baffaffb3 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 26 Jan 2024 16:31:56 +0200 Subject: [PATCH 161/211] net/tcp: improve Rx path by adding a reassembler --- driver/net/core/src/l3/mod.rs | 5 +- driver/net/core/src/l4/tcp.rs | 147 +++++++++++------ driver/net/core/src/lib.rs | 7 +- driver/net/core/src/socket.rs | 172 ++++++++++---------- driver/net/core/src/util.rs | 292 ++++++++++++++++++++++++++++++++++ 5 files changed, 482 insertions(+), 141 deletions(-) create mode 100644 driver/net/core/src/util.rs diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 89c4e011..022126ca 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -175,7 +175,10 @@ pub async fn send_l4_ip( ttl, )?; - let mut builder = PacketBuilder::new(interface.device.packet_prefix_size())?; + let mut builder = PacketBuilder::new( + interface.device.packet_prefix_size(), + size_of::() + size_of::() + size_of::() + l4_data.len(), + )?; builder.push(&EthernetFrame { source_mac: interface.mac, destination_mac: gateway_mac, diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index 1dc65c10..d233233b 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -11,13 +11,14 @@ use yggdrasil_abi::{ net::{ protocols::{InetChecksum, IpProtocol, TcpFlags, TcpFrame, TcpV4PseudoHeader}, types::NetValueImpl, - SocketAddr, + IpAddr, SocketAddr, }, }; use crate::{ l3::{self, L3Packet}, socket::{TcpListener, TcpSocket}, + util::Assembler, }; #[derive(PartialEq, Debug)] @@ -55,6 +56,8 @@ pub struct TcpConnection { // RCV.NXT = rx_window_start + rx_window.len() // TODO RCV.UP rx_buffer: SocketBuffer, + rx_segment_buffer: Vec, + rx_assembler: Assembler, // Relative RX sequence number of window start rx_window_start: usize, @@ -161,6 +164,8 @@ impl TcpConnection { remote, rx_buffer: SocketBuffer::with_capacity(window_size), + rx_assembler: Assembler::new(), + rx_segment_buffer: Vec::with_capacity(window_size), rx_window_start: 1, tx_buffer: Vec::with_capacity(window_size), @@ -323,6 +328,45 @@ impl TcpConnection { .await } + fn handle_packet_payload(&mut self, data: &[u8], seq: u32) -> bool { + // Local side + let rx_window_start = self.rx_window_start + self.rx_buffer.len(); + let rx_window_end = self.rx_window_start + self.rx_buffer.capacity(); + // Remote side + let rx_segment_start = seq.wrapping_sub(self.initial_rx_seq) as usize; + let rx_segment_end = rx_segment_start + data.len(); + + if rx_segment_end >= rx_window_end || rx_segment_start < rx_window_start { + return false; + } + + // Offset from expected seq + let segment_start_offset = rx_segment_start - rx_window_start; + + // Push the data into reassembler buffer + assert!(segment_start_offset + data.len() <= self.rx_segment_buffer.capacity()); + if segment_start_offset + data.len() > self.rx_segment_buffer.len() { + self.rx_segment_buffer + .resize(segment_start_offset + data.len(), 0); + } + self.rx_segment_buffer[segment_start_offset..segment_start_offset + data.len()] + .copy_from_slice(data); + + let amount = self + .rx_assembler + .add_then_remove_front(segment_start_offset, data.len()) + .unwrap(); + + if amount != 0 { + // Take data from reassembly buffer and append to rx_buffer + self.rx_buffer.write(&self.rx_segment_buffer[..amount]); + self.rx_segment_buffer.drain(..amount); + self.rx_notify.wake_one(); + } + + true + } + async fn handle_packet( &mut self, packet: TcpPacket, @@ -435,8 +479,6 @@ impl TcpConnection { TcpConnectionState::Established => (), } - let mut reply_flags = TcpFlags::empty(); - if self.tx_sent_unacknowledged != 0 { let tx_acknowledge_end = packet.ack.wrapping_sub(self.initial_tx_seq) as usize; @@ -448,62 +490,36 @@ impl TcpConnection { } } + let mut reply_flags = TcpFlags::empty(); + let mut behavior = TcpSocketBehavior::None; + if data.len() != 0 { - // Local side - let rx_window_start = self.rx_window_start + self.rx_buffer.len(); - let rx_window_end = self.rx_window_start + self.rx_buffer.capacity(); - // Remote side - let rx_segment_start = packet.seq.wrapping_sub(self.initial_rx_seq) as usize; - let rx_segment_end = rx_segment_start + data.len(); - - if rx_segment_end > rx_window_end || rx_segment_start < rx_window_start { - log::warn!( - "Segment {:?} outside of Rx window {:?}", - rx_segment_start..rx_segment_end, - rx_window_start..rx_window_end - ); - return Ok(TcpSocketBehavior::None); - } - if rx_segment_start != rx_window_start { - log::warn!( - "Out-of-order segment {:?} received, will not accept into {:?}", - rx_segment_start..rx_segment_end, - rx_window_start..rx_window_end - ); - return Ok(TcpSocketBehavior::None); - } - - // segment_start == window_start - let amount = data.len().min(rx_window_end - rx_window_start); - - if amount != 0 { - log::trace!( - "Received segment: {:?}, current window {:?}", - rx_segment_start..rx_segment_end, - rx_window_start..rx_window_end - ); - - self.rx_buffer.write(&data[..amount]); - self.rx_notify.wake_one(); + if self.handle_packet_payload(data, packet.seq) { reply_flags |= TcpFlags::ACK; } } + // TODO check window resize notification + let mut ack_number = self.ack_number(); - let mut behavior = TcpSocketBehavior::None; if packet.flags.contains(TcpFlags::FIN) { - // TODO wait for ACK? - reply_flags |= TcpFlags::FIN | TcpFlags::ACK; + reply_flags |= TcpFlags::ACK; ack_number = ack_number.wrapping_add(1); - self.state = TcpConnectionState::Closed; - log::trace!( - "TCP connection FIN requested by remote: {} <-> {}", - self.local, - self.remote - ); - behavior = TcpSocketBehavior::Remove; + // Only send an actual FIN after a FIN without any data + if data.len() == 0 { + reply_flags |= TcpFlags::FIN; + // TODO go to LastAck state and wait for ACK + self.state = TcpConnectionState::Closed; + log::trace!( + "TCP connection FIN requested by remote: {} <-> {}", + self.local, + self.remote + ); + + behavior = TcpSocketBehavior::Remove; + } } if reply_flags != TcpFlags::empty() { @@ -567,6 +583,29 @@ async fn send( l3::send_l4_ip(remote.ip().into(), IpProtocol::TCP, 64, &frame, data).await } +fn validate(source: IpAddr, destination: IpAddr, tcp_frame: &TcpFrame, data: &[u8]) -> bool { + // TODO TCPv6 + let source = source.into_ipv4().unwrap(); + let destination = destination.into_ipv4().unwrap(); + let tcp_length = size_of::() + data.len(); + + let pseudo_header = TcpV4PseudoHeader { + source_address: u32::from(source).to_network_order(), + destination_address: u32::from(destination).to_network_order(), + _zero: 0, + protocol: IpProtocol::TCP, + tcp_length: (tcp_length as u16).to_network_order(), + }; + + let mut checksum = InetChecksum::new(); + checksum.add_value(&pseudo_header, true); + checksum.add_value(tcp_frame, true); + checksum.add_bytes(data, true); + let checksum = checksum.finish(); + + checksum == 0 +} + pub async fn handle(packet: L3Packet) -> Result<(), Error> { if packet.data_length < size_of::() { log::warn!("Truncated TCP packet"); @@ -591,7 +630,15 @@ pub async fn handle(packet: L3Packet) -> Result<(), Error> { let seq = u32::from_network_order(tcp_frame.sequence_number); let ack = u32::from_network_order(tcp_frame.acknowledge_number); - // TODO validate checksum + if !validate( + packet.source_address, + packet.destination_address, + tcp_frame, + &l3_data[size_of::()..packet.data_length], + ) { + log::warn!("Invalid TCP packet received"); + return Ok(()); + } match tcp_frame.flags { TcpFlags::SYN => { diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index 044c6424..e93b6889 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -23,6 +23,7 @@ pub mod socket; pub mod config; pub mod interface; pub mod queue; +pub mod util; pub use interface::register_interface; @@ -40,8 +41,8 @@ pub struct PacketBuilder { } impl PacketBuilder { - pub fn new(l2_offset: usize) -> Result { - let data = PageBox::new_slice(0, 4096)?; + pub fn new(l2_offset: usize, l2_size: usize) -> Result { + let data = PageBox::new_slice(0, l2_offset + l2_size)?; Ok(Self { data, pos: l2_offset, @@ -89,11 +90,9 @@ pub fn receive_packet(packet: Packet) -> Result<(), Error> { pub fn start_network_tasks() -> Result<(), Error> { runtime::spawn(l2_packet_handler_worker())?; - for _ in 0..4 { runtime::spawn(l3_accept_worker())?; } - runtime::spawn(config::network_config_service())?; Ok(()) diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index ac094da3..bde19dcf 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -47,8 +47,8 @@ pub struct UdpSocket { } pub struct TcpSocket { - local: SocketAddr, - remote: SocketAddr, + pub(crate) local: SocketAddr, + pub(crate) remote: SocketAddr, // Listener which accepted the socket listener: Option>, connection: IrqSafeRwLock, @@ -514,65 +514,6 @@ impl TcpSocket { block!(Self::connect_async(remote).await)? } - async fn connect_async(remote: SocketAddr) -> Result<(SocketAddr, Arc), Error> { - // Lookup route to remote - let (interface_id, _) = Route::lookup(remote.ip()).ok_or(Error::HostUnreachable)?; - let interface = NetworkInterface::get(interface_id)?; - let local_ip = interface.address.read().ok_or(Error::NetworkUnreachable)?; - - let socket = { - let mut sockets = TCP_SOCKETS.write(); - sockets.try_insert_with_ephemeral_port(local_ip, remote, |port| { - let t = monotonic_timestamp()?; - let tx_seq = t.as_micros() as u32; - let local = SocketAddr::new(local_ip, port); - let connection = - TcpConnection::new(local, remote, 16384, tx_seq, 0, TcpConnectionState::Closed); - - let socket = Self { - local, - remote, - listener: None, - connection: IrqSafeRwLock::new(connection), - }; - - Ok(Arc::new(socket)) - })? - }; - - let mut t = 200; - for _ in 0..5 { - let timeout = Duration::from_millis(t); - log::debug!("Try SYN with timeout={:?}", timeout); - match socket.try_connect(timeout).await { - Ok(()) => return Ok((socket.local, socket)), - Err(Error::TimedOut) => (), - Err(error) => return Err(error), - } - t *= 2; - } - - // Couldn't establish - Err(Error::TimedOut) - } - - async fn try_connect(&self, timeout: Duration) -> Result<(), Error> { - { - let mut connection = self.connection.write(); - connection.send_syn().await?; - } - - let fut = poll_fn(|cx| { - let connection = self.connection.read(); - connection.poll_established(cx) - }); - - match run_with_timeout(timeout, fut).await { - FutureTimeout::Ok(value) => value, - FutureTimeout::Timeout => Err(Error::TimedOut), - } - } - pub fn accept_remote( listener: Arc, local: SocketAddr, @@ -648,31 +589,6 @@ impl TcpSocket { } } - async fn send_segment_async(&self, data: &[u8]) -> Result<(), Error> { - // TODO timeout here - { - let mut connection = poll_fn(|cx| { - let connection = self.connection.write(); - match connection.poll_send(cx) { - Poll::Ready(Ok(())) => Poll::Ready(Ok(connection)), - Poll::Ready(Err(error)) => Poll::Ready(Err(error)), - Poll::Pending => Poll::Pending, - } - }) - .await?; - - connection.transmit(data).await?; - } - - poll_fn(|cx| { - let connection = self.connection.read(); - connection.poll_acknowledge(cx) - }) - .await; - - Ok(()) - } - pub async fn send_async(&self, data: &[u8]) -> Result { let mut pos = 0; let mut rem = data.len(); @@ -754,6 +670,90 @@ impl TcpSocket { Poll::Pending => Poll::Pending, } } + + async fn send_segment_async(&self, data: &[u8]) -> Result<(), Error> { + // TODO timeout here + { + let mut connection = poll_fn(|cx| { + let connection = self.connection.write(); + match connection.poll_send(cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(connection)), + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Pending => Poll::Pending, + } + }) + .await?; + + connection.transmit(data).await?; + } + + poll_fn(|cx| { + let connection = self.connection.read(); + connection.poll_acknowledge(cx) + }) + .await; + + Ok(()) + } + + async fn connect_async(remote: SocketAddr) -> Result<(SocketAddr, Arc), Error> { + // Lookup route to remote + let (interface_id, _) = Route::lookup(remote.ip()).ok_or(Error::HostUnreachable)?; + let interface = NetworkInterface::get(interface_id)?; + let local_ip = interface.address.read().ok_or(Error::NetworkUnreachable)?; + + let socket = { + let mut sockets = TCP_SOCKETS.write(); + sockets.try_insert_with_ephemeral_port(local_ip, remote, |port| { + let t = monotonic_timestamp()?; + let tx_seq = t.as_micros() as u32; + let local = SocketAddr::new(local_ip, port); + let connection = + TcpConnection::new(local, remote, 16384, tx_seq, 0, TcpConnectionState::Closed); + + let socket = Self { + local, + remote, + listener: None, + connection: IrqSafeRwLock::new(connection), + }; + + Ok(Arc::new(socket)) + })? + }; + + let mut t = 200; + for _ in 0..5 { + let timeout = Duration::from_millis(t); + log::debug!("Try SYN with timeout={:?}", timeout); + match socket.try_connect(timeout).await { + Ok(()) => return Ok((socket.local, socket)), + Err(Error::TimedOut) => (), + Err(error) => return Err(error), + } + t *= 2; + } + + // Couldn't establish + Err(Error::TimedOut) + } + + async fn try_connect(&self, timeout: Duration) -> Result<(), Error> { + { + let mut connection = self.connection.write(); + connection.send_syn().await?; + } + + let fut = poll_fn(|cx| { + let connection = self.connection.read(); + connection.poll_established(cx) + }); + + match run_with_timeout(timeout, fut).await { + FutureTimeout::Ok(value) => value, + FutureTimeout::Timeout => Err(Error::TimedOut), + } + } } impl Socket for TcpSocket { diff --git a/driver/net/core/src/util.rs b/driver/net/core/src/util.rs new file mode 100644 index 00000000..1d012c69 --- /dev/null +++ b/driver/net/core/src/util.rs @@ -0,0 +1,292 @@ +// This TCP reassembler was taken from smoltcp-rs/smoltcp: +// +// https://github.com/smoltcp-rs/smoltcp + +use core::fmt; + +pub const ASSEMBLER_MAX_SEGMENT_COUNT: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TooManyHolesError; + +impl fmt::Display for TooManyHolesError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "too many holes") + } +} + +/// A contiguous chunk of absent data, followed by a contiguous chunk of present data. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Contig { + hole_size: usize, + data_size: usize, +} + +impl fmt::Display for Contig { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.has_hole() { + write!(f, "({})", self.hole_size)?; + } + if self.has_hole() && self.has_data() { + write!(f, " ")?; + } + if self.has_data() { + write!(f, "{}", self.data_size)?; + } + Ok(()) + } +} + +impl Contig { + const fn empty() -> Contig { + Contig { + hole_size: 0, + data_size: 0, + } + } + + fn hole_and_data(hole_size: usize, data_size: usize) -> Contig { + Contig { + hole_size, + data_size, + } + } + + fn has_hole(&self) -> bool { + self.hole_size != 0 + } + + fn has_data(&self) -> bool { + self.data_size != 0 + } + + fn total_size(&self) -> usize { + self.hole_size + self.data_size + } + + fn shrink_hole_by(&mut self, size: usize) { + self.hole_size -= size; + } + + fn shrink_hole_to(&mut self, size: usize) { + debug_assert!(self.hole_size >= size); + + let total_size = self.total_size(); + self.hole_size = size; + self.data_size = total_size - size; + } +} + +/// A buffer (re)assembler. +/// +/// Currently, up to a hardcoded limit of 4 or 32 holes can be tracked in the buffer. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Assembler { + contigs: [Contig; ASSEMBLER_MAX_SEGMENT_COUNT], +} + +impl fmt::Display for Assembler { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "[ ")?; + for contig in self.contigs.iter() { + if !contig.has_data() { + break; + } + write!(f, "{contig} ")?; + } + write!(f, "]")?; + Ok(()) + } +} + +// Invariant on Assembler::contigs: +// - There's an index `i` where all contigs before have data, and all contigs after don't (are unused). +// - All contigs with data must have hole_size != 0, except the first. + +impl Assembler { + /// Create a new buffer assembler. + pub const fn new() -> Assembler { + const EMPTY: Contig = Contig::empty(); + Assembler { + contigs: [EMPTY; ASSEMBLER_MAX_SEGMENT_COUNT], + } + } + + pub fn clear(&mut self) { + self.contigs.fill(Contig::empty()); + } + + fn front(&self) -> Contig { + self.contigs[0] + } + + /// Return length of the front contiguous range without removing it from the assembler + pub fn peek_front(&self) -> usize { + let front = self.front(); + if front.has_hole() { + 0 + } else { + front.data_size + } + } + + fn back(&self) -> Contig { + self.contigs[self.contigs.len() - 1] + } + + /// Return whether the assembler contains no data. + pub fn is_empty(&self) -> bool { + !self.front().has_data() + } + + /// Remove a contig at the given index. + fn remove_contig_at(&mut self, at: usize) { + debug_assert!(self.contigs[at].has_data()); + + for i in at..self.contigs.len() - 1 { + if !self.contigs[i].has_data() { + return; + } + self.contigs[i] = self.contigs[i + 1]; + } + + // Removing the last one. + self.contigs[self.contigs.len() - 1] = Contig::empty(); + } + + /// Add a contig at the given index, and return a pointer to it. + fn add_contig_at(&mut self, at: usize) -> Result<&mut Contig, TooManyHolesError> { + if self.back().has_data() { + return Err(TooManyHolesError); + } + + for i in (at + 1..self.contigs.len()).rev() { + self.contigs[i] = self.contigs[i - 1]; + } + + self.contigs[at] = Contig::empty(); + Ok(&mut self.contigs[at]) + } + + /// Add a new contiguous range to the assembler, + /// or return `Err(TooManyHolesError)` if too many discontinuities are already recorded. + pub fn add(&mut self, mut offset: usize, size: usize) -> Result<(), TooManyHolesError> { + if size == 0 { + return Ok(()); + } + + let mut i = 0; + + // Find index of the contig containing the start of the range. + loop { + if i == self.contigs.len() { + // The new range is after all the previous ranges, but there/s no space to add it. + return Err(TooManyHolesError); + } + let contig = &mut self.contigs[i]; + if !contig.has_data() { + // The new range is after all the previous ranges. Add it. + *contig = Contig::hole_and_data(offset, size); + return Ok(()); + } + if offset <= contig.total_size() { + break; + } + offset -= contig.total_size(); + i += 1; + } + + let contig = &mut self.contigs[i]; + if offset < contig.hole_size { + // Range starts within the hole. + + if offset + size < contig.hole_size { + // Range also ends within the hole. + let new_contig = self.add_contig_at(i)?; + new_contig.hole_size = offset; + new_contig.data_size = size; + + // Previous contigs[index] got moved to contigs[index+1] + self.contigs[i + 1].shrink_hole_by(offset + size); + return Ok(()); + } + + // The range being added covers both a part of the hole and a part of the data + // in this contig, shrink the hole in this contig. + contig.shrink_hole_to(offset); + } + + // coalesce contigs to the right. + let mut j = i + 1; + while j < self.contigs.len() + && self.contigs[j].has_data() + && offset + size >= self.contigs[i].total_size() + self.contigs[j].hole_size + { + self.contigs[i].data_size += self.contigs[j].total_size(); + j += 1; + } + let shift = j - i - 1; + if shift != 0 { + for x in i + 1..self.contigs.len() { + if !self.contigs[x].has_data() { + break; + } + + self.contigs[x] = self + .contigs + .get(x + shift) + .copied() + .unwrap_or_else(Contig::empty); + } + } + + if offset + size > self.contigs[i].total_size() { + // The added range still extends beyond the current contig. Increase data size. + let left = offset + size - self.contigs[i].total_size(); + self.contigs[i].data_size += left; + + // Decrease hole size of the next, if any. + if i + 1 < self.contigs.len() && self.contigs[i + 1].has_data() { + self.contigs[i + 1].hole_size -= left; + } + } + + Ok(()) + } + + /// Remove a contiguous range from the front of the assembler. + /// If no such range, return 0. + pub fn remove_front(&mut self) -> usize { + let front = self.front(); + if front.has_hole() || !front.has_data() { + 0 + } else { + self.remove_contig_at(0); + debug_assert!(front.data_size > 0); + front.data_size + } + } + + /// Add a segment, then remove_front. + /// + /// This is equivalent to calling `add` then `remove_front` individually, + /// except it's guaranteed to not fail when offset = 0. + /// This is required for TCP: we must never drop the next expected segment, or + /// the protocol might get stuck. + pub fn add_then_remove_front( + &mut self, + offset: usize, + size: usize, + ) -> Result { + // This is the only case where a segment at offset=0 would cause the + // total amount of contigs to rise (and therefore can potentially cause + // a TooManyHolesError). Handle it in a way that is guaranteed to succeed. + if offset == 0 && size < self.contigs[0].hole_size { + self.contigs[0].hole_size -= size; + return Ok(size); + } + + self.add(offset, size)?; + Ok(self.remove_front()) + } +} From f6617da3d668250f9d3aa3e3cd33a771895bf1e5 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 26 Jan 2024 18:53:07 +0200 Subject: [PATCH 162/211] net: add loopback interface --- Cargo.toml | 1 + driver/net/core/src/interface.rs | 57 ++++++++++++++++++++++++-------- driver/net/core/src/l3/mod.rs | 17 ++++++++++ driver/net/core/src/l4/tcp.rs | 1 + driver/net/core/src/socket.rs | 55 ++++++++++++++++++++++++------ driver/net/loopback/Cargo.toml | 12 +++++++ driver/net/loopback/src/lib.rs | 41 +++++++++++++++++++++++ driver/virtio/net/src/lib.rs | 9 +++-- src/init.rs | 1 + 9 files changed, 167 insertions(+), 27 deletions(-) create mode 100644 driver/net/loopback/Cargo.toml create mode 100644 driver/net/loopback/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 1d135326..3136d880 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ device-api-macros = { path = "lib/device-api/macros" } # Drivers ygg_driver_block = { path = "driver/block/core" } ygg_driver_net_core = { path = "driver/net/core" } +ygg_driver_net_loopback = { path = "driver/net/loopback" } kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs index 32d42ea3..02da9380 100644 --- a/driver/net/core/src/interface.rs +++ b/driver/net/core/src/interface.rs @@ -1,6 +1,6 @@ use core::{ mem::size_of, - sync::atomic::{AtomicU32, Ordering}, + sync::atomic::{AtomicU32, AtomicUsize, Ordering}, }; use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; @@ -8,6 +8,7 @@ use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; use kernel_util::{ mem::PageBox, sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, + util::OneTimeInit, }; use yggdrasil_abi::{ error::Error, @@ -32,13 +33,28 @@ pub struct NetworkInterface { pub(crate) id: u32, } -static ETH_INTERFACES: IrqSafeRwLock>> = +#[derive(PartialEq, Eq)] +pub enum NetworkInterfaceType { + Ethernet, + Loopback, +} + +static INTERFACES: IrqSafeRwLock>> = IrqSafeRwLock::new(BTreeMap::new()); -static ETH_INTERFACE_ID: AtomicU32 = AtomicU32::new(1); +static LAST_INTERFACE_ID: AtomicU32 = AtomicU32::new(1); +static LOOPBACK: OneTimeInit> = OneTimeInit::new(); impl NetworkInterface { + pub fn id(&self) -> u32 { + self.id + } + + pub fn loopback() -> &'static Arc { + LOOPBACK.get() + } + pub fn get(id: u32) -> Result, Error> { - ETH_INTERFACES + INTERFACES .read() .get(&id) .cloned() @@ -46,7 +62,7 @@ impl NetworkInterface { } pub fn query_by_name(name: &str) -> Result, Error> { - ETH_INTERFACES + INTERFACES .read() .iter() .find_map(|(_, iface)| { @@ -60,7 +76,7 @@ impl NetworkInterface { } pub fn list_ref() -> IrqSafeRwLockReadGuard<'static, BTreeMap>> { - ETH_INTERFACES.read() + INTERFACES.read() } pub fn set_address(&self, address: IpAddr) { @@ -94,11 +110,23 @@ impl NetworkInterface { } } -pub fn register_interface(dev: &'static dyn NetworkDevice) -> u32 { - let mac = dev.read_hardware_address(); +pub fn register_interface( + ty: NetworkInterfaceType, + dev: &'static dyn NetworkDevice, +) -> Arc { + let name = match ty { + NetworkInterfaceType::Ethernet => { + static LAST_ETHERNET_ID: AtomicUsize = AtomicUsize::new(0); + let eth_id = LAST_ETHERNET_ID.fetch_add(1, Ordering::SeqCst); + format!("eth{}", eth_id).into_boxed_str() + } + NetworkInterfaceType::Loopback => "lo".into(), + }; - let id = ETH_INTERFACE_ID.fetch_add(1, Ordering::SeqCst); - let name = format!("eth{}", id).into_boxed_str(); + let mac = dev.read_hardware_address(); + let id = LAST_INTERFACE_ID.fetch_add(1, Ordering::SeqCst); + + log::info!("Register network interface {} (#{}): {}", name, id, mac); let iface = NetworkInterface { name, @@ -107,11 +135,14 @@ pub fn register_interface(dev: &'static dyn NetworkDevice) -> u32 { address: IrqSafeRwLock::new(None), id, }; - log::info!("Registered network interface #{}: {}", id, mac); let interface = Arc::new(iface); - ETH_INTERFACES.write().insert(id, interface); + INTERFACES.write().insert(id, interface.clone()); - id + if ty == NetworkInterfaceType::Loopback { + LOOPBACK.init(interface.clone()); + } + + interface } diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 022126ca..cdc9a1c6 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -70,6 +70,23 @@ impl Route { } pub fn lookup(address: IpAddr) -> Option<(u32, Option)> { + // TODO sort routes based on their "specificity"? + // Check for local route + for (_, interface) in NetworkInterface::list_ref().iter() { + if interface + .address + .read() + .map(|addr| addr == address) + .unwrap_or(false) + { + // This is the address of loopback, return it + return Some(( + NetworkInterface::loopback().id, + Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))), + )); + } + } + let routes = ROUTES.read(); for route in routes.iter() { if route.subnet.contains(&address) { diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index d233233b..8afc80b4 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -669,6 +669,7 @@ pub async fn handle(packet: L3Packet) -> Result<(), Error> { .await } else { // RST+ACK + log::warn!("SYN {} -> {}: port not listening", remote, local); let window_size = u16::from_network_order(tcp_frame.window_size); send( local, diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index bde19dcf..f9caf4e8 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -136,6 +136,23 @@ impl SocketTable { } } + pub fn try_insert_with_ephemeral_port Result, Error>>( + &mut self, + local: IpAddr, + mut with: F, + ) -> Result, Error> { + for port in 32768..u16::MAX - 1 { + let local = SocketAddr::new(local, port); + + match self.try_insert_with(local, || with(port)) { + Ok(socket) => return Ok(socket), + Err(Error::AddrInUse) => continue, + Err(error) => return Err(error), + } + } + Err(Error::AddrInUse) + } + pub fn try_insert_with Result, Error>>( &mut self, address: SocketAddr, @@ -189,19 +206,35 @@ static TCP_LISTENERS: IrqSafeRwLock> = impl UdpSocket { pub fn bind(address: SocketAddr) -> Result, Error> { let mut sockets = UDP_SOCKETS.write(); - sockets.try_insert_with(address, move || { - let socket = Arc::new(Self { - local: address, - remote: None, - broadcast: AtomicBool::new(false), - receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), - receive_notify: QueueWaker::new(), - }); + if address.port() == 0 { + sockets.try_insert_with_ephemeral_port(address.ip(), |port| { + let socket = Arc::new(Self { + local: SocketAddr::new(address.ip(), port), + remote: None, + broadcast: AtomicBool::new(false), + receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), + receive_notify: QueueWaker::new(), + }); - log::debug!("UDP socket opened: {}", address); + log::debug!("UDP socket opened: *{}", socket.local); - Ok(socket) - }) + Ok(socket) + }) + } else { + sockets.try_insert_with(address, move || { + let socket = Arc::new(Self { + local: address, + remote: None, + broadcast: AtomicBool::new(false), + receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), + receive_notify: QueueWaker::new(), + }); + + log::debug!("UDP socket opened: {}", address); + + Ok(socket) + }) + } } pub fn connect(&self, _address: SocketAddr) -> Result<(), Error> { diff --git a/driver/net/loopback/Cargo.toml b/driver/net/loopback/Cargo.toml new file mode 100644 index 00000000..0e721cdb --- /dev/null +++ b/driver/net/loopback/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ygg_driver_net_loopback" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-util = { path = "../../../lib/kernel-util" } + +ygg_driver_net_core = { path = "../../net/core" } diff --git a/driver/net/loopback/src/lib.rs b/driver/net/loopback/src/lib.rs new file mode 100644 index 00000000..b32836e2 --- /dev/null +++ b/driver/net/loopback/src/lib.rs @@ -0,0 +1,41 @@ +#![no_std] + +use kernel_util::{mem::PageBox, util::OneTimeInit}; +use ygg_driver_net_core::{ + interface::{NetworkDevice, NetworkInterfaceType}, + Packet, +}; +use yggdrasil_abi::{ + error::Error, + net::{IpAddr, Ipv4Addr, MacAddress}, +}; + +struct LoopbackDevice; + +impl NetworkDevice for LoopbackDevice { + fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> { + let packet = Packet::new(packet, 0, *LOOPBACK_ID.get()); + ygg_driver_net_core::receive_packet(packet) + } + + fn packet_prefix_size(&self) -> usize { + 0 + } + + fn read_hardware_address(&self) -> MacAddress { + MacAddress::UNSPECIFIED + } +} + +static LOOPBACK: OneTimeInit = OneTimeInit::new(); +static LOOPBACK_ID: OneTimeInit = OneTimeInit::new(); + +pub fn init() { + let loopback = LOOPBACK.init(LoopbackDevice); + let interface = + ygg_driver_net_core::register_interface(NetworkInterfaceType::Loopback, loopback); + + LOOPBACK_ID.init(interface.id()); + + interface.set_address(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); +} diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs index b7a05c60..a6e57834 100644 --- a/driver/virtio/net/src/lib.rs +++ b/driver/virtio/net/src/lib.rs @@ -18,7 +18,10 @@ use kernel_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, util::OneTimeInit, }; -use ygg_driver_net_core::{interface::NetworkDevice, Packet}; +use ygg_driver_net_core::{ + interface::{NetworkDevice, NetworkInterfaceType}, + Packet, +}; use ygg_driver_pci::{ capability::{MsiXCapability, MsiXVectorTable}, PciConfigurationSpace, PciDeviceInfo, @@ -258,8 +261,8 @@ impl Device for VirtioNet { self.finish_init(status); - let id = ygg_driver_net_core::register_interface(self); - self.interface_id.init(id); + let iface = ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self); + self.interface_id.init(iface.id()); self.listen(64); Ok(()) diff --git a/src/init.rs b/src/init.rs index 91149743..7eaf79eb 100644 --- a/src/init.rs +++ b/src/init.rs @@ -28,6 +28,7 @@ fn setup_root() -> Result { pub fn kinit() -> Result<(), Error> { infoln!("In main"); + ygg_driver_net_loopback::init(); ygg_driver_net_core::start_network_tasks()?; #[cfg(feature = "fb_console")] From cc32e537c9d5023fab8b90c7b43125878f17c647 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 26 Jan 2024 20:40:50 +0200 Subject: [PATCH 163/211] wwnet: add L3 loopback device --- driver/net/core/src/l3/mod.rs | 43 +++++++++++++++++++++++++++------- driver/net/core/src/l4/tcp.rs | 21 +++++++++++++---- driver/net/core/src/socket.rs | 4 +++- driver/net/loopback/Cargo.toml | 2 ++ driver/net/loopback/src/lib.rs | 5 ++-- 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index cdc9a1c6..2757d617 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -69,7 +69,7 @@ impl Route { ROUTES.read() } - pub fn lookup(address: IpAddr) -> Option<(u32, Option)> { + pub fn lookup(address: IpAddr) -> Option<(u32, Option, IpAddr)> { // TODO sort routes based on their "specificity"? // Check for local route for (_, interface) in NetworkInterface::list_ref().iter() { @@ -82,7 +82,8 @@ impl Route { // This is the address of loopback, return it return Some(( NetworkInterface::loopback().id, - Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))), + Some(IpAddr::V4(Ipv4Addr::LOOPBACK)), + IpAddr::V4(Ipv4Addr::LOOPBACK), )); } } @@ -90,7 +91,7 @@ impl Route { let routes = ROUTES.read(); for route in routes.iter() { if route.subnet.contains(&address) { - return Some((route.interface, route.gateway)); + return Some((route.interface, route.gateway, address)); } } None @@ -123,18 +124,19 @@ pub async fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { } } -fn resolve_l3_route( +pub(crate) fn resolve_l3_route( destination_ip: IpAddr, -) -> Result<(Arc, IpAddr, IpAddr), Error> { +) -> Result<(Arc, IpAddr, IpAddr, IpAddr), Error> { // Find the route itself - let (interface_id, gateway) = Route::lookup(destination_ip).ok_or(Error::NetworkUnreachable)?; + let (interface_id, gateway, destination_ip) = + Route::lookup(destination_ip).ok_or(Error::NetworkUnreachable)?; let interface = NetworkInterface::get(interface_id)?; // Route exists, but has no gateway (TODO: assign subnets to interfaces) let gateway = gateway.ok_or(Error::NetworkUnreachable)?; // Route exists, but network has no address assigned (TODO: how?) let source_address = interface.address.read().ok_or(Error::NetworkUnreachable)?; - Ok((interface, source_address, gateway)) + Ok((interface, source_address, gateway, destination_ip)) } fn make_ipv4_frame( @@ -170,14 +172,16 @@ fn make_ipv4_frame( Ok(l3_frame) } -pub async fn send_l4_ip( +pub async fn send_l4_ip_resolved( + interface: &NetworkInterface, + source_ip: IpAddr, + gateway_ip: IpAddr, destination_ip: IpAddr, protocol: IpProtocol, ttl: u8, l4_frame: &L4, l4_data: &[u8], ) -> Result<(), Error> { - let (interface, source_ip, gateway_ip) = resolve_l3_route(destination_ip)?; let gateway_mac = arp::lookup(interface.id, gateway_ip, true).await?; // TODO what if source_ip/gateway_ip/destination_ip are a mix of IPv4/IPv6? @@ -208,3 +212,24 @@ pub async fn send_l4_ip( let (packet, _len) = builder.finish(); interface.device.transmit(packet) } + +pub async fn send_l4_ip( + destination_ip: IpAddr, + protocol: IpProtocol, + ttl: u8, + l4_frame: &L4, + l4_data: &[u8], +) -> Result<(), Error> { + let (interface, source_ip, gateway_ip, destination_ip) = resolve_l3_route(destination_ip)?; + send_l4_ip_resolved( + &interface, + source_ip, + gateway_ip, + destination_ip, + protocol, + ttl, + l4_frame, + l4_data, + ) + .await +} diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index 8afc80b4..478981bf 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -548,9 +548,10 @@ async fn send( flags: TcpFlags, data: &[u8], ) -> Result<(), Error> { + let (interface, source_ip, gateway_ip, destination_ip) = l3::resolve_l3_route(remote.ip())?; // TODO TCPv6 - let local = local.into_ipv4().unwrap(); - let remote = remote.into_ipv4().unwrap(); + let source_ip = source_ip.into_ipv4().unwrap(); + let destination_ip = destination_ip.into_ipv4().unwrap(); let tcp_length = size_of::() + data.len(); @@ -565,8 +566,8 @@ async fn send( ..TcpFrame::zeroed() }; let pseudo_header = TcpV4PseudoHeader { - source_address: u32::from(local.ip()).to_network_order(), - destination_address: u32::from(remote.ip()).to_network_order(), + source_address: u32::from(source_ip).to_network_order(), + destination_address: u32::from(destination_ip).to_network_order(), _zero: 0, protocol: IpProtocol::TCP, tcp_length: (tcp_length as u16).to_network_order(), @@ -580,7 +581,17 @@ async fn send( frame.checksum = checksum.to_network_order(); - l3::send_l4_ip(remote.ip().into(), IpProtocol::TCP, 64, &frame, data).await + l3::send_l4_ip_resolved( + &interface, + source_ip.into(), + destination_ip.into(), + gateway_ip, + IpProtocol::TCP, + 64, + &frame, + data, + ) + .await } fn validate(source: IpAddr, destination: IpAddr, tcp_frame: &TcpFrame, data: &[u8]) -> bool { diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index f9caf4e8..de87874c 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -731,7 +731,9 @@ impl TcpSocket { async fn connect_async(remote: SocketAddr) -> Result<(SocketAddr, Arc), Error> { // Lookup route to remote - let (interface_id, _) = Route::lookup(remote.ip()).ok_or(Error::HostUnreachable)?; + let (interface_id, _, remote_ip) = + Route::lookup(remote.ip()).ok_or(Error::HostUnreachable)?; + let remote = SocketAddr::new(remote_ip, remote.port()); let interface = NetworkInterface::get(interface_id)?; let local_ip = interface.address.read().ok_or(Error::NetworkUnreachable)?; diff --git a/driver/net/loopback/Cargo.toml b/driver/net/loopback/Cargo.toml index 0e721cdb..8ab886ab 100644 --- a/driver/net/loopback/Cargo.toml +++ b/driver/net/loopback/Cargo.toml @@ -10,3 +10,5 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-util = { path = "../../../lib/kernel-util" } ygg_driver_net_core = { path = "../../net/core" } + +bytemuck = { version = "1.14.0", features = ["derive"] } diff --git a/driver/net/loopback/src/lib.rs b/driver/net/loopback/src/lib.rs index b32836e2..af3350a6 100644 --- a/driver/net/loopback/src/lib.rs +++ b/driver/net/loopback/src/lib.rs @@ -14,8 +14,7 @@ struct LoopbackDevice; impl NetworkDevice for LoopbackDevice { fn transmit(&self, packet: PageBox<[u8]>) -> Result<(), Error> { - let packet = Packet::new(packet, 0, *LOOPBACK_ID.get()); - ygg_driver_net_core::receive_packet(packet) + ygg_driver_net_core::receive_packet(Packet::new(packet, 0, *LOOPBACK_ID.get())) } fn packet_prefix_size(&self) -> usize { @@ -37,5 +36,5 @@ pub fn init() { LOOPBACK_ID.init(interface.id()); - interface.set_address(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); + interface.set_address(IpAddr::V4(Ipv4Addr::LOOPBACK)); } From 5a7d8a7f2056f8a69f373f84c14420ae018b9e94 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 27 Jan 2024 23:35:39 +0200 Subject: [PATCH 164/211] vfs: add TimerFd --- driver/net/core/src/l3/mod.rs | 167 ++++++++++++++------------- driver/net/core/src/l4/icmp.rs | 15 +-- driver/net/core/src/l4/tcp.rs | 27 +++-- driver/net/core/src/l4/udp.rs | 10 +- lib/kernel-util/src/runtime/mod.rs | 2 +- lib/kernel-util/src/runtime/timer.rs | 34 +++--- lib/vfs/src/file/mod.rs | 13 ++- lib/vfs/src/lib.rs | 2 + lib/vfs/src/timer.rs | 62 ++++++++++ src/syscall/mod.rs | 9 ++ 10 files changed, 222 insertions(+), 119 deletions(-) create mode 100644 lib/vfs/src/timer.rs diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 2757d617..08657ab9 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -11,7 +11,7 @@ use yggdrasil_abi::{ net::{ protocols::{EtherType, EthernetFrame, InetChecksum, IpProtocol, Ipv4Frame}, types::NetValueImpl, - IpAddr, Ipv4Addr, SubnetAddr, + IpAddr, Ipv4Addr, MacAddress, SubnetAddr, }, }; @@ -52,6 +52,27 @@ pub struct Route { pub gateway: Option, } +pub struct L4ResolvedPacket<'a, 'i> { + pub interface: &'i NetworkInterface, + pub source_ip: IpAddr, + pub gateway_ip: IpAddr, + pub destination_ip: IpAddr, + pub l4_frame: &'a [u8], + pub l4_options: &'a [u8], + pub l4_data: &'a [u8], + pub protocol: IpProtocol, + pub ttl: u8, +} + +pub struct L4UnresolvedPacket<'a> { + pub destination_ip: IpAddr, + pub l4_frame: &'a [u8], + pub l4_options: &'a [u8], + pub l4_data: &'a [u8], + pub protocol: IpProtocol, + pub ttl: u8, +} + static ROUTES: IrqSafeRwLock> = IrqSafeRwLock::new(Vec::new()); impl L3Packet { @@ -124,6 +145,50 @@ pub async fn handle_accepted(l3_packet: L3Packet) -> Result<(), Error> { } } +impl<'a, 'i> L4ResolvedPacket<'a, 'i> { + pub async fn lookup_gateway_mac(&self) -> Result { + arp::lookup(self.interface.id, self.gateway_ip, true).await + } + + pub fn make_l3_frame(&self) -> Result { + // TODO what if source_ip/gateway_ip/destination_ip are a mix of IPv4/IPv6? + let source_ip = self.source_ip.into_ipv4().ok_or(Error::NotImplemented)?; + let destination_ip = self + .destination_ip + .into_ipv4() + .ok_or(Error::NotImplemented)?; + + let total_length = (self.total_l4_len() + size_of::()) + .try_into() + .map_err(|_| Error::InvalidArgument)?; + + let mut l3_frame = Ipv4Frame { + source_address: u32::to_network_order(source_ip.into()), + destination_address: u32::to_network_order(destination_ip.into()), + protocol: self.protocol, + version_length: 0x45, + total_length: u16::to_network_order(total_length), + flags_frag: u16::to_network_order(0x4000), + id: u16::to_network_order(0), + ttl: self.ttl, + ..Ipv4Frame::zeroed() + }; + + let l3_frame_bytes = bytemuck::bytes_of(&l3_frame); + let mut ip_checksum = InetChecksum::new(); + ip_checksum.add_bytes(l3_frame_bytes, true); + let ip_checksum = ip_checksum.finish(); + + l3_frame.header_checksum = u16::to_network_order(ip_checksum); + + Ok(l3_frame) + } + + pub fn total_l4_len(&self) -> usize { + self.l4_frame.len() + self.l4_options.len() + self.l4_data.len() + } +} + pub(crate) fn resolve_l3_route( destination_ip: IpAddr, ) -> Result<(Arc, IpAddr, IpAddr, IpAddr), Error> { @@ -139,97 +204,43 @@ pub(crate) fn resolve_l3_route( Ok((interface, source_address, gateway, destination_ip)) } -fn make_ipv4_frame( - protocol: IpProtocol, - source_ip: Ipv4Addr, - destination_ip: Ipv4Addr, - payload_length: usize, - ttl: u8, -) -> Result { - let total_length = (payload_length + size_of::()) - .try_into() - .map_err(|_| Error::InvalidArgument)?; +pub async fn send_l4_ip_resolved(packet: &L4ResolvedPacket<'_, '_>) -> Result<(), Error> { + let gateway_mac = packet.lookup_gateway_mac().await?; - let mut l3_frame = Ipv4Frame { - source_address: u32::to_network_order(source_ip.into()), - destination_address: u32::to_network_order(destination_ip.into()), - protocol, - version_length: 0x45, - total_length: u16::to_network_order(total_length), - flags_frag: u16::to_network_order(0x4000), - id: u16::to_network_order(0), - ttl, - ..Ipv4Frame::zeroed() - }; - - let l3_frame_bytes = bytemuck::bytes_of(&l3_frame); - let mut ip_checksum = InetChecksum::new(); - ip_checksum.add_bytes(l3_frame_bytes, true); - let ip_checksum = ip_checksum.finish(); - - l3_frame.header_checksum = u16::to_network_order(ip_checksum); - - Ok(l3_frame) -} - -pub async fn send_l4_ip_resolved( - interface: &NetworkInterface, - source_ip: IpAddr, - gateway_ip: IpAddr, - destination_ip: IpAddr, - protocol: IpProtocol, - ttl: u8, - l4_frame: &L4, - l4_data: &[u8], -) -> Result<(), Error> { - let gateway_mac = arp::lookup(interface.id, gateway_ip, true).await?; - - // TODO what if source_ip/gateway_ip/destination_ip are a mix of IPv4/IPv6? - let source_ip = source_ip.into_ipv4().ok_or(Error::NotImplemented)?; - let destination_ip = destination_ip.into_ipv4().ok_or(Error::NotImplemented)?; - - let l3_frame = make_ipv4_frame( - protocol, - source_ip, - destination_ip, - size_of::() + l4_data.len(), - ttl, - )?; + let l3_frame = packet.make_l3_frame()?; let mut builder = PacketBuilder::new( - interface.device.packet_prefix_size(), - size_of::() + size_of::() + size_of::() + l4_data.len(), + packet.interface.device.packet_prefix_size(), + size_of::() + size_of::() + packet.total_l4_len(), )?; builder.push(&EthernetFrame { - source_mac: interface.mac, + source_mac: packet.interface.mac, destination_mac: gateway_mac, ethertype: EtherType::IPV4.to_network_order(), })?; builder.push(&l3_frame)?; - builder.push(l4_frame)?; - builder.push_bytes(l4_data)?; + builder.push_bytes(packet.l4_frame)?; + builder.push_bytes(packet.l4_options)?; + builder.push_bytes(packet.l4_data)?; - let (packet, _len) = builder.finish(); - interface.device.transmit(packet) + let (sent_packet, _len) = builder.finish(); + packet.interface.device.transmit(sent_packet) } -pub async fn send_l4_ip( - destination_ip: IpAddr, - protocol: IpProtocol, - ttl: u8, - l4_frame: &L4, - l4_data: &[u8], -) -> Result<(), Error> { - let (interface, source_ip, gateway_ip, destination_ip) = resolve_l3_route(destination_ip)?; - send_l4_ip_resolved( - &interface, +pub async fn send_l4_ip(packet: &L4UnresolvedPacket<'_>) -> Result<(), Error> { + let (interface, source_ip, gateway_ip, destination_ip) = + resolve_l3_route(packet.destination_ip)?; + + send_l4_ip_resolved(&L4ResolvedPacket { + interface: &interface, source_ip, gateway_ip, destination_ip, - protocol, - ttl, - l4_frame, - l4_data, - ) + l4_frame: packet.l4_frame, + l4_options: packet.l4_options, + l4_data: packet.l4_data, + protocol: packet.protocol, + ttl: packet.ttl, + }) .await } diff --git a/driver/net/core/src/l4/icmp.rs b/driver/net/core/src/l4/icmp.rs index 754d6c93..59dfe2e5 100644 --- a/driver/net/core/src/l4/icmp.rs +++ b/driver/net/core/src/l4/icmp.rs @@ -34,13 +34,14 @@ async fn send_v4_reply( reply_frame.checksum = checksum.finish().to_network_order(); - l3::send_l4_ip( - IpAddr::V4(destination_ip), - IpProtocol::ICMP, - 255, - &reply_frame, - icmp_data, - ) + l3::send_l4_ip(&l3::L4UnresolvedPacket { + destination_ip: IpAddr::V4(destination_ip), + l4_frame: bytemuck::bytes_of(&reply_frame), + l4_options: &[], + l4_data: icmp_data, + protocol: IpProtocol::ICMP, + ttl: 255, + }) .await } diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index 478981bf..31f39a4e 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -493,10 +493,8 @@ impl TcpConnection { let mut reply_flags = TcpFlags::empty(); let mut behavior = TcpSocketBehavior::None; - if data.len() != 0 { - if self.handle_packet_payload(data, packet.seq) { - reply_flags |= TcpFlags::ACK; - } + if !data.is_empty() && self.handle_packet_payload(data, packet.seq) { + reply_flags |= TcpFlags::ACK; } // TODO check window resize notification @@ -508,7 +506,7 @@ impl TcpConnection { ack_number = ack_number.wrapping_add(1); // Only send an actual FIN after a FIN without any data - if data.len() == 0 { + if data.is_empty() { reply_flags |= TcpFlags::FIN; // TODO go to LastAck state and wait for ACK self.state = TcpConnectionState::Closed; @@ -581,16 +579,17 @@ async fn send( frame.checksum = checksum.to_network_order(); - l3::send_l4_ip_resolved( - &interface, - source_ip.into(), - destination_ip.into(), + l3::send_l4_ip_resolved(&l3::L4ResolvedPacket { + interface: &interface, + source_ip: source_ip.into(), gateway_ip, - IpProtocol::TCP, - 64, - &frame, - data, - ) + destination_ip: destination_ip.into(), + l4_frame: bytemuck::bytes_of(&frame), + l4_options: &[], + l4_data: data, + protocol: IpProtocol::TCP, + ttl: 64, + }) .await } diff --git a/driver/net/core/src/l4/udp.rs b/driver/net/core/src/l4/udp.rs index 8e51918f..e99fb4c8 100644 --- a/driver/net/core/src/l4/udp.rs +++ b/driver/net/core/src/l4/udp.rs @@ -26,7 +26,15 @@ pub async fn send( checksum: 0u16.to_network_order(), }; - l3::send_l4_ip(destination_ip, IpProtocol::UDP, ttl, &udp_frame, data).await + l3::send_l4_ip(&l3::L4UnresolvedPacket { + destination_ip, + l4_frame: bytemuck::bytes_of(&udp_frame), + l4_options: &[], + l4_data: data, + protocol: IpProtocol::UDP, + ttl, + }) + .await } // pub fn send_broadcast( diff --git a/lib/kernel-util/src/runtime/mod.rs b/lib/kernel-util/src/runtime/mod.rs index 3778f2d2..8b537a04 100644 --- a/lib/kernel-util/src/runtime/mod.rs +++ b/lib/kernel-util/src/runtime/mod.rs @@ -9,5 +9,5 @@ mod waker; pub use executor::{run_to_completion, spawn, spawn_async_worker}; pub use task_queue::init_task_queue; -pub use timer::{run_with_timeout, sleep, tick, FutureTimeout}; +pub use timer::{run_with_timeout, sleep, tick, FutureTimeout, SleepFuture}; pub use waker::QueueWaker; diff --git a/lib/kernel-util/src/runtime/timer.rs b/lib/kernel-util/src/runtime/timer.rs index 05f23544..88769d11 100644 --- a/lib/kernel-util/src/runtime/timer.rs +++ b/lib/kernel-util/src/runtime/timer.rs @@ -75,29 +75,29 @@ pub enum FutureTimeout { Timeout, } -/// Suspends the task until given duration passes -pub fn sleep(duration: Duration) -> impl Future { - struct SleepFuture { - deadline: Duration, - } +pub struct SleepFuture { + deadline: Duration, +} - impl Future for SleepFuture { - type Output = (); +impl Future for SleepFuture { + type Output = (); - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let now = unsafe { api::__monotonic_timestamp() }.unwrap(); - match self.deadline.checked_sub(now) { - // Pending - Some(duration) if !duration.is_zero() => { - register_timeout(duration, cx.waker()); - Poll::Pending - } - // Ran out - _ => Poll::Ready(()), + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + match self.deadline.checked_sub(now) { + // Pending + Some(duration) if !duration.is_zero() => { + register_timeout(duration, cx.waker()); + Poll::Pending } + // Ran out + _ => Poll::Ready(()), } } +} +/// Suspends the task until given duration passes +pub fn sleep(duration: Duration) -> SleepFuture { let now = unsafe { api::__monotonic_timestamp() }.unwrap(); let deadline = now + duration; diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index d7e9c01d..14e4518e 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -28,7 +28,7 @@ use crate::{ socket::{ConnectionSocketWrapper, ListenerSocketWrapper, PacketSocketWrapper}, traits::{Read, Seek, Write}, ConnectionSocket, FdPoll, FileReadiness, ListenerSocket, PacketSocket, PseudoTerminal, - PseudoTerminalMaster, PseudoTerminalSlave, SharedMemory, Socket, + PseudoTerminalMaster, PseudoTerminalSlave, SharedMemory, Socket, TimerFile, }; use self::{ @@ -72,6 +72,7 @@ pub enum File { AnonymousPipe(PipeEnd), Poll(FdPoll), + Timer(TimerFile), Channel(ChannelDescriptor), SharedMemory(Arc), PtySlave(PseudoTerminalSlave), @@ -122,6 +123,11 @@ impl File { )) } + /// Creates a new [TimerFile]-backed File + pub fn new_timer(repeat: bool) -> FileRef { + Arc::new(Self::Timer(TimerFile::new(repeat))) + } + /// Constructs a [File] from a [PacketSocket] pub fn from_packet_socket(socket: Arc) -> Arc { Arc::new(Self::PacketSocket(Arc::new(PacketSocketWrapper(socket)))) @@ -257,6 +263,7 @@ impl File { Self::PacketSocket(sock) => sock.poll_read(cx), Self::StreamSocket(sock) => sock.poll_read(cx), Self::ListenerSocket(sock) => sock.poll_read(cx), + Self::Timer(timer) => timer.poll_read(cx), // Polling not implemented, return ready immediately (XXX ?) _ => Poll::Ready(Err(Error::NotImplemented)), } @@ -369,6 +376,8 @@ impl Read for File { Self::AnonymousPipe(pipe) => pipe.read(buf), Self::PtySlave(pt) => pt.read(buf), Self::PtyMaster(pt) => pt.read(buf), + // TODO maybe allow reading trigger count? + Self::Timer(_) => Err(Error::InvalidOperation), // TODO maybe allow reading FDs from poll channels as if they were regular streams? Self::Poll(_) => Err(Error::InvalidOperation), // TODO maybe allow reading messages from Channels? @@ -392,6 +401,7 @@ impl Write for File { Self::AnonymousPipe(pipe) => pipe.write(buf), Self::PtySlave(pt) => pt.write(buf), Self::PtyMaster(pt) => pt.write(buf), + Self::Timer(timer) => timer.write(buf), // TODO maybe allow adding FDs to poll channels this way Self::Poll(_) => Err(Error::InvalidOperation), // TODO maybe allow writing messages to Channels? @@ -467,6 +477,7 @@ impl fmt::Debug for File { .debug_struct("ListenerSocket") .field("local", &sock.local_address()) .finish_non_exhaustive(), + Self::Timer(_) => f.debug_struct("Timer").finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 03239420..7a76bb9f 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -20,6 +20,7 @@ pub(crate) mod poll; pub(crate) mod pty; pub(crate) mod shared_memory; pub(crate) mod socket; +pub(crate) mod timer; pub(crate) mod traits; pub use channel::{Channel, ChannelDescriptor, Message, MessagePayload, Subscription}; @@ -34,4 +35,5 @@ pub use poll::FdPoll; pub use pty::{PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave}; pub use shared_memory::SharedMemory; pub use socket::{ConnectionSocket, ListenerSocket, PacketSocket, Socket}; +pub use timer::TimerFile; pub use traits::{FileReadiness, Read, Seek, Write}; diff --git a/lib/vfs/src/timer.rs b/lib/vfs/src/timer.rs new file mode 100644 index 00000000..d1370376 --- /dev/null +++ b/lib/vfs/src/timer.rs @@ -0,0 +1,62 @@ +use core::{ + mem::size_of, + task::{Context, Poll}, + time::Duration, +}; + +use alloc::boxed::Box; + +use futures_util::FutureExt; +use kernel_util::{ + runtime::{self, SleepFuture}, + sync::{mutex::Mutex, LockMethod}, +}; +use yggdrasil_abi::error::Error; + +use crate::{FileReadiness, Write}; + +/// File-like structure to generate periodic or one-shit events at certain intervals +pub struct TimerFile { + inner: Box>>, + _repeat: bool, +} + +impl TimerFile { + /// Creates a new inert timer + pub fn new(repeat: bool) -> Self { + Self { + _repeat: repeat, + inner: Box::new(Mutex::new(None)), + } + } +} + +impl FileReadiness for TimerFile { + fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { + // TODO repeat function + let mut lock = self.inner.lock()?; + let future = lock.as_mut().ok_or(Error::InvalidOperation)?; + future.poll_unpin(cx).map(Ok) + } +} + +impl Write for TimerFile { + fn write(&self, buf: &[u8]) -> Result { + if buf.len() != size_of::() { + return Err(Error::InvalidArgument); + } + let mut bytes = [0; 16]; + bytes.copy_from_slice(buf); + let tval = u128::from_ne_bytes(bytes); + + let mut lock = self.inner.lock()?; + if tval == 0 { + *lock = None; + } else { + let duration = Duration::from_micros(tval.try_into().unwrap()); + *lock = Some(runtime::sleep(duration)); + } + + Ok(buf.len()) + } +} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index c66c1061..2efd2a52 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -499,6 +499,15 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(fd.0 as usize) }) } + SyscallFunction::CreateTimer => { + let repeat = args[0] != 0; + + run_with_io(process, |mut io| { + let file = File::new_timer(repeat); + let fd = io.files.place_file(file, true)?; + Ok(fd.0 as _) + }) + } // Process management SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; From 21692f748f538a219ebe43f172f618ce4d0f94a1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 28 Jan 2024 15:25:08 +0200 Subject: [PATCH 165/211] net/udp: fix UDP ephemeral port binding --- driver/net/core/src/socket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index de87874c..18f1a8dd 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -159,7 +159,7 @@ impl SocketTable { with: F, ) -> Result, Error> { if self.inner.contains_key(&address) { - return Err(Error::AlreadyExists); + return Err(Error::AddrInUse); } let socket = with()?; self.inner.insert(address, socket.clone()); From 9c559ca0eed52923021d26e2515cb2f88f77bf39 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 29 Jan 2024 22:33:21 +0200 Subject: [PATCH 166/211] x86_64: implement FPU context save/restore --- src/arch/x86_64/context.rs | 26 +++++++++++++++++--------- src/arch/x86_64/exception.rs | 3 +-- src/arch/x86_64/registers/mod.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index d3d0965b..810cc2fe 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -1,12 +1,12 @@ //! x86-64 task context setup and switching -use core::{arch::global_asm, cell::UnsafeCell}; +use core::{arch::global_asm, cell::UnsafeCell, mem::MaybeUninit}; use abi::error::Error; use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw}; use crate::{arch::x86_64::mem::KERNEL_TABLES, mem::phys, task::context::TaskContextImpl}; -use super::syscall::SyscallFrame; +use super::{registers::FpuContext, syscall::SyscallFrame}; struct StackBuilder { base: usize, @@ -25,6 +25,7 @@ struct Inner { #[allow(dead_code)] pub struct TaskContext { inner: UnsafeCell, + fpu_context: UnsafeCell, stack_base: usize, stack_size: usize, } @@ -117,10 +118,10 @@ impl TaskContext { let sp = stack.build(); let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; - debugln!("Fork TSS.RSP0 = {:#x}, cr3 = {:#x}", rsp0, cr3); Ok(Self { inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + fpu_context: UnsafeCell::new(FpuContext::new()), stack_base, stack_size: USER_TASK_PAGES * 0x1000, }) @@ -149,12 +150,12 @@ impl TaskContextImpl for TaskContext { ); let sp = stack.build(); - infoln!("Kernel stack {:#x}", sp); // TODO stack is leaked Ok(Self { inner: UnsafeCell::new(Inner { sp, tss_rsp0: 0 }), + fpu_context: UnsafeCell::new(FpuContext::new()), stack_base, stack_size: KERNEL_TASK_PAGES * 0x1000, }) @@ -181,25 +182,32 @@ impl TaskContextImpl for TaskContext { let sp = stack.build(); let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; - debugln!("TSS.RSP0 = {:#x}", rsp0); Ok(Self { inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + fpu_context: UnsafeCell::new(FpuContext::new()), stack_base, stack_size: USER_TASK_PAGES * 0x1000, }) } unsafe fn enter(&self) -> ! { + FpuContext::restore(self.fpu_context.get()); + __x86_64_enter_task(self.inner.get()) } unsafe fn switch(&self, from: &Self) { - let to = self.inner.get(); - let from = from.inner.get(); + let dst = self.inner.get(); + let src = from.inner.get(); - if to != from { - __x86_64_switch_task(to, from) + if dst != src { + // Save the old context + FpuContext::save(from.fpu_context.get()); + // Load next context + FpuContext::restore(self.fpu_context.get()); + + __x86_64_switch_task(dst, src); } } } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 0922109d..3ac46a63 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -383,8 +383,7 @@ fn kernel_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) -> ! { fatalln!("cr2 = {:#x}", cr2); } - fatalln!("CS:RIP = {:#x}:{:#x}", frame.cs, frame.rip); - fatalln!("SS:RSP = {:#x}:{:#x}", frame.ss, frame.rsp); + frame.dump(debug::LogLevel::Fatal); panic!("Irrecoverable exception"); } diff --git a/src/arch/x86_64/registers/mod.rs b/src/arch/x86_64/registers/mod.rs index 0d05e99a..04bd8c30 100644 --- a/src/arch/x86_64/registers/mod.rs +++ b/src/arch/x86_64/registers/mod.rs @@ -388,6 +388,8 @@ mod xcr0 { pub const XCR0: Reg = Reg; } +use core::ptr::NonNull; + pub use cr0::CR0; pub use cr3::CR3; pub use cr4::CR4; @@ -398,3 +400,27 @@ pub use msr_ia32_lstar::MSR_IA32_LSTAR; pub use msr_ia32_sfmask::MSR_IA32_SFMASK; pub use msr_ia32_star::MSR_IA32_STAR; pub use xcr0::XCR0; + +#[repr(C, align(0x10))] +pub struct FpuContext { + data: [u8; 512], +} + +impl FpuContext { + pub fn new() -> Self { + let mut value = Self { data: [0; 512] }; + unsafe { + let ptr = value.data.as_mut_ptr(); + core::arch::asm!("fninit; fxsave64 ({})", in(reg) ptr, options(att_syntax)); + } + value + } + + pub unsafe fn save(dst: *mut FpuContext) { + core::arch::asm!("fxsave64 ({})", in(reg) dst, options(att_syntax)); + } + + pub unsafe fn restore(src: *mut FpuContext) { + core::arch::asm!("fxrstor64 ({})", in(reg) src, options(att_syntax)); + } +} From 366143594d1d418968a12c58333d7b3a2d54d2a9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 30 Jan 2024 01:32:44 +0200 Subject: [PATCH 167/211] refactor: common async data structures --- driver/net/core/src/l3/ip.rs | 2 +- driver/net/core/src/lib.rs | 15 +- driver/net/core/src/queue.rs | 53 ------- driver/net/core/src/socket.rs | 151 ++++-------------- lib/kernel-util/src/util/event.rs | 140 +++++++++++++++++ lib/kernel-util/src/{util.rs => util/mod.rs} | 2 + lib/kernel-util/src/util/queue.rs | 119 ++++++++++++++ lib/kernel-util/src/util/ring.rs | 101 +++++++++++- lib/vfs/src/pty.rs | 156 +++++-------------- src/arch/x86_64/context.rs | 2 +- src/device/input.rs | 106 +------------ src/device/tty.rs | 1 + src/syscall/mod.rs | 7 +- src/task/process.rs | 93 +++++------ src/task/thread.rs | 52 ++----- src/util/mod.rs | 2 - src/util/queue.rs | 69 -------- 17 files changed, 495 insertions(+), 576 deletions(-) delete mode 100644 driver/net/core/src/queue.rs create mode 100644 lib/kernel-util/src/util/event.rs rename lib/kernel-util/src/{util.rs => util/mod.rs} (99%) create mode 100644 lib/kernel-util/src/util/queue.rs delete mode 100644 src/util/queue.rs diff --git a/driver/net/core/src/l3/ip.rs b/driver/net/core/src/l3/ip.rs index c0ae9b3d..9b9451e6 100644 --- a/driver/net/core/src/l3/ip.rs +++ b/driver/net/core/src/l3/ip.rs @@ -85,7 +85,7 @@ pub fn handle_v4_packet(packet: L2Packet) { data: packet.data, }; - ACCEPT_QUEUE.receive_packet(l3_packet).ok(); + ACCEPT_QUEUE.push_back(l3_packet); } else { // TODO forwarding log::debug!( diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index e93b6889..3dd8cb8f 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -9,9 +9,8 @@ use core::mem::size_of; use alloc::sync::Arc; use bytemuck::Pod; use ethernet::L2Packet; -use kernel_util::{mem::PageBox, runtime}; +use kernel_util::{mem::PageBox, runtime, util::queue::UnboundedMpmcQueue}; use l3::L3Packet; -use queue::Queue; use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; pub mod ethernet; @@ -22,7 +21,6 @@ pub mod socket; pub mod config; pub mod interface; -pub mod queue; pub mod util; pub use interface::register_interface; @@ -80,12 +78,13 @@ impl Packet { } } -static PACKET_QUEUE: Queue = Queue::new(); -static ACCEPT_QUEUE: Queue = Queue::new(); +static PACKET_QUEUE: UnboundedMpmcQueue = UnboundedMpmcQueue::new(); +static ACCEPT_QUEUE: UnboundedMpmcQueue = UnboundedMpmcQueue::new(); #[inline] pub fn receive_packet(packet: Packet) -> Result<(), Error> { - PACKET_QUEUE.receive_packet(packet) + PACKET_QUEUE.push_back(packet); + Ok(()) } pub fn start_network_tasks() -> Result<(), Error> { @@ -100,7 +99,7 @@ pub fn start_network_tasks() -> Result<(), Error> { async fn l2_packet_handler_worker() { loop { - let packet = PACKET_QUEUE.wait().await; + let packet = PACKET_QUEUE.pop_front().await; let eth_frame: &EthernetFrame = bytemuck::from_bytes( &packet.buffer[packet.offset..packet.offset + size_of::()], @@ -124,7 +123,7 @@ async fn l2_packet_handler_worker() { async fn l3_accept_worker() { loop { - let l3_packet = ACCEPT_QUEUE.wait().await; + let l3_packet = ACCEPT_QUEUE.pop_front().await; // log::debug!( // "INPUT {} {}:{:?} -> {}:{:?}: ACCEPT", diff --git a/driver/net/core/src/queue.rs b/driver/net/core/src/queue.rs deleted file mode 100644 index 759754ed..00000000 --- a/driver/net/core/src/queue.rs +++ /dev/null @@ -1,53 +0,0 @@ -use core::{ - future::Future, - pin::Pin, - task::{Context, Poll}, -}; - -use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBuffer}; -use yggdrasil_abi::error::Error; - -pub struct Queue { - queue: IrqSafeSpinlock>, - notify: QueueWaker, -} - -impl Queue { - pub const fn new() -> Self { - Self { - queue: IrqSafeSpinlock::new(RingBuffer::with_capacity(1024)), - notify: QueueWaker::new(), - } - } - - pub fn receive_packet(&self, packet: T) -> Result<(), Error> { - self.queue.lock().write(packet); - self.notify.wake_one(); - // TODO notify of dropped packets - Ok(()) - } - - pub fn wait(&self) -> impl Future + '_ { - struct F<'f, T> { - queue: &'f Queue, - } - - impl<'f, T> Future for F<'f, T> { - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.queue.notify.register(cx.waker()); - let mut lock = self.queue.queue.lock(); - - if lock.is_readable() { - self.queue.notify.remove(cx.waker()); - Poll::Ready(unsafe { lock.read_single_unchecked() }) - } else { - Poll::Pending - } - } - } - - F { queue: self } - } -} diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 18f1a8dd..ce466dfd 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -13,11 +13,10 @@ use kernel_util::{ monotonic_timestamp, runtime::{run_with_timeout, FutureTimeout, QueueWaker}, sync::{ - mutex::{Mutex, MutexGuard}, spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, - IrqSafeSpinlock, IrqSafeSpinlockGuard, LockMethod, + IrqSafeSpinlock, IrqSafeSpinlockGuard, }, - util::ring::RingBuffer, + util::queue::BoundedMpmcQueue, }; use vfs::{ConnectionSocket, FileReadiness, ListenerSocket, PacketSocket, Socket}; use yggdrasil_abi::{ @@ -42,8 +41,7 @@ pub struct UdpSocket { broadcast: AtomicBool, // TODO just place packets here for one less copy? - receive_queue: Mutex)>>, - receive_notify: QueueWaker, + receive_queue: BoundedMpmcQueue<(SocketAddr, Vec)>, } pub struct TcpSocket { @@ -66,8 +64,7 @@ pub struct TcpListener { pub struct RawSocket { id: u32, bound: IrqSafeSpinlock>, - receive_queue: Mutex>, - receive_notify: QueueWaker, + receive_queue: BoundedMpmcQueue, } pub struct SocketTable { @@ -204,36 +201,24 @@ static TCP_LISTENERS: IrqSafeRwLock> = IrqSafeRwLock::new(SocketTable::new()); impl UdpSocket { + fn create_socket(local: SocketAddr) -> Arc { + log::debug!("UDP socket opened: {}", local); + Arc::new(UdpSocket { + local, + remote: None, + broadcast: AtomicBool::new(false), + receive_queue: BoundedMpmcQueue::new(128), + }) + } + pub fn bind(address: SocketAddr) -> Result, Error> { let mut sockets = UDP_SOCKETS.write(); if address.port() == 0 { sockets.try_insert_with_ephemeral_port(address.ip(), |port| { - let socket = Arc::new(Self { - local: SocketAddr::new(address.ip(), port), - remote: None, - broadcast: AtomicBool::new(false), - receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), - receive_notify: QueueWaker::new(), - }); - - log::debug!("UDP socket opened: *{}", socket.local); - - Ok(socket) + Ok(Self::create_socket(SocketAddr::new(address.ip(), port))) }) } else { - sockets.try_insert_with(address, move || { - let socket = Arc::new(Self { - local: address, - remote: None, - broadcast: AtomicBool::new(false), - receive_queue: Mutex::new(RingBuffer::try_with_capacity(128)?), - receive_notify: QueueWaker::new(), - }); - - log::debug!("UDP socket opened: {}", address); - - Ok(socket) - }) + sockets.try_insert_with(address, move || Ok(Self::create_socket(address))) } } @@ -245,53 +230,16 @@ impl UdpSocket { UDP_SOCKETS.read().get(local) } - fn poll_receive( - &self, - cx: &mut Context<'_>, - ) -> Poll)>>, Error>> { - self.receive_notify.register(cx.waker()); - let lock = self.receive_queue.lock()?; - if lock.is_readable() { - self.receive_notify.remove(cx.waker()); - Poll::Ready(Ok(lock)) - } else { - Poll::Pending - } - } - - pub fn receive_raw(&self) -> impl Future), Error>> + '_ { - struct F<'f> { - socket: &'f UdpSocket, - } - - impl<'f> Future for F<'f> { - type Output = Result<(SocketAddr, Vec), Error>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match self.socket.poll_receive(cx)? { - Poll::Ready(mut lock) => { - let (source, data) = unsafe { lock.read_single_unchecked() }; - Poll::Ready(Ok((source, data))) - } - Poll::Pending => Poll::Pending, - } - } - } - - F { socket: self } - } - pub fn packet_received(&self, source: SocketAddr, data: &[u8]) -> Result<(), Error> { - let mut lock = self.receive_queue.lock()?; - lock.write((source, Vec::from(data))); - self.receive_notify.wake_one(); - Ok(()) + self.receive_queue + .try_push_back((source, Vec::from(data))) + .map_err(|_| Error::QueueFull) } } impl FileReadiness for UdpSocket { fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { - self.poll_receive(cx).map_ok(|_| ()) + self.receive_queue.poll_not_empty(cx).map(Ok) } } @@ -323,9 +271,10 @@ impl PacketSocket for UdpSocket { } fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error> { - let (source, data) = block!(self.receive_raw().await)??; + let (source, data) = block!(self.receive_queue.pop_front().await)?; if data.len() > buffer.len() { - todo!() + // TODO check how other OSs handle this + return Err(Error::BufferTooSmall); } buffer[..data.len()].copy_from_slice(&data); Ok((source, data.len())) @@ -374,8 +323,7 @@ impl RawSocket { let socket = Self { id, bound: IrqSafeSpinlock::new(None), - receive_queue: Mutex::new(RingBuffer::with_capacity(128)), - receive_notify: QueueWaker::new(), + receive_queue: BoundedMpmcQueue::new(256), }; let socket = Arc::new(socket); RAW_SOCKETS.write().insert(id, socket.clone()); @@ -384,22 +332,8 @@ impl RawSocket { } fn bound_packet_received(&self, packet: L2Packet) { - self.receive_queue.lock().unwrap().write(packet); - self.receive_notify.wake_one(); - } - - fn poll_receive( - &self, - cx: &mut Context<'_>, - ) -> Poll>, Error>> { - self.receive_notify.register(cx.waker()); - let lock = self.receive_queue.lock()?; - if lock.is_readable() { - self.receive_notify.remove(cx.waker()); - Poll::Ready(Ok(lock)) - } else { - Poll::Pending - } + // TODO do something with the dropped packet? + self.receive_queue.try_push_back(packet).ok(); } pub fn packet_received(packet: L2Packet) { @@ -413,33 +347,11 @@ impl RawSocket { } } } - - pub fn receive_raw(&self) -> impl Future> + '_ { - struct F<'f> { - socket: &'f RawSocket, - } - - impl<'f> Future for F<'f> { - type Output = Result; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match self.socket.poll_receive(cx)? { - Poll::Ready(mut lock) => { - let packet = unsafe { lock.read_single_unchecked() }; - Poll::Ready(Ok(packet)) - } - Poll::Pending => Poll::Pending, - } - } - } - - F { socket: self } - } } impl FileReadiness for RawSocket { fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { - self.poll_receive(cx).map_ok(|_| ()) + self.receive_queue.poll_not_empty(cx).map(Ok) } } @@ -529,16 +441,13 @@ impl PacketSocket for RawSocket { } fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error> { - let data = block!(self.receive_raw().await)??; + let data = block!(self.receive_queue.pop_front().await)?; let len = 4096 - data.l2_offset; if buffer.len() < len { - todo!() + return Err(Error::BufferTooSmall); } buffer[..len].copy_from_slice(&data.data[data.l2_offset..]); - Ok(( - SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)), - len, - )) + Ok((SocketAddr::NULL_V4, len)) } } diff --git a/lib/kernel-util/src/util/event.rs b/lib/kernel-util/src/util/event.rs new file mode 100644 index 00000000..09455a0c --- /dev/null +++ b/lib/kernel-util/src/util/event.rs @@ -0,0 +1,140 @@ +use core::{ + future::poll_fn, + sync::atomic::{AtomicBool, Ordering}, + task::Poll, +}; + +use crate::{runtime::QueueWaker, sync::spin_rwlock::IrqSafeRwLock}; + +pub struct OneTimeEvent { + // TODO lockless like OneTimeInit? + value: IrqSafeRwLock>, + notify: QueueWaker, +} + +pub struct BoolEvent { + state: AtomicBool, + notify: QueueWaker, +} + +impl OneTimeEvent { + /// Constructs an [OneTimeEvent] + pub const fn new() -> Self { + Self { + value: IrqSafeRwLock::new(None), + notify: QueueWaker::new(), + } + } + + /// Signals a value into the event. Returns an [Err] with the value if the event + /// has already been signalled before. + pub fn try_signal(&self, value: T) -> Result<(), T> { + let mut lock = self.value.write(); + if lock.is_some() { + return Err(value); + } + lock.replace(value); + self.notify.wake_all(); + Ok(()) + } + + /// Signals a value into the event. + /// + /// # Panics + /// + /// Will panic if the event has already been signal. Use this function only when + /// the event is known to only be signalled once. + pub fn signal(&self, value: T) { + if self.try_signal(value).is_err() { + panic!("OneTimeEvent signalled more than once"); + } + } + + /// Returns `true` if a value has been signalled into the event + pub fn is_signalled(&self) -> bool { + self.value.read().is_some() + } + + /// Returns [Some] with the copy of the value if the event has been signalled, + /// [None] otherwise + pub fn try_read_copy(&self) -> Option + where + T: Copy, + { + *self.value.read() + } + + /// Waits for event to happen without returning anything + pub async fn wait(&self) { + poll_fn(|cx| { + self.notify.register(cx.waker()); + if self.is_signalled() { + self.notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await + } + + /// Waits for event to happen and copies its data as a return value + pub async fn wait_copy(&self) -> T + where + T: Copy, + { + poll_fn(|cx| { + self.notify.register(cx.waker()); + if let Some(value) = self.try_read_copy() { + self.notify.remove(cx.waker()); + Poll::Ready(value) + } else { + Poll::Pending + } + }) + .await + } +} + +impl BoolEvent { + pub const fn new() -> Self { + Self { + state: AtomicBool::new(false), + notify: QueueWaker::new(), + } + } + + pub fn is_signalled(&self) -> bool { + self.state.load(Ordering::Acquire) + } + + pub fn signal(&self) { + if self.try_signal().is_err() { + panic!("OneTimeBoolEvent signalled more than once"); + } + } + + pub fn signal_saturating(&self) { + self.try_signal().ok(); + } + + pub fn try_signal(&self) -> Result<(), bool> { + self.state + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)?; + self.notify.wake_all(); + Ok(()) + } + + pub async fn wait(&self) { + poll_fn(|cx| { + self.notify.register(cx.waker()); + if self.is_signalled() { + self.notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await + } +} diff --git a/lib/kernel-util/src/util.rs b/lib/kernel-util/src/util/mod.rs similarity index 99% rename from lib/kernel-util/src/util.rs rename to lib/kernel-util/src/util/mod.rs index 252ac535..2f306efb 100644 --- a/lib/kernel-util/src/util.rs +++ b/lib/kernel-util/src/util/mod.rs @@ -7,6 +7,8 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; +pub mod event; +pub mod queue; pub mod ring; pub enum ConstAssert {} diff --git a/lib/kernel-util/src/util/queue.rs b/lib/kernel-util/src/util/queue.rs new file mode 100644 index 00000000..c42dc39f --- /dev/null +++ b/lib/kernel-util/src/util/queue.rs @@ -0,0 +1,119 @@ +use core::{ + future::poll_fn, + task::{Context, Poll}, +}; + +use crossbeam_queue::{ArrayQueue, SegQueue}; + +use crate::runtime::QueueWaker; + +pub type BoundedQueue = ArrayQueue; +pub type UnboundedQueue = SegQueue; + +pub struct BoundedMpmcQueue { + queue: BoundedQueue, + read_notify: QueueWaker, + write_notify: QueueWaker, +} + +pub struct UnboundedMpmcQueue { + queue: UnboundedQueue, + read_notify: QueueWaker, +} + +impl BoundedMpmcQueue { + pub fn new(capacity: usize) -> Self { + Self { + queue: BoundedQueue::new(capacity), + read_notify: QueueWaker::new(), + write_notify: QueueWaker::new(), + } + } + + pub fn try_push_back(&self, value: T) -> Result<(), T> { + self.queue.push(value)?; + self.read_notify.wake_one(); + Ok(()) + } + + pub fn try_pop_front(&self) -> Option { + let value = self.queue.pop()?; + self.write_notify.wake_one(); + Some(value) + } + + pub async fn pop_front(&self) -> T { + poll_fn(|cx| { + self.read_notify.register(cx.waker()); + match self.try_pop_front() { + Some(value) => { + self.read_notify.remove(cx.waker()); + Poll::Ready(value) + } + None => Poll::Pending, + } + }) + .await + } + + pub async fn push_back(&self, value: T) { + let mut value = Some(value); + poll_fn(|cx| { + self.write_notify.register(cx.waker()); + match self.try_push_back(value.take().unwrap()) { + Ok(()) => { + // Succeeded + self.write_notify.remove(cx.waker()); + Poll::Ready(()) + } + Err(back) => { + // Failed, put the value back + value.replace(back); + Poll::Pending + } + } + }) + .await + } + + pub fn poll_not_empty(&self, cx: &mut Context<'_>) -> Poll<()> { + self.read_notify.register(cx.waker()); + if !self.queue.is_empty() { + self.read_notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } +} + +impl UnboundedMpmcQueue { + pub const fn new() -> Self { + Self { + queue: SegQueue::new(), + read_notify: QueueWaker::new(), + } + } + + pub fn push_back(&self, value: T) { + self.queue.push(value); + self.read_notify.wake_one(); + } + + pub fn try_pop_front(&self) -> Option { + self.queue.pop() + } + + pub async fn pop_front(&self) -> T { + poll_fn(|cx| { + self.read_notify.register(cx.waker()); + if let Some(value) = self.try_pop_front() { + self.read_notify.remove(cx.waker()); + Poll::Ready(value) + } else { + Poll::Pending + } + }) + .await + } +} diff --git a/lib/kernel-util/src/util/ring.rs b/lib/kernel-util/src/util/ring.rs index 009e2a80..cbacddd8 100644 --- a/lib/kernel-util/src/util/ring.rs +++ b/lib/kernel-util/src/util/ring.rs @@ -1,19 +1,35 @@ //! Ring buffer implementation -use core::mem::MaybeUninit; +use core::{ + future::poll_fn, + mem::MaybeUninit, + task::{Context, Poll}, +}; use alloc::boxed::Box; use yggdrasil_abi::error::Error; +use crate::{ + runtime::QueueWaker, + sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, +}; + +// TODO use lockless ring data structure + /// Ring buffer base -pub struct RingBuffer { +pub struct RingBuffer { rd: usize, wr: usize, capacity: usize, data: Option]>>, } -impl RingBuffer { +pub struct LossyRingQueue { + ring: IrqSafeSpinlock>, + read_notify: QueueWaker, +} + +impl RingBuffer { /// Constructs an empty [RingBuffer]. /// /// NOTE does not allocate until the first write operation. Any read accesses @@ -126,4 +142,83 @@ impl RingBuffer { data[self.wr].write(ch); self.wr = (self.wr + 1) % self.capacity; } + + /// Returns [Some] with a value if it could be read, [None] otherwise + pub fn try_read(&mut self) -> Option { + if self.is_readable() { + Some(unsafe { self.read_single_unchecked() }) + } else { + None + } + } +} + +impl LossyRingQueue { + pub const fn with_capacity(capacity: usize) -> Self { + Self { + ring: IrqSafeSpinlock::new(RingBuffer::with_capacity(capacity)), + read_notify: QueueWaker::new(), + } + } + + pub fn try_with_capacity(capacity: usize) -> Result { + Ok(Self { + ring: IrqSafeSpinlock::new(RingBuffer::try_with_capacity(capacity)?), + read_notify: QueueWaker::new(), + }) + } + + pub fn write(&self, value: T) { + self.ring.lock().write(value); + self.read_notify.wake_one(); + } + + pub fn write_multiple(&self, values: &[T]) { + let mut lock = self.ring.lock(); + for &value in values { + lock.write(value); + } + self.read_notify.wake_one(); + } + + pub fn try_read(&self) -> Option { + // SAFETY If lock is acquired, then there's at least one readable element + self.try_read_lock() + .map(|mut f| unsafe { f.read_single_unchecked() }) + } + + #[inline] + pub fn try_read_lock(&self) -> Option>> { + let lock = self.ring.lock(); + lock.is_readable().then_some(lock) + } + + pub async fn read(&self) -> T { + let mut lock = self.read_lock().await; + // SAFETY If lock is acquired, then there's at least one readable element + unsafe { lock.read_single_unchecked() } + } + + pub async fn read_lock(&self) -> IrqSafeSpinlockGuard> { + poll_fn(|cx| self.poll_lock(cx)).await + } + + #[inline] + pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll>> { + self.read_notify.register(cx.waker()); + if let Some(lock) = self.try_read_lock() { + self.read_notify.remove(cx.waker()); + Poll::Ready(lock) + } else { + Poll::Pending + } + } + + pub fn poll_readable(&self, cx: &mut Context<'_>) -> Poll<()> { + self.poll_lock(cx).map(|_| ()) + } + + pub fn notify_all(&self) { + self.read_notify.wake_all(); + } } diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 3b78a3d6..bc762dbb 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -3,19 +3,15 @@ // TODO handle werase key use core::{ mem::MaybeUninit, - pin::Pin, sync::atomic::{AtomicBool, Ordering}, task::{Context, Poll}, }; use alloc::{boxed::Box, sync::Arc}; -use futures_util::Future; use kernel_util::{ - block, - runtime::QueueWaker, - signal_process_group, - sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, - util::ring::RingBuffer, + block, signal_process_group, + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + util::ring::{LossyRingQueue, RingBuffer}, }; use yggdrasil_abi::{ error::Error, @@ -29,24 +25,20 @@ use yggdrasil_abi::{ const CAPACITY: usize = 8192; struct PtySlaveToMasterHalf { - ring: IrqSafeSpinlock>, - notify: QueueWaker, + ring: LossyRingQueue, shutdown: AtomicBool, } struct MasterToSlaveBuffer { pending: Box<[MaybeUninit]>, - ready: RingBuffer, - position: usize, } struct PtyMasterToSlaveHalf { // Actual data to be read by the slave buffer: IrqSafeSpinlock, + ready_ring: LossyRingQueue, signal_pgroup: IrqSafeRwLock>, - // Notified when data is available in the buffer - notify: QueueWaker, } /// Pseudo-terminal shared device @@ -69,8 +61,9 @@ pub struct PseudoTerminalMaster { fn read_all(source: &mut RingBuffer, target: &mut [u8], eof: Option) -> usize { let mut pos = 0; - while pos < target.len() && source.is_readable() { - let ch = unsafe { source.read_single_unchecked() }; + while pos < target.len() + && let Some(ch) = source.try_read() + { if eof.map(|eof| eof == ch).unwrap_or(false) { break; } @@ -83,7 +76,7 @@ fn read_all(source: &mut RingBuffer, target: &mut [u8], eof: Option) -> impl MasterToSlaveBuffer { pub fn write_pending(&mut self, byte: u8) { if self.position == self.pending.len() { - // TODO not sure how to handle this case + // TODO flush the buffer todo!(); } @@ -100,21 +93,10 @@ impl MasterToSlaveBuffer { } } - pub fn write_ready(&mut self, byte: u8) { - self.ready.write(byte); - } - - pub fn flush(&mut self) { - if self.position == 0 { - return; - } - log::debug!("m->s: flush {}", self.position); - - for &ch in unsafe { MaybeUninit::slice_assume_init_ref(&self.pending[..self.position]) } { - self.ready.write(ch); - } - + pub fn flush(&mut self) -> &[u8] { + let data = unsafe { MaybeUninit::slice_assume_init_ref(&self.pending[..self.position]) }; self.position = 0; + data } } @@ -123,98 +105,48 @@ impl PtyMasterToSlaveHalf { Ok(Self { buffer: IrqSafeSpinlock::new(MasterToSlaveBuffer { pending: Box::new_uninit_slice(256), - ready: RingBuffer::try_with_capacity(capacity)?, position: 0, }), + ready_ring: LossyRingQueue::try_with_capacity(capacity)?, signal_pgroup: IrqSafeRwLock::new(None), - notify: QueueWaker::new(), }) } pub fn read(&self, buf: &mut [u8], eof: Option) -> Result { - if let Some(mut lock) = self.try_read() { - Ok(read_all(&mut lock.ready, buf, eof)) + if let Some(mut lock) = self.ready_ring.try_read_lock() { + Ok(read_all(&mut lock, buf, eof)) } else { block!(self.read_async(buf, eof).await)? } } - pub fn poll(&self, cx: &mut Context<'_>) -> Poll> { - self.notify.register(cx.waker()); - - if self.try_read().is_some() { - self.notify.remove(cx.waker()); - Poll::Ready(Ok(())) - } else { - Poll::Pending - } + pub fn poll_readable(&self, cx: &mut Context<'_>) -> Poll> { + self.ready_ring.poll_readable(cx).map(Ok) } pub fn flush(&self) { - self.buffer.lock().flush(); - self.notify.wake_all(); + let mut lock = self.buffer.lock(); + let data = lock.flush(); + self.ready_ring.write_multiple(data); } - fn try_read(&self) -> Option> { - let lock = self.buffer.lock(); - lock.ready.is_readable().then_some(lock) - } - - fn read_async<'a>( - &'a self, - buffer: &'a mut [u8], - eof: Option, - ) -> impl Future> + 'a { - struct F<'f> { - this: &'f PtyMasterToSlaveHalf, - buffer: &'f mut [u8], - eof: Option, - } - - impl<'f> Future for F<'f> { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let eof = self.eof; - self.this.notify.register(cx.waker()); - - if let Some(mut lock) = self.this.try_read() { - self.this.notify.remove(cx.waker()); - let count = read_all(&mut lock.ready, self.buffer, eof); - Poll::Ready(Ok(count)) - } else { - Poll::Pending - } - } - } - - F { - this: self, - buffer, - eof, - } + async fn read_async(&self, buffer: &mut [u8], eof: Option) -> Result { + let mut lock = self.ready_ring.read_lock().await; + Ok(read_all(&mut lock, buffer, eof)) } } impl PtySlaveToMasterHalf { pub fn with_capacity(capacity: usize) -> Result { Ok(Self { - ring: IrqSafeSpinlock::new(RingBuffer::try_with_capacity(capacity)?), - notify: QueueWaker::new(), + ring: LossyRingQueue::try_with_capacity(capacity)?, shutdown: AtomicBool::new(false), }) } pub fn handle_input(&self, byte: u8, _config: &TerminalOutputOptions) { - let mut lock = self.ring.lock(); - - // if byte == b'\n' && config.output.contains(TerminalOutputOptions::NL_TO_CRNL) { - // self.slave_to_master.putc(b'\r', true); - // } - - lock.write(byte); - - self.notify.wake_one(); + // TODO handle output flags + self.ring.write(byte); } pub fn read(&self, buf: &mut [u8]) -> Result { @@ -222,7 +154,7 @@ impl PtySlaveToMasterHalf { return Ok(0); } - if let Some(mut lock) = self.try_read() { + if let Some(mut lock) = self.ring.try_read_lock() { let count = read_all(&mut lock, buf, None); Ok(count) } else { @@ -230,21 +162,13 @@ impl PtySlaveToMasterHalf { } } - pub fn poll(&self, cx: &mut Context<'_>) -> Poll> { - self.notify.register(cx.waker()); - - if self.shutdown.load(Ordering::Acquire) || self.ring.lock().is_readable() { - self.notify.remove(cx.waker()); + pub fn poll_readable(&self, cx: &mut Context<'_>) -> Poll> { + if self.shutdown.load(Ordering::Acquire) || self.ring.poll_readable(cx).is_ready() { Poll::Ready(Ok(())) } else { Poll::Pending } } - - fn try_read(&self) -> Option>> { - let lock = self.ring.lock(); - lock.is_readable().then_some(lock) - } } impl PseudoTerminal { @@ -315,11 +239,11 @@ impl PseudoTerminal { if byte == config.chars.interrupt { if config.line.contains(TerminalLineOptions::SIGNAL) { - self.slave_to_master.notify.wake_all(); + self.slave_to_master.ring.notify_all(); if let Some(group_id) = *self.master_to_slave.signal_pgroup.read() { signal_process_group(group_id, Signal::Interrupted); - self.master_to_slave.notify.wake_all(); + self.master_to_slave.ready_ring.notify_all(); return; } } else { @@ -330,13 +254,12 @@ impl PseudoTerminal { } if byte == b'\n' || byte == config.chars.eof { - buffer.flush(); - self.master_to_slave.notify.wake_one(); + let data = buffer.flush(); + self.master_to_slave.ready_ring.write_multiple(data); } } else { // Raw line processing - buffer.write_ready(byte); - self.master_to_slave.notify.wake_one(); + self.master_to_slave.ready_ring.write(byte); } } @@ -367,19 +290,16 @@ impl PseudoTerminal { } fn poll_from_slave(&self, cx: &mut Context<'_>) -> Poll> { - self.slave_to_master.poll(cx) + self.slave_to_master.poll_readable(cx) } fn poll_from_master(&self, cx: &mut Context<'_>) -> Poll> { - self.master_to_slave.poll(cx) + self.master_to_slave.poll_readable(cx) } fn close_master(&self) { let config = self.config.read(); - let mut buffer = self.master_to_slave.buffer.lock(); - - buffer.write_ready(config.chars.eof); - self.master_to_slave.notify.wake_one(); + self.master_to_slave.ready_ring.write(config.chars.eof); } fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { @@ -471,6 +391,6 @@ impl Drop for PseudoTerminalSlave { .slave_to_master .shutdown .store(true, Ordering::Release); - self.pty.slave_to_master.notify.wake_all(); + self.pty.slave_to_master.ring.notify_all(); } } diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 810cc2fe..13dc52c8 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -1,5 +1,5 @@ //! x86-64 task context setup and switching -use core::{arch::global_asm, cell::UnsafeCell, mem::MaybeUninit}; +use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw}; diff --git a/src/device/input.rs b/src/device/input.rs index 20539110..1b7bdf13 100644 --- a/src/device/input.rs +++ b/src/device/input.rs @@ -1,114 +1,20 @@ // TODO #![allow(missing_docs)] -use core::{ - marker::PhantomData, - pin::Pin, - task::{Context, Poll}, -}; +use core::task::{Context, Poll}; use abi::{ error::Error, io::{DeviceRequest, KeyboardKeyEvent}, }; -use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{ - block, - sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, - util::ring::RingBuffer, -}; +use kernel_util::{block, util::ring::LossyRingQueue}; use vfs::{CharDevice, FileReadiness}; -// #[derive(Default)] -// pub struct InputState { -// lshift: bool, -// rshift: bool, -// lalt: bool, -// ralt: bool, -// lcontrol: bool, -// rcontrol: bool, -// } - -pub struct MpscChannel { - queue: IrqSafeSpinlock>, - notify: AtomicWaker, -} - pub struct KeyboardDevice; -// impl InputState { -// pub fn shift(&self) -> bool { -// self.lshift || self.rshift -// } -// -// pub fn alt(&self) -> bool { -// self.lalt || self.ralt -// } -// -// pub fn control(&self) -> bool { -// self.lcontrol || self.rcontrol -// } -// } - -impl MpscChannel { - pub const fn new() -> Self { - Self { - queue: IrqSafeSpinlock::new(RingBuffer::with_capacity(128)), - notify: AtomicWaker::new(), - } - } - - pub fn send(&self, value: T) { - self.queue.lock().write(value); - self.notify.wake(); - } - - pub fn poll_read(&self, cx: &mut Context<'_>) -> Poll>> { - let lock = self.queue.lock(); - if lock.is_readable() { - return Poll::Ready(lock); - } - drop(lock); - - self.notify.register(cx.waker()); - - let lock = self.queue.lock(); - if lock.is_readable() { - Poll::Ready(lock) - } else { - Poll::Pending - } - } - - pub fn receive(&self) -> impl Future + '_ - where - T: Copy, - { - struct F<'f, T: Copy> { - channel: &'f MpscChannel, - _pd: PhantomData, - } - - impl<'f, T: Copy> Future for F<'f, T> { - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.channel - .poll_read(cx) - .map(|mut lock| unsafe { lock.read_single_unchecked() }) - } - } - - F { - channel: self, - _pd: PhantomData, - } - } -} - impl FileReadiness for KeyboardDevice { fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { - INPUT_QUEUE.poll_read(cx).map(|_| Ok(())) + INPUT_QUEUE.poll_readable(cx).map(Ok) } } @@ -118,7 +24,7 @@ impl CharDevice for KeyboardDevice { return Ok(0); } - let ev = block!(INPUT_QUEUE.receive().await)?; + let ev = block!(INPUT_QUEUE.read().await)?; buf[..4].copy_from_slice(&ev.as_bytes()); @@ -138,9 +44,9 @@ impl CharDevice for KeyboardDevice { } } -static INPUT_QUEUE: MpscChannel = MpscChannel::new(); +static INPUT_QUEUE: LossyRingQueue = LossyRingQueue::with_capacity(32); pub static KEYBOARD_DEVICE: KeyboardDevice = KeyboardDevice; pub fn send_event(ev: KeyboardKeyEvent) { - INPUT_QUEUE.send(ev); + INPUT_QUEUE.write(ev); } diff --git a/src/device/tty.rs b/src/device/tty.rs index 3a54f8bc..8cb96ef4 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -91,6 +91,7 @@ pub struct TtyContext { inner: IrqSafeSpinlock, } +// TODO merge this code with PTY /// Terminal device interface pub trait TtyDevice: SerialDevice { /// Returns the ring buffer associated with the device diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 2efd2a52..760a88f0 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -659,12 +659,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let target = Process::get(pid).ok_or(Error::DoesNotExist)?; - *status = match block! { - Process::wait_for_exit(target).await - } { - Ok(status) => status, - Err(err) => return Err(err), - }; + *status = block!(target.exit.wait_copy().await)?; Ok(0) } diff --git a/src/task/process.rs b/src/task/process.rs index 11f4543e..b38f7fc1 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -3,9 +3,7 @@ use core::{ fmt, mem::size_of, - pin::Pin, sync::atomic::{AtomicU64, Ordering}, - task::{Context, Poll}, }; use abi::{ @@ -13,10 +11,9 @@ use abi::{ process::{ExitCode, Signal, ThreadSpawnOptions}, }; use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; -use futures_util::Future; use kernel_util::{ - runtime::QueueWaker, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + util::event::OneTimeEvent, }; use vfs::{IoContext, NodeRef}; @@ -33,15 +30,6 @@ use super::{ TaskContext, }; -/// Represents a process state -#[derive(PartialEq)] -pub enum ProcessState { - /// Process is running, meaning it still has at least one thread alive - Running, - /// Process has finished with some [ExitCode] - Terminated(ExitCode), -} - /// Unique number assigned to each [Process] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] @@ -139,8 +127,6 @@ pub struct ProcessImage { } struct ProcessInner { - state: ProcessState, - session_id: ProcessId, group_id: ProcessId, @@ -159,7 +145,8 @@ pub struct Process { inner: IrqSafeRwLock, - exit_waker: QueueWaker, + pub(crate) exit: OneTimeEvent, + /// Process I/O information pub io: IrqSafeSpinlock, } @@ -183,7 +170,6 @@ impl Process { id, inner: IrqSafeRwLock::new(ProcessInner { - state: ProcessState::Running, session_id: id, group_id: id, session_terminal: None, @@ -194,7 +180,7 @@ impl Process { space: space.clone(), }), - exit_waker: QueueWaker::new(), + exit: OneTimeEvent::new(), io: IrqSafeSpinlock::new(ProcessIo::new()), }); @@ -381,38 +367,40 @@ impl Process { /// Returns the [ExitCode] of the process, if it has exited pub fn get_exit_status(&self) -> Option { - match self.inner.read().state { - ProcessState::Running => None, - ProcessState::Terminated(x) => Some(x), - } + self.exit.try_read_copy() } - /// Suspends the task until the process exits - pub fn wait_for_exit(process: Arc) -> impl Future { - struct ProcessExitFuture { - process: Arc, - } - - impl Future for ProcessExitFuture { - type Output = ExitCode; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let process = &self.process; - - process.exit_waker.register(cx.waker()); - - if let Some(exit_status) = process.get_exit_status() { - process.exit_waker.remove(cx.waker()); - Poll::Ready(exit_status) - } else { - Poll::Pending - } - } - } - - ProcessExitFuture { process } + /// Returns `true` if the process has exited + pub fn has_exited(&self) -> bool { + self.exit.is_signalled() } + // /// Suspends the task until the process exits + // pub fn wait_for_exit(process: Arc) -> impl Future { + // // struct ProcessExitFuture { + // // process: Arc, + // // } + + // // impl Future for ProcessExitFuture { + // // type Output = ExitCode; + + // // fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // // let process = &self.process; + + // // process.exit_waker.register(cx.waker()); + + // // if let Some(exit_status) = process.get_exit_status() { + // // process.exit_waker.remove(cx.waker()); + // // Poll::Ready(exit_status) + // // } else { + // // Poll::Pending + // // } + // // } + // // } + + // // ProcessExitFuture { process } + // } + /// Handles exit of a single child thread pub fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { debugln!("Thread {} of process {}: {:?}", thread, self.id, code); @@ -428,14 +416,12 @@ impl Process { if last_thread { debugln!("Last thread of {} exited", self.id); - inner.state = ProcessState::Terminated(code); - inner.space.clear().unwrap(); self.io.lock().handle_exit(); drop(inner); - self.exit_waker.wake_all(); + self.exit.signal(code); } } @@ -450,8 +436,13 @@ impl Process { let processes = PROCESSES.lock(); for (_, proc) in processes.iter() { let inner = proc.inner.read(); - if !matches!(inner.state, ProcessState::Terminated(_)) && inner.group_id == group_id { - debugln!("Deliver group signal to {}: {:?}", proc.id(), signal); + if !proc.has_exited() && inner.group_id == group_id { + debugln!( + "Deliver group ({}) signal to {}: {:?}", + group_id, + proc.id(), + signal + ); drop(inner); proc.raise_signal(signal); } diff --git a/src/task/thread.rs b/src/task/thread.rs index 0cf92f74..8c9a32ff 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -5,9 +5,7 @@ use core::{ fmt, mem::size_of, ops::Deref, - pin::Pin, - sync::atomic::{AtomicBool, AtomicU64, Ordering}, - task::{Context, Poll}, + sync::atomic::{AtomicU64, Ordering}, }; use abi::{ @@ -17,11 +15,10 @@ use abi::{ use alloc::{collections::BTreeMap, string::String, sync::Arc}; use atomic_enum::atomic_enum; use crossbeam_queue::SegQueue; -use futures_util::Future; use kernel_util::{ block, - runtime::QueueWaker, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, + util::event::BoolEvent, }; use crate::{ @@ -106,8 +103,9 @@ pub struct Thread { inner: IrqSafeSpinlock, signal_queue: SegQueue, - pub(super) terminated: AtomicBool, - pub(super) exit_notify: Arc, + + pub(super) exit: BoolEvent, + /// CPU scheduling affinity mask pub affinity: ThreadAffinity, } @@ -183,8 +181,8 @@ impl Thread { inner: IrqSafeSpinlock::new(ThreadInner { signal_entry: None }), signal_queue: SegQueue::new(), - exit_notify: Arc::new(QueueWaker::new()), - terminated: AtomicBool::new(false), + exit: BoolEvent::new(), + affinity: ThreadAffinity::any_cpu(), }); @@ -257,9 +255,7 @@ impl Thread { /// Updates the thread's terminated status and wakes up any other threads waiting for it to /// exit pub fn set_terminated(&self) { - if !self.terminated.swap(true, Ordering::Release) { - self.exit_notify.wake_all(); - } + self.exit.signal_saturating(); } // Signals @@ -342,41 +338,11 @@ impl Thread { self.enqueue_to(queue); } - fn is_terminated(&self) -> bool { - self.terminated.load(Ordering::Acquire) - } - - /// Waits until thread is terminated and removed from scheduling queues - pub fn wait_for_exit(self: &Arc) -> impl Future { - struct F(Arc); - - impl Future for F { - type Output = (); - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let F(thread) = self.deref(); - - thread.exit_notify.register(cx.waker()); - - if thread.is_terminated() { - thread.exit_notify.remove(cx.waker()); - Poll::Ready(()) - } else { - Poll::Pending - } - } - } - - infoln!("wait_for_exit: pending {:?}", self.id); - F(self.clone()) - } - /// Requests thread termination and blocks until said thread finishes fully pub async fn terminate(self: &Arc) { // Will not abort the execution: called from another thread self.dequeue(ThreadState::Terminated); - - self.wait_for_exit().await + self.exit.wait().await; } /// Returns the current thread on the CPU. diff --git a/src/util/mod.rs b/src/util/mod.rs index 26f1d001..11d685cb 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,7 +1,5 @@ //! Various kernel utility functions -pub mod queue; - /// Extension trait for [Iterator]s of [Result]s pub trait ResultIterator { /// Drops entries from the iterator until the first error diff --git a/src/util/queue.rs b/src/util/queue.rs deleted file mode 100644 index 67969503..00000000 --- a/src/util/queue.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Asynchronous array queue implementation - -use core::{ - pin::Pin, - task::{Context, Poll}, -}; - -use alloc::sync::Arc; -use crossbeam_queue::ArrayQueue; -use futures_util::Future; -use kernel_util::runtime::QueueWaker; - -/// Asynchronous queue -pub struct AsyncQueue { - waker: Arc, - queue: Arc>, -} - -impl AsyncQueue { - /// Constructs a new [AsyncQueue] of requested capacity - pub fn new(capacity: usize) -> Self { - Self { - waker: Arc::new(QueueWaker::new()), - queue: Arc::new(ArrayQueue::new(capacity)), - } - } - - /// Pushes an entry to the queue and signals a single task about the available value - pub fn send(&self, value: T) -> Result<(), T> { - let result = self.queue.push(value); - if result.is_ok() { - self.waker.wake_one(); - } - result - } - - /// Asynchronously receives an entry from the queue - pub fn recv(&self) -> impl Future { - struct AsyncQueueRecvFuture { - waker: Arc, - queue: Arc>, - } - - impl Future for AsyncQueueRecvFuture { - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - if let Some(value) = self.queue.pop() { - return Poll::Ready(value); - } - - self.waker.register(cx.waker()); - - match self.queue.pop() { - Some(value) => { - self.waker.remove(cx.waker()); - Poll::Ready(value) - } - None => Poll::Pending, - } - } - } - - AsyncQueueRecvFuture { - queue: self.queue.clone(), - waker: self.waker.clone(), - } - } -} From 0670cdbb80d8eb961bcf6203b639de2be26979cd Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 30 Jan 2024 11:37:11 +0200 Subject: [PATCH 168/211] chore: fix aarch64 build --- lib/kernel-util/src/api.rs | 2 -- lib/kernel-util/src/mem/mod.rs | 4 ---- src/arch/aarch64/mem/mod.rs | 7 ++++--- src/arch/mod.rs | 8 -------- src/arch/x86_64/mod.rs | 5 ----- tools/gentables/src/x86_64.rs | 1 - 6 files changed, 4 insertions(+), 23 deletions(-) diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index b84cb34b..fae48fe4 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -27,8 +27,6 @@ extern "Rust" { pub fn __virtualize(phys: u64) -> usize; pub fn __physicalize(virt: usize) -> u64; - pub fn __translate_kernel(virt: usize) -> Option; - pub fn __map_device_pages( base: PhysicalAddress, count: usize, diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index d68e80aa..e43a63c1 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -262,7 +262,3 @@ pub fn allocate_page() -> Result { pub fn allocate_contiguous_pages(count: usize) -> Result { unsafe { api::__allocate_contiguous_pages(count) } } - -pub fn translate_kernel_address(virt: usize) -> Option { - unsafe { api::__translate_kernel(virt) } -} diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index b596b426..9c12ef90 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -2,6 +2,7 @@ use core::{ alloc::Layout, ops::{Deref, DerefMut}, + ptr::addr_of, }; use abi::error::Error; @@ -338,9 +339,9 @@ pub(super) unsafe fn load_fixed_tables() { /// Unsafe, must only be called by BSP during its early init, must already be in "higher-half" pub(super) unsafe fn init_fixed_tables() { // TODO this could be built in compile-time too? - let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; - let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; - let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + let early_mapping_l3_phys = addr_of!(EARLY_MAPPING_L3) as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = addr_of!(DEVICE_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = addr_of!(HEAP_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; for i in 0..DEVICE_MAPPING_L3_COUNT { let device_mapping_l3_phys = PhysicalAddress::from_raw( diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 707c9cfb..2332174f 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -122,9 +122,6 @@ pub trait Architecture { /// Converts a virtual address created by [Architecture::virtualize] back to its physical form fn physicalize(address: usize) -> u64; - /// Translates a kernel virtual address into its physical page - fn translate_kernel_address(address: usize) -> Option; - // Architecture intrinsics /// Suspends CPU until an interrupt is received @@ -359,8 +356,3 @@ fn __monotonic_timestamp() -> Result { fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { ARCHITECTURE.message_interrupt_controller() } - -#[no_mangle] -fn __translate_kernel(address: usize) -> Option { - ArchitectureImpl::translate_kernel_address(address) -} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 7d168791..049a741d 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -275,11 +275,6 @@ impl Architecture for X86_64 { (address - RAM_MAPPING_OFFSET) as _ } - #[inline] - fn translate_kernel_address(address: usize) -> Option { - mem::translate_kernel_address(address) - } - fn external_interrupt_controller( &'static self, ) -> &'static dyn ExternalInterruptController { diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs index c1a5f7d3..d7b12f5c 100644 --- a/tools/gentables/src/x86_64.rs +++ b/tools/gentables/src/x86_64.rs @@ -166,7 +166,6 @@ impl X8664Builder { let entry = page | (PageFlags::PRESENT | flags).bits(); - let l1i = (address >> 30) as usize & 0x1FF; let l2i = ((address >> 21) as usize & 0x1FF) - l2i_offset; let l3i = (address >> 12) as usize & 0x1FF; From 0aa5111123b39ef9946e58c98a389d66d6fdc7ba Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 31 Jan 2024 19:56:49 +0200 Subject: [PATCH 169/211] net/sys: add GetSystemInfo, QueryRoute --- driver/net/core/src/config.rs | 40 +++++++++++++++++++++++++++++++-- driver/net/core/src/l4/icmp.rs | 1 + driver/net/core/src/socket.rs | 5 +++-- lib/vfs/src/ioctx.rs | 10 +++++++++ src/mem/phys/manager.rs | 41 +++++++++++++++++++++++++++++++++- src/mem/phys/mod.rs | 7 +++++- src/mem/process.rs | 2 +- src/syscall/mod.rs | 39 +++++++++++++++++++++++++++----- src/task/context.rs | 2 +- src/task/sched.rs | 10 +++++++-- src/task/thread.rs | 18 +++++++++++++++ 11 files changed, 160 insertions(+), 15 deletions(-) diff --git a/driver/net/core/src/config.rs b/driver/net/core/src/config.rs index ac1b19d7..8865d547 100644 --- a/driver/net/core/src/config.rs +++ b/driver/net/core/src/config.rs @@ -5,12 +5,18 @@ use yggdrasil_abi::{ error::Error, io::MessageDestination, net::{ - netconfig::{InterfaceInfo, InterfaceQuery, NetConfigRequest, NetConfigResult, RouteInfo}, + netconfig::{ + InterfaceInfo, InterfaceQuery, NetConfigRequest, NetConfigResult, RouteInfo, + RoutingInfo, + }, IpAddr, SubnetAddr, }, }; -use crate::{interface::NetworkInterface, l3::Route}; +use crate::{ + interface::NetworkInterface, + l3::{arp, Route}, +}; async fn receive_request(channel: &ChannelDescriptor) -> Result<(u32, NetConfigRequest), Error> { loop { @@ -59,6 +65,7 @@ fn describe_interface(interface: &NetworkInterface) -> InterfaceInfo { fn describe_route(route: &Route) -> RouteInfo { // NOTE: must exist let interface = NetworkInterface::get(route.interface).unwrap(); + RouteInfo { interface_name: interface.name.clone(), interface_id: route.interface, @@ -67,6 +74,21 @@ fn describe_route(route: &Route) -> RouteInfo { } } +fn query_route(destination: IpAddr) -> Option { + let (interface_id, gateway, destination) = Route::lookup(destination)?; + let interface = NetworkInterface::get(interface_id).unwrap(); + let source = *interface.address.read(); + + Some(RoutingInfo { + interface_name: interface.name.clone(), + interface_id: interface_id, + destination, + gateway, + source, + source_mac: interface.mac, + }) +} + fn query_interface(query: InterfaceQuery) -> Option> { match query { InterfaceQuery::ById(id) => NetworkInterface::get(id).ok(), @@ -145,6 +167,20 @@ pub async fn network_config_service() -> Result<(), Error> { send_reply(&channel, sender_id, result)?; } NetConfigRequest::ClearNetworkAddress(_interface) => todo!(), + NetConfigRequest::QueryRoute(destination) => { + let result = match query_route(destination) { + Some(route) => NetConfigResult::Ok(route), + None => NetConfigResult::err("No route to host"), + }; + send_reply(&channel, sender_id, result)?; + } + NetConfigRequest::QueryArp(interface_id, destination, perform_query) => { + let result = match arp::lookup(interface_id, destination, perform_query).await { + Ok(mac) => NetConfigResult::Ok(mac), + Err(_) => NetConfigResult::err("No ARP entry"), + }; + send_reply(&channel, sender_id, result)?; + } } } } diff --git a/driver/net/core/src/l4/icmp.rs b/driver/net/core/src/l4/icmp.rs index 59dfe2e5..f62ed7df 100644 --- a/driver/net/core/src/l4/icmp.rs +++ b/driver/net/core/src/l4/icmp.rs @@ -62,6 +62,7 @@ async fn handle_v4(source_address: Ipv4Addr, l3_packet: L3Packet) -> Result<(), match (icmp_frame.ty, icmp_frame.code) { (8, 0) => send_v4_reply(source_address, icmp_frame, icmp_data).await, + (0, 0) => Ok(()), _ => { log::debug!( "Ignoring unknown ICMPv4 type:code: {}:{}", diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index ce466dfd..6951b674 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -442,11 +442,12 @@ impl PacketSocket for RawSocket { fn receive(&self, buffer: &mut [u8]) -> Result<(SocketAddr, usize), Error> { let data = block!(self.receive_queue.pop_front().await)?; - let len = 4096 - data.l2_offset; + let full_len = data.data.len(); + let len = full_len - data.l2_offset; if buffer.len() < len { return Err(Error::BufferTooSmall); } - buffer[..len].copy_from_slice(&data.data[data.l2_offset..]); + buffer[..len].copy_from_slice(&data.data[data.l2_offset..full_len]); Ok((SocketAddr::NULL_V4, len)) } } diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index c29a4487..65c76475 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -76,6 +76,16 @@ impl IoContext { self.uid = uid; } + /// Updates the context's permission mask + pub fn set_umask(&mut self, mask: FileMode) { + self.umask = mask; + } + + /// Returns the context's permission mask + pub fn umask(&self) -> FileMode { + self.umask + } + /// Sets the current group ID of the context. Returns [Error::PermissionDenied] if current user /// is not root. pub fn set_gid(&mut self, gid: GroupId) -> Result<(), Error> { diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 9e8106ec..47bc6e2f 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -1,10 +1,15 @@ //! Physical memory manager implementation -use abi::error::Error; +use core::sync::atomic::{AtomicUsize, Ordering}; + +use abi::{error::Error, system::SystemMemoryStats}; use kernel_util::mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, + table::EntryLevel, }; +use crate::arch::L3; + pub type BitmapWord = u64; pub(super) const BITMAP_WORD_SIZE: usize = BitmapWord::BITS as usize; @@ -14,6 +19,16 @@ const HUGE_PAGE_WORD_COUNT: usize = 512 / BITMAP_WORD_SIZE; pub(super) const TRACKED_PAGE_LIMIT: usize = (BITMAP_PAGE_COUNT * 4096) * 8; +struct MemoryStats { + available_pages: AtomicUsize, + used_pages: AtomicUsize, +} + +static STATS: MemoryStats = MemoryStats { + available_pages: AtomicUsize::new(0), + used_pages: AtomicUsize::new(0), +}; + /// Physical memory management interface pub struct PhysicalMemoryManager { bitmap: PhysicalRefMut<'static, [u64]>, @@ -66,6 +81,8 @@ impl PhysicalMemoryManager { self.last_free_bit = i + 1; self.mark_alloc(i); + STATS.used_pages.fetch_add(1, Ordering::Relaxed); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } @@ -92,6 +109,8 @@ impl PhysicalMemoryManager { } self.last_free_bit = i + 512; + STATS.used_pages.fetch_add(512, Ordering::Relaxed); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } @@ -117,6 +136,8 @@ impl PhysicalMemoryManager { } self.last_free_bit = i + count; + STATS.used_pages.fetch_add(count, Ordering::Relaxed); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } @@ -138,6 +159,8 @@ impl PhysicalMemoryManager { assert!(page >= self.offset); let index = (page - self.offset) / 0x1000; + STATS.used_pages.fetch_sub(1, Ordering::Relaxed); + assert!(self.is_alloc(index)); self.mark_free(index); } @@ -152,7 +175,23 @@ impl PhysicalMemoryManager { assert!(page >= self.offset); let index = (page - self.offset) / 0x1000; + STATS.available_pages.fetch_add(1, Ordering::Relaxed); + assert!(self.is_alloc(index)); self.mark_free(index); } + + /// Returns memory usage stats + pub fn stats() -> SystemMemoryStats { + let available = STATS.available_pages.load(Ordering::Relaxed); + let used = STATS.used_pages.load(Ordering::Relaxed); + let free = available - used; + + SystemMemoryStats { + total_usable_pages: available, + allocated_pages: used, + free_pages: free, + page_size: L3::SIZE, + } + } } diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index fe6fa775..cd61af6f 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -2,7 +2,7 @@ use core::ops::Range; -use abi::error::Error; +use abi::{error::Error, system::SystemMemoryStats}; use kernel_util::{ mem::address::{FromRaw, IntoRaw, PhysicalAddress}, sync::IrqSafeSpinlock, @@ -77,6 +77,11 @@ pub fn alloc_2m_page() -> Result { PHYSICAL_MEMORY.get().lock().alloc_2m_page() } +/// Returns physical memory stats +pub fn stats() -> SystemMemoryStats { + PhysicalMemoryManager::stats() +} + /// Deallocates a physical memory page. /// /// # Safety diff --git a/src/mem/process.rs b/src/mem/process.rs index 061bcebe..23dedd8f 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -233,7 +233,7 @@ impl Inner { if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { // TODO rollback - todo!(); + todo!("alloc_range({})", page_count); }; Ok(address) diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 760a88f0..5e41c16a 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -11,10 +11,11 @@ use abi::{ mem::MappingSource, net::{SocketOption, SocketType}, process::{ - ExecveOptions, ExitCode, MutexOperation, Signal, SpawnOption, SpawnOptions, - ThreadSpawnOptions, + ExecveOptions, ExitCode, MutexOperation, ProcessInfoElement, Signal, SpawnOption, + SpawnOptions, ThreadSpawnOptions, }, syscall::SyscallFunction, + system::SystemInfo, }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; @@ -26,7 +27,7 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::{process::VirtualRangeBacking, table::MapAttributes}, + mem::{phys, process::VirtualRangeBacking, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, @@ -630,8 +631,36 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result process.set_group_id(group_id); Ok(0) } - SyscallFunction::GetProcessInfo => todo!(), - SyscallFunction::SetProcessInfo => todo!(), + SyscallFunction::GetProcessInfo => { + let element = arg_user_mut::(args[0] as _)?; + + run_with_io(process, |mut io| match element { + ProcessInfoElement::Umask(mask) => { + *mask = io.ioctx().umask(); + Ok(0) + } + }) + } + SyscallFunction::SetProcessInfo => { + let element = arg_user_ref::(args[0] as _)?; + + run_with_io(process, |mut io| match element { + &ProcessInfoElement::Umask(mask) => { + io.ioctx().set_umask(mask); + Ok(0) + } + }) + } + SyscallFunction::GetSystemInfo => { + let element = arg_user_mut::(args[0] as _)?; + + match element { + SystemInfo::MemoryStats(stats) => { + *stats = phys::stats(); + Ok(0) + } + } + } SyscallFunction::StartSession => { let session_terminal = process.clear_session_terminal(); diff --git a/src/task/context.rs b/src/task/context.rs index a90f7c21..5182312b 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -1,7 +1,7 @@ //! Platform-specific task context manipulation interfaces use abi::{arch::SavedFrame, error::Error}; -use alloc::boxed::Box; +use alloc::{boxed::Box, sync::Arc}; use cfg_if::cfg_if; use kernel_util::thread::Termination; diff --git a/src/task/sched.rs b/src/task/sched.rs index 3d8ac7e6..d77de62d 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -108,7 +108,7 @@ impl CpuQueue { let current_id = cpu.current_thread_id(); let current = current_id.and_then(Thread::get); - if let Some(current) = current.as_ref() { + let drop_current = if let Some(current) = current.as_ref() { let mut sched = current.sched.lock(); let q = sched.queue.unwrap(); @@ -118,24 +118,30 @@ impl CpuQueue { match sched.state { ThreadState::Ready => { self.queue.push(current.id); + false } ThreadState::Running => { sched.state = ThreadState::Ready; self.queue.push(current.id); + false } ThreadState::Terminated => { sched.in_queue = false; sched.queue = None; current.set_terminated(); + true } ThreadState::Suspended => { sched.queue = None; sched.in_queue = false; + false } } - } + } else { + false + }; let (next, next_id) = self.pop(); diff --git a/src/task/thread.rs b/src/task/thread.rs index 8c9a32ff..de014f4b 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -129,6 +129,10 @@ impl GlobalThreadList { debug_assert!(!self.data.contains_key(&id)); self.data.insert(id, thread); } + + pub fn remove(&mut self, id: ThreadId) -> Option> { + self.data.remove(&id) + } } impl ThreadAffinity { @@ -252,6 +256,14 @@ impl Thread { self.process.as_ref().unwrap() } + /// Removes the thread from the thread list + pub fn remove_from_list(id: ThreadId) { + THREADS + .write() + .remove(id) + .expect("Thread was not in the thread list?"); + } + /// Updates the thread's terminated status and wakes up any other threads waiting for it to /// exit pub fn set_terminated(&self) { @@ -370,6 +382,12 @@ impl Thread { } } +impl Drop for Thread { + fn drop(&mut self) { + debugln!("DROP {:p}", self); + } +} + impl CurrentThread { /// Terminate the current thread pub fn exit(&self, code: ExitCode) -> ! { From 2780b010bfdfb72eec846c6e3cab20b926410c02 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 1 Feb 2024 17:23:51 +0200 Subject: [PATCH 170/211] proc: better process page dropping --- lib/kernel-util/src/runtime/executor.rs | 19 +++++--- lib/kernel-util/src/runtime/waker.rs | 58 +++++++++++++++++++++++-- lib/kernel-util/src/thread.rs | 16 ++++++- src/arch/aarch64/context.S | 13 ++++++ src/arch/aarch64/context.rs | 44 ++++++++++++++++--- src/arch/aarch64/mem/process.rs | 19 +++++++- src/arch/aarch64/mem/table.rs | 44 ++++++++++++++++++- src/arch/x86_64/context.S | 19 ++++++++ src/arch/x86_64/context.rs | 55 +++++++++++++++++++---- src/arch/x86_64/mem/process.rs | 19 +++++++- src/arch/x86_64/mem/table.rs | 44 ++++++++++++++++++- src/fs/sysfs.rs | 20 ++++++++- src/init.rs | 2 +- src/mem/phys/manager.rs | 2 +- src/mem/phys/mod.rs | 2 +- src/mem/process.rs | 16 ++++++- src/mem/table.rs | 16 ++++++- src/proc/exec.rs | 14 ++++-- src/syscall/mod.rs | 1 + src/task/context.rs | 7 +++ src/task/process.rs | 50 ++++++++++++++------- src/task/sched.rs | 6 ++- 22 files changed, 427 insertions(+), 59 deletions(-) diff --git a/lib/kernel-util/src/runtime/executor.rs b/lib/kernel-util/src/runtime/executor.rs index f4ff9ac2..a6aadfb0 100644 --- a/lib/kernel-util/src/runtime/executor.rs +++ b/lib/kernel-util/src/runtime/executor.rs @@ -1,4 +1,4 @@ -use core::task::{Context, Poll}; +use core::task::{Context, Poll, Waker}; use alloc::{boxed::Box, format, sync::Arc}; use futures_util::{task::waker_ref, Future}; @@ -8,7 +8,7 @@ use crate::thread::Thread; use super::{ task::{Task, Termination}, - task_queue, + task_queue, waker, }; /// Pushes a task into the executor's queue @@ -46,17 +46,22 @@ pub fn spawn + Send + 'static>( pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> Result { let mut future = Box::pin(future); - loop { - let thread = Thread::current(); + // Make a weak ref for the waker + let weak = Thread::current().downgrade(); - let waker = waker_ref(&thread); + let waker = waker::weak_thread_waker(weak.clone()); + let waker = unsafe { Waker::from_raw(waker) }; + + loop { let context = &mut Context::from_waker(&waker); match future.as_mut().poll(context) { Poll::Ready(value) => break Ok(value), Poll::Pending => { - if let Err(error) = thread.suspend() { - break Err(error); + if let Some(thread) = unsafe { Thread::upgrade(&weak) } { + if let Err(error) = thread.suspend() { + break Err(error); + } } } } diff --git a/lib/kernel-util/src/runtime/waker.rs b/lib/kernel-util/src/runtime/waker.rs index 24b3df55..8ca256b9 100644 --- a/lib/kernel-util/src/runtime/waker.rs +++ b/lib/kernel-util/src/runtime/waker.rs @@ -1,8 +1,8 @@ -use core::task::Waker; +use core::task::{RawWaker, RawWakerVTable, Waker}; -use alloc::collections::VecDeque; +use alloc::{collections::VecDeque, sync::Weak}; -use crate::sync::IrqSafeSpinlock; +use crate::{sync::IrqSafeSpinlock, thread::Thread}; /// Async/await primitive to suspend and wake up tasks waiting on some shared resource pub struct QueueWaker { @@ -70,3 +70,55 @@ impl QueueWaker { self.wake_some(usize::MAX) } } + +unsafe fn drop_weak_waker(ptr: *const ()) { + let weak = Weak::from_raw(ptr as *const Thread); + + // Explicitly drop the thing + drop(weak); +} + +unsafe fn wake_weak_waker(ptr: *const ()) { + let weak = Weak::from_raw(ptr as *const Thread); + + if let Some(strong) = weak.upgrade() { + strong.enqueue(); + } + + // Weak gets dropped +} + +unsafe fn wake_by_ref_weak_waker(ptr: *const ()) { + let weak = Weak::from_raw(ptr as *const Thread); + + if let Some(strong) = weak.upgrade() { + strong.enqueue(); + } + + // Weak doesn't get dropped + core::mem::forget(weak); +} + +unsafe fn clone_weak_waker(ptr: *const ()) -> RawWaker { + let weak = Weak::from_raw(ptr as *const Thread); + + let waker = weak_thread_waker(weak.clone()); + + // Prevent decrement of the refcount + core::mem::forget(weak); + + waker +} + +pub fn weak_thread_waker(weak: Weak) -> RawWaker { + const VTABLE: RawWakerVTable = RawWakerVTable::new( + clone_weak_waker, + wake_weak_waker, + wake_by_ref_weak_waker, + drop_weak_waker, + ); + + let raw = Weak::into_raw(weak); + + RawWaker::new(raw as *const (), &VTABLE) +} diff --git a/lib/kernel-util/src/thread.rs b/lib/kernel-util/src/thread.rs index 6ef469b5..6ddb99ce 100644 --- a/lib/kernel-util/src/thread.rs +++ b/lib/kernel-util/src/thread.rs @@ -1,6 +1,10 @@ use core::{fmt, ops::Deref}; -use alloc::{boxed::Box, string::String, sync::Arc}; +use alloc::{ + boxed::Box, + string::String, + sync::{Arc, Weak}, +}; use futures_util::task::ArcWake; use yggdrasil_abi::{error::Error, process::ExitCode}; @@ -77,6 +81,12 @@ impl Thread { pub fn enqueue(self: &Arc) { unsafe { api::__enqueue(self) } } + + pub unsafe fn upgrade(weak: &Weak) -> Option { + let guard = IrqGuard::acquire(); + let strong = weak.upgrade()?; + Some(CurrentThread(strong, guard)) + } } impl CurrentThread { @@ -87,6 +97,10 @@ impl CurrentThread { pub fn exit(&self, code: ExitCode) { unsafe { api::__exit_current(self, code) } } + + pub fn downgrade(self) -> Weak { + Arc::downgrade(&self.0) + } } impl ArcWake for Thread { diff --git a/src/arch/aarch64/context.S b/src/arch/aarch64/context.S index 83d9ef03..04bc0eaf 100644 --- a/src/arch/aarch64/context.S +++ b/src/arch/aarch64/context.S @@ -98,6 +98,19 @@ __aarch64_switch_task: ret +// x0 -- destination context +// x1 -- source (dropped) thread +__aarch64_switch_task_and_drop: + ldr x0, [x0] + mov sp, x0 + + mov x0, x1 + bl __aarch64_drop_context + + LOAD_TASK_STATE + + ret + __aarch64_enter_task: ldr x0, [x0] mov sp, x0 diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 06f50d9f..b0a8e637 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -2,8 +2,13 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; +use alloc::sync::Arc; +use kernel_util::mem::address::PhysicalAddress; -use crate::{mem::phys, task::context::TaskContextImpl}; +use crate::{ + mem::phys, + task::{context::TaskContextImpl, thread::Thread}, +}; struct StackBuilder { base: usize, @@ -20,8 +25,7 @@ struct TaskContextInner { #[allow(unused)] pub struct TaskContext { inner: UnsafeCell, - // TODO will be used when I write proper task cleanup - stack_base: usize, + stack_base_phys: PhysicalAddress, stack_size: usize, } @@ -83,7 +87,8 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 8; - let stack_base = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw(); + let stack_base_phys = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?; + let stack_base = stack_base_phys.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -99,7 +104,7 @@ impl TaskContextImpl for TaskContext { Ok(Self { inner: UnsafeCell::new(TaskContextInner { sp }), - stack_base, + stack_base_phys, stack_size: KERNEL_TASK_PAGES * 0x1000, }) } @@ -112,7 +117,8 @@ impl TaskContextImpl for TaskContext { tpidr_el0: usize, ) -> Result { const USER_TASK_PAGES: usize = 16; - let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); + let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); @@ -127,7 +133,7 @@ impl TaskContextImpl for TaskContext { Ok(Self { inner: UnsafeCell::new(TaskContextInner { sp }), - stack_base, + stack_base_phys, stack_size: USER_TASK_PAGES * 0x1000, }) } @@ -139,11 +145,35 @@ impl TaskContextImpl for TaskContext { unsafe fn switch(&self, from: &Self) { __aarch64_switch_task(self.inner.get(), from.inner.get()) } + + unsafe fn switch_and_drop(&self, thread: Arc) { + __aarch64_switch_task_and_drop(self.inner.get(), Arc::into_raw(thread) as _); + } +} + +impl Drop for TaskContext { + fn drop(&mut self) { + assert_eq!(self.stack_size % 0x1000, 0); + + for offset in (0..self.stack_size).step_by(0x1000) { + unsafe { + phys::free_page(self.stack_base_phys.add(offset)); + } + } + } +} + +#[no_mangle] +unsafe extern "C" fn __aarch64_drop_context(thread_ptr: *const Thread) { + let thread = Arc::from_raw(thread_ptr); + Thread::remove_from_list(thread.id); + Arc::decrement_strong_count(thread_ptr); } extern "C" { fn __aarch64_enter_task(to: *mut TaskContextInner) -> !; fn __aarch64_switch_task(to: *mut TaskContextInner, from: *mut TaskContextInner); + fn __aarch64_switch_task_and_drop(to: *mut TaskContextInner, thread: *const ()) -> !; fn __aarch64_task_enter_kernel(); fn __aarch64_task_enter_user(); } diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index bedc8b8e..afdbb71c 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -11,7 +11,7 @@ use kernel_util::mem::{ use crate::mem::{ phys, process::ProcessAddressSpaceManager, - table::{MapAttributes, NextPageTable}, + table::{EntryLevelDrop, MapAttributes, NextPageTable}, }; use super::table::{PageEntry, PageTable, L1, L2, L3}; @@ -67,6 +67,11 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { fn as_address_with_asid(&self) -> u64 { unsafe { u64::from(self.l1.as_physical_address()) | ((self.asid as u64) << 48) } } + + unsafe fn clear(&mut self) { + self.l1 + .drop_range(0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::())); + } } impl ProcessAddressSpaceImpl { @@ -125,6 +130,18 @@ impl ProcessAddressSpaceImpl { } } +impl Drop for ProcessAddressSpaceImpl { + fn drop(&mut self) { + // SAFETY: with safe usage of the ProcessAddressSpaceImpl, clearing and dropping + // is safe, no one refers to the memory + unsafe { + self.clear(); + let l1_phys = self.l1.as_physical_address(); + phys::free_page(l1_phys); + } + } +} + #[inline] fn tlb_flush_vaae1(mut page: usize) { page >>= 12; diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs index f61f2c32..3f44c383 100644 --- a/src/arch/aarch64/mem/table.rs +++ b/src/arch/aarch64/mem/table.rs @@ -1,6 +1,6 @@ use core::{ marker::PhantomData, - ops::{Index, IndexMut}, + ops::{Index, IndexMut, Range}, }; use abi::error::Error; @@ -13,7 +13,7 @@ use kernel_util::mem::{ use crate::mem::{ phys, - table::{MapAttributes, NextPageTable, NonTerminalEntryLevel}, + table::{EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel}, }; bitflags! { @@ -146,6 +146,46 @@ impl NextPageTable for PageTable { } } +impl EntryLevelDrop for PageTable { + const FULL_RANGE: Range = 0..512; + + // Do nothing + unsafe fn drop_range(&mut self, _range: Range) {} +} + +impl EntryLevelDrop for PageTable +where + PageTable: EntryLevelDrop, +{ + const FULL_RANGE: Range = 0..512; + + unsafe fn drop_range(&mut self, range: Range) { + for index in range { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + let mut table_ref: PhysicalRefMut> = + PhysicalRefMut::map(table); + + table_ref.drop_all(); + + // Drop the table + drop(table_ref); + + phys::free_page(table); + } else if entry.is_present() { + // Memory must've been cleared beforehand, so no non-table entries must be present + panic!( + "Expected a table containing only tables, got table[{}] = {:#x?}", + index, entry.0 + ); + } + + self[index] = PageEntry::INVALID; + } + } +} + impl PageEntry { pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self { Self( diff --git a/src/arch/x86_64/context.S b/src/arch/x86_64/context.S index f9b0a65c..27d7e389 100644 --- a/src/arch/x86_64/context.S +++ b/src/arch/x86_64/context.S @@ -149,6 +149,25 @@ __x86_64_switch_task: ret +__x86_64_switch_and_drop: + // TSS.RSP0 + mov 8(%rdi), %rax + // Kernel stack + mov 0(%rdi), %rdi + + mov %rdi, %rsp + + // Load TSS.RSP0 + mov %gs:(8), %rdi + mov %rax, 4(%rdi) + + mov %rsi, %rdi + call __x86_64_drop_thread + + LOAD_TASK_STATE + + ret + // %rdi - to struct ptr __x86_64_enter_task: // TSS.RSP0 diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 13dc52c8..31a3b15e 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -2,9 +2,14 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; -use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw}; +use alloc::sync::Arc; +use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; -use crate::{arch::x86_64::mem::KERNEL_TABLES, mem::phys, task::context::TaskContextImpl}; +use crate::{ + arch::x86_64::mem::KERNEL_TABLES, + mem::phys, + task::{context::TaskContextImpl, thread::Thread}, +}; use super::{registers::FpuContext, syscall::SyscallFrame}; @@ -26,7 +31,7 @@ struct Inner { pub struct TaskContext { inner: UnsafeCell, fpu_context: UnsafeCell, - stack_base: usize, + stack_base_phys: PhysicalAddress, stack_size: usize, } @@ -85,7 +90,8 @@ impl TaskContext { pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result { const USER_TASK_PAGES: usize = 8; - let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); + let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); @@ -122,7 +128,7 @@ impl TaskContext { Ok(Self { inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), fpu_context: UnsafeCell::new(FpuContext::new()), - stack_base, + stack_base_phys, stack_size: USER_TASK_PAGES * 0x1000, }) } @@ -135,7 +141,8 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 32; - let stack_base = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw(); + let stack_base_phys = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?; + let stack_base = stack_base_phys.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -156,7 +163,7 @@ impl TaskContextImpl for TaskContext { Ok(Self { inner: UnsafeCell::new(Inner { sp, tss_rsp0: 0 }), fpu_context: UnsafeCell::new(FpuContext::new()), - stack_base, + stack_base_phys, stack_size: KERNEL_TASK_PAGES * 0x1000, }) } @@ -170,7 +177,8 @@ impl TaskContextImpl for TaskContext { ) -> Result { const USER_TASK_PAGES: usize = 8; - let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); + let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.virtualize_raw(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); @@ -186,7 +194,7 @@ impl TaskContextImpl for TaskContext { Ok(Self { inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), fpu_context: UnsafeCell::new(FpuContext::new()), - stack_base, + stack_base_phys, stack_size: USER_TASK_PAGES * 0x1000, }) } @@ -210,6 +218,34 @@ impl TaskContextImpl for TaskContext { __x86_64_switch_task(dst, src); } } + + unsafe fn switch_and_drop(&self, thread: Arc) { + let dst = self.inner.get(); + + FpuContext::restore(self.fpu_context.get()); + + __x86_64_switch_and_drop(dst, Arc::into_raw(thread) as _) + } +} + +impl Drop for TaskContext { + fn drop(&mut self) { + assert_eq!(self.stack_size % 0x1000, 0); + + for offset in (0..self.stack_size).step_by(0x1000) { + unsafe { + phys::free_page(self.stack_base_phys.add(offset)); + } + } + } +} + +// TODO merge with aarch64 +#[no_mangle] +unsafe extern "C" fn __x86_64_drop_thread(thread_ptr: *const Thread) { + let thread = Arc::from_raw(thread_ptr); + Thread::remove_from_list(thread.id); + Arc::decrement_strong_count(thread_ptr); } extern "C" { @@ -218,6 +254,7 @@ extern "C" { fn __x86_64_task_enter_from_fork(); fn __x86_64_enter_task(to: *mut Inner) -> !; fn __x86_64_switch_task(to: *mut Inner, from: *mut Inner); + fn __x86_64_switch_and_drop(to: *mut Inner, from: *const ()); } global_asm!( diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index 9352c4b1..c3a09527 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -11,7 +11,7 @@ use crate::{ mem::{ phys, process::ProcessAddressSpaceManager, - table::{EntryLevel, MapAttributes, NextPageTable}, + table::{EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable}, }, }; @@ -70,6 +70,11 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { // TODO x86-64 PCID/ASID? unsafe { self.l0.as_physical_address().into_raw() } } + + unsafe fn clear(&mut self) { + self.l0 + .drop_range(0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::())); + } } impl ProcessAddressSpaceImpl { @@ -137,3 +142,15 @@ impl ProcessAddressSpaceImpl { Some((page, l3[l3i].attributes().into())) } } + +impl Drop for ProcessAddressSpaceImpl { + fn drop(&mut self) { + // SAFETY: with safe usage of the ProcessAddressSpaceImpl, clearing and dropping + // is safe, no one refers to the memory + unsafe { + self.clear(); + let l0_phys = self.l0.as_physical_address(); + phys::free_page(l0_phys); + } + } +} diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index f6db3d8d..8032f066 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -1,7 +1,7 @@ //! x86-64-specific memory translation table management interfaces and functions use core::{ marker::PhantomData, - ops::{Index, IndexMut}, + ops::{Index, IndexMut, Range}, }; use abi::error::Error; @@ -13,7 +13,7 @@ use kernel_util::mem::{ use crate::mem::{ phys, - table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, + table::{EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel}, }; bitflags! { @@ -249,6 +249,46 @@ impl NextPageTable for PageTable { } } +impl EntryLevelDrop for PageTable { + const FULL_RANGE: Range = 0..512; + + // Do nothing + unsafe fn drop_range(&mut self, _range: Range) {} +} + +impl EntryLevelDrop for PageTable +where + PageTable: EntryLevelDrop, +{ + const FULL_RANGE: Range = 0..512; + + unsafe fn drop_range(&mut self, range: Range) { + for index in range { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + let mut table_ref: PhysicalRefMut> = + PhysicalRefMut::map(table); + + table_ref.drop_all(); + + // Drop the table + drop(table_ref); + + phys::free_page(table); + } else if entry.is_present() { + // Memory must've been cleared beforehand, so no non-table entries must be present + panic!( + "Expected a table containing only tables, got table[{}] = {:#x?}", + index, entry.0 + ); + } + + self[index] = PageEntry::INVALID; + } + } +} + impl Index for PageTable { type Output = PageEntry; diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 367d73c8..4dd02c3a 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -4,11 +4,11 @@ use abi::error::Error; use git_version::git_version; use kernel_util::util::OneTimeInit; use vfs::{ - impls::{const_value_node, mdir, read_fn_node}, + impls::{const_value_node, mdir, read_fn_node, ReadOnlyFnValueNode}, NodeRef, }; -use crate::{debug, util}; +use crate::{debug, mem::phys, util}; static ROOT: OneTimeInit = OneTimeInit::new(); @@ -28,8 +28,24 @@ pub fn init() { ("rev", const_value_node(git_version!())), ("log", read_fn_node(read_kernel_log)), ]); + let d_mem_phys = mdir([ + ( + "total_pages", + ReadOnlyFnValueNode::new(|| Ok(phys::stats().total_usable_pages)), + ), + ( + "free_pages", + ReadOnlyFnValueNode::new(|| Ok(phys::stats().free_pages)), + ), + ( + "allocated_pages", + ReadOnlyFnValueNode::new(|| Ok(phys::stats().allocated_pages)), + ), + ]); + let d_mem = mdir([("phys", d_mem_phys)]); let root = mdir([ ("kernel", d_kernel), + ("mem", d_mem), ("arch", const_value_node(util::arch_str())), ]); diff --git a/src/init.rs b/src/init.rs index 7eaf79eb..50c78db2 100644 --- a/src/init.rs +++ b/src/init.rs @@ -51,7 +51,7 @@ pub fn kinit() -> Result<(), Error> { { let (user_init, user_init_main) = - proc::exec::load(&mut ioctx, "/init", &["/init", "xxx"], &[])?; + proc::exec::load(&mut ioctx, None, "/init", &["/init", "xxx"], &[])?; let mut io = user_init.io.lock(); io.set_ioctx(ioctx); diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 47bc6e2f..76d1d231 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -13,7 +13,7 @@ use crate::arch::L3; pub type BitmapWord = u64; pub(super) const BITMAP_WORD_SIZE: usize = BitmapWord::BITS as usize; -pub(super) const BITMAP_PAGE_COUNT: usize = 256; +pub(super) const BITMAP_PAGE_COUNT: usize = 512; const HUGE_PAGE_WORD_COUNT: usize = 512 / BITMAP_WORD_SIZE; diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index cd61af6f..07046f77 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -180,7 +180,7 @@ pub unsafe fn init_from_iter + Clone>( let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, phys_start.into_raw(), total_count); let mut collected = 0; - const MAX_MEMORY: usize = 16 * 1024; + const MAX_MEMORY: usize = 64 * 1024; for (start, end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { for page in (start..end).step_by(0x1000) { diff --git a/src/mem/process.rs b/src/mem/process.rs index 23dedd8f..e238d1e7 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -65,6 +65,9 @@ pub trait ProcessAddressSpaceManager: Sized { /// Returns the implementation specific physical address of this space, with ASID applied fn as_address_with_asid(&self) -> u64; + + /// Clears the address space by dropping and non-global tables + unsafe fn clear(&mut self); } /// Describes how the physical memory is provided for the mapping @@ -272,7 +275,12 @@ impl Inner { } Ok(()) - }) + })?; + + // Drop the tables + self.table.clear(); + + Ok(()) } fn clone_range( @@ -448,3 +456,9 @@ impl ProcessAddressSpace { unsafe { inner.clear() } } } + +impl Drop for ProcessAddressSpace { + fn drop(&mut self) { + self.clear().ok(); + } +} diff --git a/src/mem/table.rs b/src/mem/table.rs index 1555bdd3..89ff3312 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -1,5 +1,5 @@ //! Virtual memory table interface -use core::ops::{Deref, DerefMut}; +use core::ops::{Deref, DerefMut, Range}; use abi::error::Error; use bitflags::bitflags; @@ -19,6 +19,20 @@ bitflags! { } } +/// Interface for destroying memory translation tables +pub trait EntryLevelDrop { + /// Range covering the whole table + const FULL_RANGE: Range; + + /// Recursively destroys the specified range within the table + unsafe fn drop_range(&mut self, range: Range); + + /// Recursively destroys all the entries in within the table + unsafe fn drop_all(&mut self) { + self.drop_range(Self::FULL_RANGE) + } +} + /// Interface for non-terminal tables to retrieve the next level of address translation tables pub trait NextPageTable { /// Type for the next-level page table diff --git a/src/proc/exec.rs b/src/proc/exec.rs index fadb1f72..3d017919 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -8,7 +8,12 @@ use abi::{ path::Path, process::ProgramArgumentInner, }; -use alloc::{borrow::ToOwned, string::String, sync::Arc, vec::Vec}; +use alloc::{ + borrow::ToOwned, + string::String, + sync::{Arc, Weak}, + vec::Vec, +}; use kernel_util::mem::pointer::PhysicalRefMut; use vfs::{FileRef, IoContext, Read, Seek}; @@ -173,13 +178,15 @@ fn setup_context( fn setup_binary>( name: S, + parent: Option>, space: ProcessAddressSpace, image: ProcessImage, args: &Vec, envs: &Vec, ) -> Result<(Arc, Arc), Error> { let context = setup_context(&space, &image, args, envs)?; - let (process, main) = Process::new_with_main(name, Arc::new(space), context, Some(image)); + let (process, main) = + Process::new_with_main(name, parent, Arc::new(space), context, Some(image)); infoln!("exec::setup_binary -> {:?}", process.id()); Ok((process, main)) } @@ -236,6 +243,7 @@ fn xxx_load_program>( /// Loads a program from given `path` pub fn load>( ioctx: &mut IoContext, + parent: Option>, path: P, args: &[&str], envs: &[&str], @@ -246,7 +254,7 @@ pub fn load>( let space = ProcessAddressSpace::new()?; let (image, args, envs) = xxx_load_program(&space, ioctx, path, args, envs)?; - setup_binary(path.display(), space, image, &args, &envs) + setup_binary(path.display(), parent, space, image, &args, &envs) } /// XXX diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 5e41c16a..c8f2e1ef 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -517,6 +517,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // Setup a new process from the file let (child_process, child_main) = proc::exec::load( io.ioctx(), + Some(Arc::downgrade(process)), options.program, options.arguments, options.environment, diff --git a/src/task/context.rs b/src/task/context.rs index 5182312b..b7ad8906 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -93,6 +93,13 @@ pub trait TaskContextImpl: Sized { /// Only meant to be called from the scheduler code. unsafe fn switch(&self, from: &Self); + /// Performs a context switch and drops the source thread. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code after the `thread` has terminated. + unsafe fn switch_and_drop(&self, thread: Arc); + /// Constructs a safe wrapper process to execute a kernel-space closure fn kernel_closure T + Send + 'static>( f: F, diff --git a/src/task/process.rs b/src/task/process.rs index b38f7fc1..ba7ff411 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -10,9 +10,17 @@ use abi::{ error::{Error, SyscallResult}, process::{ExitCode, Signal, ThreadSpawnOptions}, }; -use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; +use alloc::{ + collections::BTreeMap, + string::String, + sync::{Arc, Weak}, + vec::Vec, +}; use kernel_util::{ - sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + sync::{ + spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, + IrqSafeSpinlock, + }, util::event::OneTimeEvent, }; use vfs::{IoContext, NodeRef}; @@ -134,7 +142,7 @@ struct ProcessInner { threads: Vec>, mutexes: BTreeMap>, - space: Arc, + space: Option>, image: Option, } @@ -142,6 +150,7 @@ struct ProcessInner { pub struct Process { name: String, id: ProcessId, + parent: Option>, inner: IrqSafeRwLock, @@ -158,6 +167,7 @@ impl Process { /// Creates a new process with given main thread pub fn new_with_main>( name: S, + parent: Option>, space: Arc, context: TaskContext, image: Option, @@ -168,6 +178,7 @@ impl Process { let process = Arc::new(Self { name, id, + parent, inner: IrqSafeRwLock::new(ProcessInner { session_id: id, @@ -177,7 +188,7 @@ impl Process { mutexes: BTreeMap::new(), image, - space: space.clone(), + space: Some(space.clone()), }), exit: OneTimeEvent::new(), @@ -202,13 +213,14 @@ impl Process { ); let mut inner = self.inner.write(); + let space = inner.space.clone().unwrap(); + let tls_address = if let Some(image) = inner.image.as_ref() { - proc::elf::clone_tls(&inner.space, image)? + proc::elf::clone_tls(&space, image)? } else { 0 }; - let space = inner.space.clone(); let context = TaskContext::user( options.entry as _, options.argument as _, @@ -226,13 +238,14 @@ impl Process { Ok(id) } - unsafe fn fork_inner(&self, frame: &F) -> Result { + unsafe fn fork_inner(self: &Arc, frame: &F) -> Result { let src_inner = self.inner.read(); - let new_space = src_inner.space.fork()?; + let new_space = src_inner.space.as_ref().unwrap().fork()?; let new_context = frame.fork(new_space.as_address_with_asid())?; let (new_process, new_main) = Self::new_with_main( self.name(), + Some(Arc::downgrade(self)), Arc::new(new_space), new_context, src_inner.image.clone(), @@ -301,7 +314,7 @@ impl Process { /// Returns the address space of the process pub fn space(&self) -> Arc { - self.inner.read().space.clone() + self.inner.read().space.clone().unwrap() } /// Returns the [ProcessId] of this process @@ -401,6 +414,12 @@ impl Process { // // ProcessExitFuture { process } // } + /// Cleans up process resources + fn cleanup(&self, mut inner: IrqSafeRwLockWriteGuard) { + self.io.lock().handle_exit(); + inner.space = None; + } + /// Handles exit of a single child thread pub fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { debugln!("Thread {} of process {}: {:?}", thread, self.id, code); @@ -415,12 +434,7 @@ impl Process { if last_thread { debugln!("Last thread of {} exited", self.id); - - inner.space.clear().unwrap(); - self.io.lock().handle_exit(); - - drop(inner); - + self.cleanup(inner); self.exit.signal(code); } } @@ -484,6 +498,12 @@ impl Process { } } +impl Drop for ProcessInner { + fn drop(&mut self) { + debugln!("ProcessInner dropped {:p}", self); + } +} + impl fmt::Display for ProcessId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "", self.0) diff --git a/src/task/sched.rs b/src/task/sched.rs index d77de62d..fbe17b92 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -164,7 +164,11 @@ impl CpuQueue { if !core::ptr::eq(current_ctx, next_ctx) { // Perform the switch - (*next_ctx).switch(&*current_ctx); + if drop_current { + (*next_ctx).switch_and_drop(current.unwrap()); + } else { + (*next_ctx).switch(&*current_ctx); + } true } else { From 5c5e70ade119bdeea4cd21b3d5926573b5c6690e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 2 Feb 2024 10:43:14 +0200 Subject: [PATCH 171/211] vfs: improve code quality in poll/shm --- lib/vfs/src/poll.rs | 43 ++++++++++-------------------------- lib/vfs/src/shared_memory.rs | 13 +++++++---- src/syscall/mod.rs | 2 +- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index 8668509f..ec61e220 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -1,12 +1,12 @@ use core::{ - pin::Pin, + future::poll_fn, task::{Context, Poll}, time::Duration, }; use alloc::collections::BTreeMap; -use futures_util::{future::BoxFuture, Future, FutureExt}; +use futures_util::{future::BoxFuture, FutureExt}; use kernel_util::{ runtime, sync::{mutex::Mutex, LockMethod}, @@ -21,11 +21,6 @@ pub struct FdPoll { fds: Mutex>, } -struct FdPollFuture<'a> { - fds: &'a Mutex>, - timeout: Option>, -} - impl FdPoll { /// Creates a new poll channel pub fn new() -> Self { @@ -39,26 +34,18 @@ impl FdPoll { self.fds.lock().unwrap().insert(fd, file); } - fn begin(&self, timeout: Option) -> FdPollFuture<'_> { - let timeout = timeout.map(|t| runtime::sleep(t).boxed()); - - FdPollFuture { - fds: &self.fds, - timeout, - } - } - /// Polls the channel once, returning either a file descriptor or timeout - pub async fn poll_once(&self, timeout: Option) -> Option<(RawFd, Result<(), Error>)> { - self.begin(timeout).await + pub async fn wait(&self, timeout: Option) -> Option<(RawFd, Result<(), Error>)> { + let mut timeout = timeout.map(|t| runtime::sleep(t).boxed()); + poll_fn(|cx| self.poll_once(cx, &mut timeout)).await } -} -impl<'a> Future for FdPollFuture<'a> { - type Output = Option<(RawFd, Result<(), Error>)>; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - if let Some(timeout) = self.timeout.as_mut() + fn poll_once( + &self, + cx: &mut Context<'_>, + timeout: &mut Option>, + ) -> Poll)>> { + if let Some(timeout) = timeout.as_mut() && timeout.as_mut().poll(cx).is_ready() { // Timeout @@ -78,12 +65,6 @@ impl<'a> Future for FdPollFuture<'a> { impl FileReadiness for FdPoll { fn poll_read(&self, cx: &mut Context<'_>) -> Poll> { - for (_, file) in self.fds.lock().unwrap().iter() { - if file.poll_read(cx).is_ready() { - return Poll::Ready(Ok(())); - } - } - - Poll::Pending + self.poll_once(cx, &mut None).map(|_| Ok(())) } } diff --git a/lib/vfs/src/shared_memory.rs b/lib/vfs/src/shared_memory.rs index 5ab1d387..86e0f4f5 100644 --- a/lib/vfs/src/shared_memory.rs +++ b/lib/vfs/src/shared_memory.rs @@ -1,10 +1,15 @@ +use core::mem::MaybeUninit; + use alloc::vec::Vec; -use kernel_util::mem::{address::PhysicalAddress, allocate_page, PageProvider}; +use kernel_util::mem::{ + address::{AsPhysicalAddress, PhysicalAddress}, + PageBox, PageProvider, +}; use yggdrasil_abi::error::Error; /// Shared memory VFS object pub struct SharedMemory { - pages: Vec, + pages: Vec>>, } impl SharedMemory { @@ -14,7 +19,7 @@ impl SharedMemory { let page_count = size / 0x1000; let pages = (0..page_count) - .map(|_| allocate_page()) + .map(|_| PageBox::new_uninit()) .collect::>()?; Ok(Self { pages }) @@ -27,7 +32,7 @@ impl PageProvider for SharedMemory { let index = (offset / 0x1000) as usize; self.pages .get(index) - .copied() + .map(|bx| unsafe { bx.as_physical_address() }) .ok_or(Error::InvalidMemoryOperation) } diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index c8f2e1ef..3a7fdd3f 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -378,7 +378,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let poll = poll_file.as_poll_channel()?; *output = block! { - poll.poll_once(*timeout).await + poll.wait(*timeout).await }?; Ok(0) From 26479eea848e285932b66687694ab48abf5ef0e9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 2 Feb 2024 16:10:17 +0200 Subject: [PATCH 172/211] block/nvme: fix lba formats off-by-1 --- driver/block/nvme/src/command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/block/nvme/src/command.rs b/driver/block/nvme/src/command.rs index d3eb069e..5b48e5b4 100644 --- a/driver/block/nvme/src/command.rs +++ b/driver/block/nvme/src/command.rs @@ -175,7 +175,7 @@ impl IdentifyNamespaceResponse { } pub fn lba_fmt(&self, idx: usize) -> Option { - if idx >= self.NLBAF.get() as usize { + if idx > self.NLBAF.get() as usize { return None; } Some(LbaFormat(self.LBAFS[idx].get())) From f03b390933345a5ae1d363dabcd21fb586e79eab Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 3 Feb 2024 20:44:04 +0200 Subject: [PATCH 173/211] bus/pci: basic PCI support for AArch64 --- Cargo.toml | 4 +- driver/block/ahci/src/lib.rs | 40 ++-- driver/block/nvme/src/lib.rs | 56 ++--- driver/bus/pci/Cargo.toml | 1 + driver/bus/pci/src/capability.rs | 60 +++-- driver/bus/pci/src/device.rs | 222 ++++++++++++++++++ driver/bus/pci/src/lib.rs | 298 ++++++++++++++++++++---- driver/bus/pci/src/space/ecam.rs | 4 +- driver/bus/pci/src/space/mod.rs | 83 ++++++- driver/virtio/core/src/error.rs | 1 + driver/virtio/core/src/queue.rs | 12 +- driver/virtio/core/src/transport/mod.rs | 3 + driver/virtio/core/src/transport/pci.rs | 57 +++-- driver/virtio/net/src/lib.rs | 107 +++++---- lib/device-api/src/interrupt.rs | 10 +- lib/kernel-util/src/api.rs | 13 +- lib/kernel-util/src/lib.rs | 11 +- lib/kernel-util/src/mem/device.rs | 41 +++- lib/kernel-util/src/util/mod.rs | 29 +++ src/arch/aarch64/cpu.rs | 5 +- src/arch/aarch64/gic/gicd.rs | 30 ++- src/arch/aarch64/gic/mod.rs | 54 ++++- src/arch/aarch64/mem/mod.rs | 30 ++- src/arch/aarch64/mem/process.rs | 13 +- src/arch/aarch64/mod.rs | 36 ++- src/arch/aarch64/timer.rs | 2 +- src/arch/mod.rs | 39 +++- src/arch/x86_64/acpi.rs | 2 +- src/arch/x86_64/apic/ioapic.rs | 7 +- src/arch/x86_64/apic/local.rs | 12 +- src/arch/x86_64/mem/mod.rs | 36 ++- src/arch/x86_64/mod.rs | 5 +- src/arch/x86_64/peripherals/i8253.rs | 2 +- src/arch/x86_64/peripherals/ps2/mod.rs | 2 +- src/device/bus/dt_pci.rs | 189 +++++++++++++++ src/device/bus/mod.rs | 2 + src/device/devtree.rs | 20 ++ src/device/display/linear_fb.rs | 16 +- src/device/mod.rs | 2 - src/device/serial/pl011.rs | 4 +- src/mem/mod.rs | 4 +- src/mem/process.rs | 69 ++++-- 42 files changed, 1328 insertions(+), 305 deletions(-) create mode 100644 driver/bus/pci/src/device.rs create mode 100644 src/device/bus/dt_pci.rs diff --git a/Cargo.toml b/Cargo.toml index 3136d880..5433a7a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,9 +19,11 @@ vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } # Drivers +ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_block = { path = "driver/block/core" } ygg_driver_net_core = { path = "driver/net/core" } ygg_driver_net_loopback = { path = "driver/net/loopback" } +ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] } kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } @@ -55,10 +57,8 @@ acpi_lib = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branc acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } # TODO currently only supported here xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } -ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_nvme = { path = "driver/block/nvme" } ygg_driver_ahci = { path = "driver/block/ahci" } -ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] } [features] default = ["fb_console"] diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index aed1d4b2..53e9cc2a 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -6,27 +6,19 @@ extern crate alloc; use alloc::{boxed::Box, format, vec, vec::Vec}; use device_api::{ - interrupt::{InterruptAffinity, MsiHandler}, + interrupt::{InterruptAffinity, InterruptHandler}, Device, }; use error::AhciError; use kernel_fs::devfs; -use kernel_util::{ - mem::{ - address::{FromRaw, PhysicalAddress}, - device::DeviceMemoryIo, - }, - message_interrupt_controller, runtime, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use kernel_util::{mem::device::DeviceMemoryIo, runtime, sync::IrqSafeSpinlock, util::OneTimeInit}; use port::AhciPort; use regs::{PortRegs, Regs}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use ygg_driver_block::{probe_partitions, NgBlockDeviceWrapper}; use ygg_driver_pci::{ - capability::MsiCapability, PciBaseAddress, PciCommandRegister, PciConfigurationSpace, - PciDeviceInfo, + device::{PciDeviceInfo, PreferredInterruptMode}, + PciCommandRegister, PciConfigurationSpace, }; use yggdrasil_abi::error::Error; @@ -162,8 +154,8 @@ impl AhciController { } } -impl MsiHandler for AhciController { - fn handle_msi(&self, _vector: usize) -> bool { +impl InterruptHandler for AhciController { + fn handle_irq(&self, _vector: Option) -> bool { let regs = self.regs.lock(); let is = regs.IS.get(); @@ -198,18 +190,18 @@ static SATA_DRIVES: IrqSafeSpinlock> = IrqSafeSpinlock::n pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?; - let PciBaseAddress::Memory(bar5) = bar5 else { - return Err(Error::InvalidOperation); - }; + let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?; - // TODO support regular PCI interrupts (ACPI dependency) - let Some(mut msi) = info.config_space.capability::() else { - log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)"); - return Err(Error::InvalidOperation); - }; + info.init_interrupts(PreferredInterruptMode::Msi)?; + + // // TODO support regular PCI interrupts (ACPI dependency) + // let Some(mut msi) = info.config_space.capability::() else { + // log::warn!("Ignoring AHCI: does not support MSI (and the OS doesn't yet support PCI IRQ)"); + // return Err(Error::InvalidOperation); + // }; // Map the registers - let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar5)) }?; + let regs = unsafe { DeviceMemoryIo::::map(bar5, Default::default()) }?; let version = Version::try_from(regs.VS.get())?; let ahci_only = regs.CAP.matches_all(CAP::SAM::SET); let max_port_count = regs.CAP.read(CAP::NP) as usize; @@ -232,7 +224,7 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { })); // TODO use multiple vectors if capable - msi.register(message_interrupt_controller(), InterruptAffinity::Any, ahci)?; + info.map_interrupt(InterruptAffinity::Any, ahci)?; Ok(ahci) } diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index 9a0c6cb0..7c68d104 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -13,17 +13,17 @@ use core::{ use alloc::{boxed::Box, collections::BTreeMap, vec::Vec}; use command::{IdentifyActiveNamespaceIdListRequest, IdentifyControllerRequest}; use device_api::{ - interrupt::{InterruptAffinity, MsiHandler}, + interrupt::{InterruptAffinity, InterruptHandler}, Device, }; use drive::NvmeDrive; use kernel_util::{ cpu_count, cpu_index, mem::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, + address::{IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, }, - message_interrupt_controller, runtime, + runtime, sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; @@ -33,8 +33,8 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; use ygg_driver_pci::{ - capability::{MsiXCapability, MsiXVectorTable}, - PciBaseAddress, PciCommandRegister, PciConfigurationSpace, PciDeviceInfo, + device::{PciDeviceInfo, PreferredInterruptMode}, + PciCommandRegister, PciConfigurationSpace, }; use yggdrasil_abi::error::Error; @@ -128,7 +128,7 @@ pub struct NvmeController { drive_table: IrqSafeSpinlock>, controller_id: OneTimeInit, - vector_table: IrqSafeSpinlock>, + pci: PciDeviceInfo, doorbell_shift: usize, } @@ -208,18 +208,9 @@ impl NvmeController { self.io_queue_count.store(io_queue_count, Ordering::Release); { - // Register io_queue_count + 1 vectors - // TODO register vectors on different CPUs - let mut vt = self.vector_table.lock(); - - let range = vt - .register_range( - 0, - io_queue_count + 1, - message_interrupt_controller(), - InterruptAffinity::Any, - self, - ) + let range = self + .pci + .map_interrupt_multiple(0..io_queue_count + 1, InterruptAffinity::Any, self) .unwrap(); // TODO handle different MSI range allocations @@ -322,8 +313,10 @@ impl NvmeController { } } -impl MsiHandler for NvmeController { - fn handle_msi(&self, vector: usize) -> bool { +impl InterruptHandler for NvmeController { + fn handle_irq(&self, vector: Option) -> bool { + let vector = vector.expect("Only MSI-X interrupts are supported"); + if vector == 0 { self.admin_q.get().process_completions() != 0 } else if vector <= self.io_queue_count.load(Ordering::Acquire) @@ -421,26 +414,21 @@ static NVME_CONTROLLERS: IrqSafeSpinlock> = IrqSafeSpinlock::new(Vec::new()); pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { - let PciBaseAddress::Memory(bar0) = info.config_space.bar(0).unwrap() else { - panic!(); - }; + let bar0 = info + .config_space + .bar(0) + .unwrap() + .as_memory() + .expect("Expected a memory BAR0"); - // TODO also support MSI - let mut msix = info.config_space.capability::().unwrap(); - let mut vt = msix.vector_table()?; - - // TODO is this really needed? PCI spec says this is masked on reset, though I'm not sure if - // firmware puts it back in masked state after loading the kernel - vt.mask_all(); - msix.set_function_mask(false); - msix.set_enabled(true); + info.init_interrupts(PreferredInterruptMode::Msi)?; let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; info.config_space.set_command(cmd.bits()); - let regs = unsafe { DeviceMemoryIo::::map(PhysicalAddress::from_raw(bar0)) }?; + let regs = unsafe { DeviceMemoryIo::::map(bar0, Default::default()) }?; // Disable the controller regs.CC.modify(CC::ENABLE::CLEAR); @@ -454,7 +442,7 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { drive_table: IrqSafeSpinlock::new(BTreeMap::new()), controller_id: OneTimeInit::new(), - vector_table: IrqSafeSpinlock::new(vt), + pci: info.clone(), io_queue_count: AtomicUsize::new(1), doorbell_shift, diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index d411e98f..d45c7865 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -15,4 +15,5 @@ log = "0.4.20" bitflags = "2.3.3" tock-registers = "0.8.1" +[target.'cfg(target_arch = "x86_64")'.dependencies] acpi = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" } diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index 6e99c2e9..b8c3b92a 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -1,11 +1,10 @@ //! PCI capability structures and queries use alloc::{vec, vec::Vec}; -use device_api::interrupt::{InterruptAffinity, MessageInterruptController, MsiHandler, MsiInfo}; -use kernel_util::mem::{ - address::{FromRaw, PhysicalAddress}, - device::DeviceMemoryIoMut, +use device_api::interrupt::{ + InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo, }; +use kernel_util::mem::{address::PhysicalAddress, device::DeviceMemoryIoMut}; use tock_registers::{ interfaces::{Readable, Writeable}, registers::{ReadWrite, WriteOnly}, @@ -13,7 +12,6 @@ use tock_registers::{ use yggdrasil_abi::error::Error; use super::{PciCapability, PciCapabilityId, PciConfigurationSpace}; -use crate::PciBaseAddress; pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized { fn from_space_offset(space: &'s S, offset: usize) -> Self; @@ -56,6 +54,8 @@ pub struct VirtioDeviceConfigCapability; pub struct VirtioCommonConfigCapability; /// VirtIO notify configuration pub struct VirtioNotifyConfigCapability; +/// VirtIO interrupt status +pub struct VirtioInterruptStatusCapability; /// Represents an entry in MSI-X vector table #[repr(C)] @@ -99,6 +99,11 @@ pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> { offset: usize, } +pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> { + space: &'s S, + offset: usize, +} + impl PciCapability for T { const ID: PciCapabilityId = PciCapabilityId::VendorSpecific; type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>; @@ -213,6 +218,34 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S> } } +impl VirtioCapability for VirtioInterruptStatusCapability { + const CFG_TYPE: u8 = 0x03; + const MIN_LEN: usize = 1; + type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>; +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> { + pub fn read_status(&self) -> (bool, bool) { + todo!() + } +} + +impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S> + for VirtioInterruptStatusData<'s, S> +{ + fn from_space_offset(space: &'s S, offset: usize) -> Self { + Self { space, offset } + } + + fn space(&self) -> &'s S { + self.space + } + + fn offset(&self) -> usize { + self.offset + } +} + impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { // TODO use pending bits as well /// Maps and returns the vector table associated with the device's MSI-X capability @@ -227,18 +260,13 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { let Some(base) = self.space.bar(bir) else { return Err(Error::DoesNotExist); }; - let PciBaseAddress::Memory(base) = base else { + let Some(base) = base.as_memory() else { return Err(Error::InvalidOperation); }; - log::debug!("MSI-X table address: {:#x}", base + table_offset); + log::debug!("MSI-X table address: {:#x}", base.add(table_offset)); - unsafe { - MsiXVectorTable::from_raw_parts( - PhysicalAddress::from_raw(base + table_offset), - table_size, - ) - } + unsafe { MsiXVectorTable::from_raw_parts(base.add(table_offset), table_size) } } /// Changes the global enable status for the device's MSI-X capability. If set, regular IRQs @@ -266,7 +294,7 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> { impl MsiXVectorTable<'_> { unsafe fn from_raw_parts(base: PhysicalAddress, len: usize) -> Result { - let vectors = DeviceMemoryIoMut::map_slice(base, len)?; + let vectors = DeviceMemoryIoMut::map_slice(base, len, Default::default())?; Ok(Self { vectors }) } @@ -282,7 +310,7 @@ impl MsiXVectorTable<'_> { end: usize, ic: &C, affinity: InterruptAffinity, - handler: &'static dyn MsiHandler, + handler: &'static dyn InterruptHandler, ) -> Result, Error> { assert!(end > start); let mut range = vec![ @@ -321,7 +349,7 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> { &mut self, ic: &C, affinity: InterruptAffinity, - handler: &'static dyn MsiHandler, + handler: &'static dyn InterruptHandler, ) -> Result { let info = ic.register_msi(affinity, handler)?; diff --git a/driver/bus/pci/src/device.rs b/driver/bus/pci/src/device.rs new file mode 100644 index 00000000..98548174 --- /dev/null +++ b/driver/bus/pci/src/device.rs @@ -0,0 +1,222 @@ +use core::ops::Range; + +use alloc::{sync::Arc, vec::Vec}; +use device_api::{ + interrupt::{InterruptAffinity, InterruptHandler, IrqOptions, MsiInfo}, + Device, +}; +use kernel_util::{ + message_interrupt_controller, register_global_interrupt, sync::spin_rwlock::IrqSafeRwLock, + util::OneTimeInit, +}; +use yggdrasil_abi::error::Error; + +use crate::{ + capability::{MsiCapability, MsiXCapability, MsiXVectorTable}, + PciAddress, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo, +}; + +/// Describes a PCI device +#[derive(Clone)] +pub struct PciDeviceInfo { + /// Address of the device + pub address: PciAddress, + /// Configuration space access method + pub config_space: PciConfigSpace, + /// Describes the PCI segment this device is a part of + pub segment: Arc, + + pub(crate) interrupt_config: Arc>>, +} + +pub struct InterruptConfig { + #[allow(unused)] + preferred_mode: PreferredInterruptMode, + configured_mode: ConfiguredInterruptMode, +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub enum PciInterruptPin { + A, + B, + C, + D, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PreferredInterruptMode { + Msi, + Legacy, +} + +enum ConfiguredInterruptMode { + MsiX(MsiXVectorTable<'static>), + Msi, + Legacy(PciInterruptPin), + None, +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct PciInterrupt { + pub address: PciAddress, + pub pin: PciInterruptPin, +} + +#[derive(Clone, Copy, Debug)] +pub struct PciInterruptRoute { + pub number: u32, + pub options: IrqOptions, +} + +pub enum PciMatch { + Generic(fn(&PciDeviceInfo) -> bool), + Vendor(u16, u16), + Class(u8, Option, Option), +} + +pub struct PciDriver { + pub(crate) name: &'static str, + pub(crate) check: PciMatch, + pub(crate) probe: fn(&PciDeviceInfo) -> Result<&'static dyn Device, Error>, +} + +/// Used to store PCI bus devices which were enumerated by the kernel +pub struct PciBusDevice { + pub(crate) info: PciDeviceInfo, + pub(crate) driver: Option<&'static dyn Device>, +} + +impl PciDeviceInfo { + pub fn init_interrupts(&self, preferred_mode: PreferredInterruptMode) -> Result<(), Error> { + self.interrupt_config + .try_init_with(|| { + let configured_mode = + if self.segment.has_msi && preferred_mode == PreferredInterruptMode::Msi { + if let Some(mut msix) = self.config_space.capability::() { + let mut vt = msix.vector_table().unwrap(); + + vt.mask_all(); + + msix.set_function_mask(false); + msix.set_enabled(true); + + ConfiguredInterruptMode::MsiX(vt) + } else if self.config_space.capability::().is_some() { + ConfiguredInterruptMode::Msi + } else { + self.interrupt_mode_from_pin() + } + } else { + // Ignore preferred_mode, the only supported is Legacy + self.interrupt_mode_from_pin() + }; + + IrqSafeRwLock::new(InterruptConfig { + preferred_mode, + configured_mode, + }) + }) + .expect("Attempted to double-configure interrupts for a PCI device"); + + Ok(()) + } + + fn interrupt_mode_from_pin(&self) -> ConfiguredInterruptMode { + match self.config_space.interrupt_pin() { + Some(pin) => ConfiguredInterruptMode::Legacy(pin), + None => ConfiguredInterruptMode::None, + } + } + + pub fn map_interrupt( + &self, + affinity: InterruptAffinity, + handler: &'static dyn InterruptHandler, + ) -> Result, Error> { + let mut irq = self.interrupt_config.get().write(); + + match &mut irq.configured_mode { + ConfiguredInterruptMode::MsiX(msix) => { + let info = + msix.register_range(0, 1, message_interrupt_controller(), affinity, handler)?; + Ok(Some(info[0])) + } + ConfiguredInterruptMode::Msi => { + let mut msi = self + .config_space + .capability::() + .ok_or(Error::InvalidOperation)?; + + let info = msi.register(message_interrupt_controller(), affinity, handler)?; + + Ok(Some(info)) + } + ConfiguredInterruptMode::Legacy(pin) => { + self.try_map_legacy(*pin, handler)?; + Ok(None) + } + ConfiguredInterruptMode::None => Err(Error::InvalidOperation), + } + } + + pub fn map_interrupt_multiple( + &self, + vector_range: Range, + affinity: InterruptAffinity, + handler: &'static dyn InterruptHandler, + ) -> Result, Error> { + let mut irq = self.interrupt_config.get().write(); + let start = vector_range.start; + let end = vector_range.end; + + match &mut irq.configured_mode { + ConfiguredInterruptMode::MsiX(msix) => msix.register_range( + start, + end, + message_interrupt_controller(), + affinity, + handler, + ), + _ => Err(Error::InvalidOperation), + } + } + + fn try_map_legacy( + &self, + pin: PciInterruptPin, + handler: &'static dyn InterruptHandler, + ) -> Result<(), Error> { + let src = PciInterrupt { + address: self.address, + pin, + }; + let route = self + .segment + .irq_translation_map + .get(&src) + .ok_or(Error::InvalidOperation)?; + + log::debug!( + "PCI {} pin {:?} -> system IRQ #{}", + src.address, + src.pin, + route.number + ); + + register_global_interrupt(route.number, route.options, handler) + } +} + +impl TryFrom for PciInterruptPin { + type Error = (); + + fn try_from(value: u32) -> Result { + match value { + 1 => Ok(Self::A), + 2 => Ok(Self::B), + 3 => Ok(Self::C), + 4 => Ok(Self::D), + _ => Err(()), + } + } +} diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index 4aebb4b6..823c7f9c 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -5,17 +5,21 @@ extern crate alloc; use core::fmt; +#[cfg(target_arch = "x86_64")] use acpi::mcfg::McfgEntry; -use alloc::vec::Vec; +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use bitflags::bitflags; +use device::{PciBusDevice, PciDeviceInfo, PciDriver, PciInterrupt, PciInterruptRoute, PciMatch}; use device_api::Device; use kernel_util::{ mem::address::{FromRaw, PhysicalAddress}, sync::IrqSafeSpinlock, + util::OneTimeInit, }; use yggdrasil_abi::error::Error; pub mod capability; +pub mod device; mod space; pub use space::{ @@ -45,7 +49,7 @@ bitflags! { } /// Represents the address of a single object on a bus (or the bus itself) -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] pub struct PciAddress { /// PCIe segment group, ignored (?) with PCI pub segment: u8, @@ -58,10 +62,12 @@ pub struct PciAddress { } /// Address provided by PCI configuration space Base Address Register -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub enum PciBaseAddress { - /// 32/64-bit memory address - Memory(usize), + /// 32-bit memory address + Memory32(u32), + /// 64-bit memory address + Memory64(u64), /// I/O space address Io(u16), } @@ -101,62 +107,141 @@ pub trait PciCapability { ) -> Self::CapabilityData<'s, S>; } -/// Describes a PCI device +struct BusAddressAllocator { + pci_base_64: u64, + pci_base_32: u32, + // pci_base_io: u16, + host_base_64: PhysicalAddress, + host_base_32: PhysicalAddress, + // host_base_io: PhysicalAddress, + size_64: usize, + size_32: usize, + // size_io: usize, + offset_64: u64, + offset_32: u32, +} + +#[cfg_attr(target_arch = "x86_64", allow(dead_code))] +impl BusAddressAllocator { + pub fn from_ranges(ranges: &[PciAddressRange]) -> Self { + let mut range_32 = None; + let mut range_64 = None; + // let mut range_io = None; + + for range in ranges { + let range_val = (range.pci_base, range.host_base, range.size); + match range.ty { + // PciRangeType::Io if range_io.is_none() => { + // range_io.replace(range_val); + // } + PciRangeType::Memory32 if range_32.is_none() => { + range_32.replace(range_val); + } + PciRangeType::Memory64 if range_64.is_none() => { + range_64.replace(range_val); + } + _ => (), + } + } + + let (pci_base_32, host_base_32, size_32) = range_32.unwrap(); + let (pci_base_64, host_base_64, size_64) = range_64.unwrap(); + // let (pci_base_io, host_base_io, size_io) = range_io.unwrap(); + + Self { + pci_base_64, + pci_base_32: pci_base_32.try_into().unwrap(), + // pci_base_io: pci_base_io.try_into().unwrap(), + host_base_64, + host_base_32, + // host_base_io, + size_64, + size_32, + // size_io, + offset_64: 0, + offset_32: 0, + } + } + + pub fn allocate(&mut self, ty: PciRangeType, size: usize) -> (PciBaseAddress, PhysicalAddress) { + match ty { + PciRangeType::Io => todo!(), + PciRangeType::Memory32 => { + if self.offset_32 as usize + size >= self.size_32 { + todo!(); + } + let bar = PciBaseAddress::Memory32(self.pci_base_32 + self.offset_32); + let host = self.host_base_32.add(self.offset_32 as usize); + self.offset_32 += size as u32; + (bar, host) + } + PciRangeType::Memory64 => { + if self.offset_64 as usize + size >= self.size_64 { + todo!(); + } + let bar = PciBaseAddress::Memory64(self.pci_base_64 + self.offset_64); + let host = self.host_base_64.add(self.offset_64 as usize); + self.offset_64 += size as u64; + (bar, host) + } + PciRangeType::Configuration => unimplemented!(), + } + } +} + #[derive(Debug)] -pub struct PciDeviceInfo { - /// Address of the device - pub address: PciAddress, - /// Configuration space access method - pub config_space: PciConfigSpace, -} +pub struct PciSegmentInfo { + pub segment_number: u8, + pub bus_number_start: u8, + pub bus_number_end: u8, + pub ecam_phys_base: Option, -pub enum PciMatch { - Generic(fn(&PciDeviceInfo) -> bool), - Vendor(u16, u16), - Class(u8, Option, Option), -} - -pub struct PciDriver { - name: &'static str, - check: PciMatch, - probe: fn(&PciDeviceInfo) -> Result<&'static dyn Device, Error>, -} - -/// Used to store PCI bus devices which were enumerated by the kernel -pub struct PciBusDevice { - info: PciDeviceInfo, - driver: Option<&'static dyn Device>, + pub irq_translation_map: BTreeMap, + pub has_msi: bool, } /// Represents a single PCIe bus segment pub struct PciBusSegment { - segment_number: u8, - bus_number_start: u8, - bus_number_end: u8, - ecam_phys_base: Option, - + allocator: Option, + info: Arc, devices: Vec, } +pub enum PciRangeType { + Configuration, + Io, + Memory32, + Memory64, +} + +pub struct PciAddressRange { + pub ty: PciRangeType, + pub bus_number: u8, + pub pci_base: u64, + pub host_base: PhysicalAddress, + pub size: usize, +} + /// Manager struct to store and control all PCI devices in the system pub struct PciBusManager { segments: Vec, } impl PciBaseAddress { - pub fn as_memory(self) -> usize { + pub fn as_memory(self) -> Option { match self { - Self::Memory(address) => address, - _ => panic!("Not a memory BAR"), + Self::Memory32(address) => Some(PhysicalAddress::from_raw(address as u64)), + Self::Memory64(address) => Some(PhysicalAddress::from_raw(address)), + _ => None, } } } impl PciBusSegment { fn probe_config_space(&self, address: PciAddress) -> Result, Error> { - match self.ecam_phys_base { + match self.info.ecam_phys_base { Some(ecam_phys_base) => Ok(unsafe { - PciEcam::probe_raw_parts(ecam_phys_base, self.bus_number_start, address)? + PciEcam::probe_raw_parts(ecam_phys_base, self.info.bus_number_start, address)? } .map(PciConfigSpace::Ecam)), None => todo!(), @@ -183,9 +268,86 @@ impl PciBusSegment { // // TODO // } + if let Some(allocator) = self.allocator.as_mut() { + log::debug!("Remapping BARs for {}", address); + + // Find valid BARs + let mut i = 0; + let mut bar_mask = 0; + + while i < 6 { + let w0 = config.read_u32(0x10 + i * 4); + + let bar_width = match w0 & 1 == 0 { + // Memory BAR + true => match (w0 >> 1) & 3 { + // 32-bit BAR + 0 => 1, + // Reserved + 1 => unimplemented!(), + // 64-bit BAR + 2 => 2, + // Unknown + _ => unreachable!(), + }, + false => 1, + }; + + bar_mask |= 1 << i; + i += bar_width; + } + + for i in 0..6 { + if (1 << i) & bar_mask != 0 { + let orig_value = config.bar(i).unwrap(); + let size = unsafe { config.bar_size(i) }; + + if size != 0 { + log::debug!("BAR{}: size={:#x}", i, size); + + match orig_value { + PciBaseAddress::Io(_) => (), + PciBaseAddress::Memory64(_) => { + let (bar, host) = allocator.allocate(PciRangeType::Memory64, size); + let bar_address = bar.as_memory().unwrap(); + unsafe { + config.set_bar(i, bar); + } + log::debug!( + "Mapped BAR{} -> pci {:#x} host {:#x}", + i, + bar_address, + host + ); + // TODO Don't yet differentiate between Host/PCI addresses, lol + assert_eq!(bar_address, host); + } + PciBaseAddress::Memory32(_) => { + let (bar, host) = allocator.allocate(PciRangeType::Memory32, size); + let bar_address = bar.as_memory().unwrap(); + unsafe { + config.set_bar(i, bar); + } + log::debug!( + "Mapped BAR{} -> pci {:#x} host {:#x}", + i, + bar_address, + host + ); + // TODO Don't yet differentiate between Host/PCI addresses, lol + assert_eq!(bar_address, host); + } + } + } + } + } + } + let info = PciDeviceInfo { address, + segment: self.info.clone(), config_space: config, + interrupt_config: Arc::new(OneTimeInit::new()), }; self.devices.push(PciBusDevice { info, driver: None }); @@ -193,7 +355,7 @@ impl PciBusSegment { } fn enumerate_bus(&mut self, bus: u8) -> Result<(), Error> { - let address = PciAddress::for_bus(self.segment_number, bus); + let address = PciAddress::for_bus(self.info.segment_number, bus); for i in 0..32 { let device_address = address.with_device(i); @@ -206,7 +368,7 @@ impl PciBusSegment { /// Enumerates the bus segment, placing found devices into the manager pub fn enumerate(&mut self) -> Result<(), Error> { - for bus in self.bus_number_start..self.bus_number_end { + for bus in self.info.bus_number_start..self.info.bus_number_end { self.enumerate_bus(bus)?; } Ok(()) @@ -250,12 +412,50 @@ impl PciBusManager { } /// Enumerates a bus segment provided by ACPI MCFG table entry + #[cfg(target_arch = "x86_64")] pub fn add_segment_from_mcfg(entry: &McfgEntry) -> Result<(), Error> { let mut bus_segment = PciBusSegment { - segment_number: entry.pci_segment_group as u8, - bus_number_start: entry.bus_number_start, - bus_number_end: entry.bus_number_end, - ecam_phys_base: Some(PhysicalAddress::from_raw(entry.base_address)), + info: Arc::new(PciSegmentInfo { + segment_number: entry.pci_segment_group as u8, + bus_number_start: entry.bus_number_start, + bus_number_end: entry.bus_number_end, + ecam_phys_base: Some(PhysicalAddress::from_raw(entry.base_address)), + + // TODO obtain this from ACPI SSDT + irq_translation_map: BTreeMap::new(), + has_msi: true, + }), + // Firmware done this for us + allocator: None, + + devices: Vec::new(), + }; + + let mut this = PCI_MANAGER.lock(); + bus_segment.enumerate()?; + this.segments.push(bus_segment); + + Ok(()) + } + + #[cfg(target_arch = "aarch64")] + pub fn add_segment_from_device_tree( + cfg_base: PhysicalAddress, + bus_range: core::ops::Range, + ranges: Vec, + interrupt_map: BTreeMap, + ) -> Result<(), Error> { + let mut bus_segment = PciBusSegment { + info: Arc::new(PciSegmentInfo { + segment_number: 0, + bus_number_start: bus_range.start, + bus_number_end: bus_range.end, + ecam_phys_base: Some(cfg_base), + + irq_translation_map: interrupt_map, + has_msi: false, + }), + allocator: Some(BusAddressAllocator::from_ranges(&ranges)), devices: Vec::new(), }; @@ -285,6 +485,16 @@ impl PciAddress { } } + /// Constructs a [PciAddress] representing a specific function + pub const fn for_function(segment: u8, bus: u8, device: u8, function: u8) -> Self { + Self { + segment, + bus, + device, + function, + } + } + /// Constructs a [PciAddress] representing a device on a given bus pub const fn with_device(self, device: u8) -> Self { Self { diff --git a/driver/bus/pci/src/space/ecam.rs b/driver/bus/pci/src/space/ecam.rs index 5f948e20..06490553 100644 --- a/driver/bus/pci/src/space/ecam.rs +++ b/driver/bus/pci/src/space/ecam.rs @@ -5,7 +5,7 @@ use yggdrasil_abi::error::Error; use super::{PciAddress, PciConfigurationSpace}; /// PCI Express Enhanced Configuration Access Mechanism -#[derive(Debug)] +#[derive(Debug, Clone)] #[repr(transparent)] pub struct PciEcam { mapping: DeviceMemoryMapping, @@ -32,7 +32,7 @@ impl PciEcam { /// regions. The address must be aligned to a 4KiB boundary and be valid for accesses within a /// 4KiB-sized range. pub unsafe fn map(phys_addr: PhysicalAddress) -> Result { - let mapping = DeviceMemoryMapping::map(phys_addr, 0x1000)?; + let mapping = DeviceMemoryMapping::map(phys_addr, 0x1000, Default::default())?; Ok(Self { mapping }) } diff --git a/driver/bus/pci/src/space/mod.rs b/driver/bus/pci/src/space/mod.rs index 206d5326..2f860926 100644 --- a/driver/bus/pci/src/space/mod.rs +++ b/driver/bus/pci/src/space/mod.rs @@ -1,5 +1,7 @@ +use alloc::sync::Arc; + use super::{PciAddress, PciBaseAddress, PciCapability, PciCapabilityId, PciEcam}; -use crate::PciStatusRegister; +use crate::{device::PciInterruptPin, PciCommandRegister, PciStatusRegister}; pub(super) mod ecam; @@ -60,7 +62,7 @@ pub struct PciLegacyConfigurationSpace { } /// Describes a configuration space access method for a PCI device -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum PciConfigSpace { /// Legacy configuration space. /// @@ -220,6 +222,63 @@ pub trait PciConfigurationSpace { capability_pointer ); + fn interrupt_pin(&self) -> Option { + PciInterruptPin::try_from(self.read_u8(0x3D) as u32).ok() + } + + unsafe fn bar_size(&self, index: usize) -> usize { + let cmd = self.command(); + + // Disable I/O and memory + self.set_command( + cmd & !(PciCommandRegister::ENABLE_IO | PciCommandRegister::ENABLE_MEMORY).bits(), + ); + + let orig_value = self.bar(index).unwrap(); + // TODO preserve prefetch bit + let mask_value = match orig_value { + PciBaseAddress::Io(_) => PciBaseAddress::Io(0xFFFC), + PciBaseAddress::Memory32(_) => PciBaseAddress::Memory32(0xFFFFFFF0), + PciBaseAddress::Memory64(_) => PciBaseAddress::Memory64(0xFFFFFFFFFFFFFFF0), + }; + self.set_bar(index, mask_value); + let new_value = self.bar(index).unwrap(); + + let size = match new_value { + PciBaseAddress::Io(address) if address != 0 => ((!address) + 1) as usize, + PciBaseAddress::Memory32(address) if address != 0 => ((!address) + 1) as usize, + PciBaseAddress::Memory64(address) if address != 0 => ((!address) + 1) as usize, + _ => 0, + }; + + self.set_bar(index, orig_value); + self.set_command(cmd); + + size + } + + /// Updates the value of the Base Address Register with given index. + /// + /// # Note + /// + /// The function is only valid for devices with `header_type() == 0` + /// + /// The `index` corresponds to the actual configuration space BAR index. + unsafe fn set_bar(&self, index: usize, value: PciBaseAddress) { + assert!(index < 6); + + match value { + PciBaseAddress::Io(value) => { + self.write_u32(0x10 + index * 4, ((value as u32) & !0x3) | 1) + } + PciBaseAddress::Memory32(address) => self.write_u32(0x10 + index * 4, address & !0xF), + PciBaseAddress::Memory64(address) => { + self.write_u32(0x10 + index * 4, ((address as u32) & !0xF) | (2 << 1)); + self.write_u32(0x10 + (index + 1) * 4, (address >> 32) as u32); + } + } + } + /// Returns the value of the Base Address Register with given index. /// /// # Note @@ -238,18 +297,18 @@ pub trait PciConfigurationSpace { 0 => match (w0 >> 1) & 3 { 0 => { // 32-bit memory BAR - Some(PciBaseAddress::Memory((w0 as usize) & !0xF)) + Some(PciBaseAddress::Memory32(w0 & !0xF)) } 2 => { // 64-bit memory BAR let w1 = self.read_u32(0x10 + (index + 1) * 4); - Some(PciBaseAddress::Memory( - ((w1 as usize) << 32) | ((w0 as usize) & !0xF), + Some(PciBaseAddress::Memory64( + ((w1 as u64) << 32) | ((w0 as u64) & !0xF), )) } _ => unimplemented!(), }, - 1 => todo!(), + 1 => Some(PciBaseAddress::Io((w0 as u16) & !0x3)), _ => unreachable!(), } } else { @@ -265,7 +324,7 @@ pub trait PciConfigurationSpace { 0 => match (w0 >> 1) & 3 { 0 => { // 32-bit memory BAR - Some(PciBaseAddress::Memory((w0 as usize) & !0xF)) + Some(PciBaseAddress::Memory32(w0 & !0xF)) } // TODO can 64-bit BARs not be on a 64-bit boundary? 2 => todo!(), @@ -311,3 +370,13 @@ pub trait PciConfigurationSpace { }) } } + +impl PciConfigurationSpace for Arc { + fn read_u32(&self, offset: usize) -> u32 { + T::read_u32(self.as_ref(), offset) + } + + fn write_u32(&self, offset: usize, value: u32) { + T::write_u32(self.as_ref(), offset, value); + } +} diff --git a/driver/virtio/core/src/error.rs b/driver/virtio/core/src/error.rs index 6538e069..f04cb2f4 100644 --- a/driver/virtio/core/src/error.rs +++ b/driver/virtio/core/src/error.rs @@ -5,6 +5,7 @@ pub enum Error { NoCommonConfigCapability, NoNotifyConfigCapability, NoDeviceConfigCapability, + NoInterruptStatusCapability, QueueTooLarge, InvalidQueueSize, EmptyTransaction, diff --git a/driver/virtio/core/src/queue.rs b/driver/virtio/core/src/queue.rs index 4043b556..0c738cbf 100644 --- a/driver/virtio/core/src/queue.rs +++ b/driver/virtio/core/src/queue.rs @@ -66,9 +66,13 @@ pub struct VirtQueue { } impl AvailableRing { - pub fn with_capacity(capacity: usize) -> Result { + pub fn with_capacity(no_irq: bool, capacity: usize) -> Result { let mut data = PageBox::new_uninit_slice(capacity + 3)?; + if no_irq { + data[0].write(1); + } + data[1].write(0); Ok(Self { data }) @@ -112,6 +116,7 @@ impl VirtQueue { index: u16, capacity: usize, msix_vector: Option, + no_avail_irq: bool, ) -> Result { // TODO check if queue is already set up @@ -126,7 +131,7 @@ impl VirtQueue { } let descriptor_table = PageBox::new_uninit_slice(capacity)?; - let available = AvailableRing::with_capacity(capacity)?; + let available = AvailableRing::with_capacity(no_avail_irq, capacity)?; let used = UsedRing::with_capacity(capacity)?; transport.set_queue( @@ -164,11 +169,12 @@ impl VirtQueue { index: u16, capacity: usize, msix_vector: Option, + no_avail_irq: bool, ) -> Result { let max_capacity = transport.max_queue_size(index); let capacity = capacity.min(max_capacity as usize); - Self::with_capacity(transport, index, capacity, msix_vector) + Self::with_capacity(transport, index, capacity, msix_vector, no_avail_irq) } pub unsafe fn add<'a, 'b>( diff --git a/driver/virtio/core/src/transport/mod.rs b/driver/virtio/core/src/transport/mod.rs index 4d761147..1ae3aa00 100644 --- a/driver/virtio/core/src/transport/mod.rs +++ b/driver/virtio/core/src/transport/mod.rs @@ -19,6 +19,7 @@ pub trait Transport { fn notify_off_mul(&self) -> usize; fn supports_msix(&self) -> bool; fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>>; + fn read_interrupt_status(&self) -> (bool, bool); fn read_device_features(&mut self) -> u64 { let cfg = self.common_cfg(); @@ -71,6 +72,8 @@ pub trait Transport { cfg.queue_device.set(used_ring_phys.into_raw()); if self.supports_msix() { cfg.queue_msix_vector.set(msix_vector.unwrap_or(0xFFFF)); + } else { + cfg.queue_msix_vector.set(0xFFFF); } cfg.queue_enable.set(1); } diff --git a/driver/virtio/core/src/transport/pci.rs b/driver/virtio/core/src/transport/pci.rs index eca3d1e7..320730ee 100644 --- a/driver/virtio/core/src/transport/pci.rs +++ b/driver/virtio/core/src/transport/pci.rs @@ -1,12 +1,12 @@ -use kernel_util::mem::{ - address::{FromRaw, PhysicalAddress}, - device::DeviceMemoryIo, +use kernel_util::mem::device::DeviceMemoryIo; +use tock_registers::{ + interfaces::Readable, + registers::{ReadOnly, WriteOnly}, }; -use tock_registers::registers::WriteOnly; use ygg_driver_pci::{ capability::{ VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability, - VirtioNotifyConfigCapability, + VirtioInterruptStatusCapability, VirtioNotifyConfigCapability, }, PciCommandRegister, PciConfigurationSpace, }; @@ -19,6 +19,7 @@ pub struct PciTransport { common_cfg: DeviceMemoryIo<'static, CommonConfiguration>, device_cfg: DeviceMemoryIo<'static, [u8]>, notify_cfg: DeviceMemoryIo<'static, [WriteOnly]>, + isr: DeviceMemoryIo<'static, ReadOnly>, notify_cfg_mul: usize, } @@ -42,6 +43,11 @@ impl Transport for PciTransport { fn device_cfg(&self) -> Option<&DeviceMemoryIo<[u8]>> { Some(&self.device_cfg) } + + fn read_interrupt_status(&self) -> (bool, bool) { + let value = self.isr.get(); + (value & 1 != 0, value & 2 != 0) + } } impl PciTransport { @@ -73,6 +79,9 @@ impl PciTransport { let notify_cfg_cap = space .capability::() .ok_or(Error::NoNotifyConfigCapability)?; + let isr_cap = space + .capability::() + .ok_or(Error::NoInterruptStatusCapability)?; // TODO MSI/MSI-X @@ -82,38 +91,52 @@ impl PciTransport { .bar(common_cfg_cap.bar_index().unwrap()) .unwrap() .as_memory() - + common_cfg_cap.bar_offset(); + .unwrap() + .add(common_cfg_cap.bar_offset()); let device_cfg_base = space .bar(device_cfg_cap.bar_index().unwrap()) .unwrap() .as_memory() - + device_cfg_cap.bar_offset(); + .unwrap() + .add(device_cfg_cap.bar_offset()); let device_cfg_len = device_cfg_cap.length(); let notify_cfg_base = space .bar(notify_cfg_cap.bar_index().unwrap()) .unwrap() .as_memory() - + notify_cfg_cap.bar_offset(); + .unwrap() + .add(notify_cfg_cap.bar_offset()); let notify_cfg_len = notify_cfg_cap.length(); let notify_cfg_mul = notify_cfg_cap.offset_multiplier(); + let isr_base = space + .bar(isr_cap.bar_index().unwrap()) + .unwrap() + .as_memory() + .unwrap() + .add(isr_cap.bar_offset()); + let isr_len = isr_cap.length(); - let common_cfg_base = PhysicalAddress::from_raw(common_cfg_base); - let device_cfg_base = PhysicalAddress::from_raw(device_cfg_base); - let notify_cfg_base = PhysicalAddress::from_raw(notify_cfg_base); - + assert!(isr_len >= 4); assert_eq!(notify_cfg_len % 2, 0); - let common_cfg = unsafe { DeviceMemoryIo::map(common_cfg_base) }.unwrap(); - let device_cfg = - unsafe { DeviceMemoryIo::map_slice(device_cfg_base, device_cfg_len) }.unwrap(); - let notify_cfg = - unsafe { DeviceMemoryIo::map_slice(notify_cfg_base, notify_cfg_len / 2) }.unwrap(); + let common_cfg = + unsafe { DeviceMemoryIo::map(common_cfg_base, Default::default()) }.unwrap(); + let device_cfg = unsafe { + DeviceMemoryIo::map_slice(device_cfg_base, device_cfg_len, Default::default()) + } + .unwrap(); + let notify_cfg = unsafe { + DeviceMemoryIo::map_slice(notify_cfg_base, notify_cfg_len / 2, Default::default()) + } + .unwrap(); + let isr = unsafe { DeviceMemoryIo::map(isr_base, Default::default()) }.unwrap(); Ok(Self { common_cfg, device_cfg, notify_cfg, notify_cfg_mul, + isr, }) } } diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs index a6e57834..e3679eaa 100644 --- a/driver/virtio/net/src/lib.rs +++ b/driver/virtio/net/src/lib.rs @@ -9,12 +9,11 @@ use core::mem::size_of; use alloc::{boxed::Box, collections::BTreeMap}; use bytemuck::{Pod, Zeroable}; use device_api::{ - interrupt::{InterruptAffinity, MsiHandler}, + interrupt::{InterruptAffinity, InterruptHandler}, Device, }; use kernel_util::{ mem::PageBox, - message_interrupt_controller, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, util::OneTimeInit, }; @@ -22,10 +21,7 @@ use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, Packet, }; -use ygg_driver_pci::{ - capability::{MsiXCapability, MsiXVectorTable}, - PciConfigurationSpace, PciDeviceInfo, -}; +use ygg_driver_pci::device::{PciDeviceInfo, PreferredInterruptMode}; use ygg_driver_virtio_core::{ queue::VirtQueue, transport::{pci::PciTransport, Transport}, @@ -36,10 +32,9 @@ use yggdrasil_abi::{error::Error, net::MacAddress}; struct Queues { receive: IrqSafeSpinlock, transmit: IrqSafeSpinlock, - - #[allow(unused)] - configuration_vector: usize, - receive_vector: usize, + // #[allow(unused)] + // configuration_vector: usize, + receive_vector: Option, } pub struct VirtioNet { @@ -51,7 +46,7 @@ pub struct VirtioNet { pending_packets: IrqSafeRwLock>>, - vector_table: IrqSafeRwLock>, + pci_device_info: Option, } #[derive(Clone, Copy, Debug, Pod, Zeroable)] @@ -77,7 +72,7 @@ impl Queues { impl VirtioNet { const PACKET_SIZE: usize = 4096; - pub fn new(transport: T, vector_table: MsiXVectorTable<'static>) -> Self { + pub fn new(transport: T, pci_device_info: Option) -> Self { // Read MAC from device config let device_cfg = transport .device_cfg() @@ -94,7 +89,8 @@ impl VirtioNet { mac: IrqSafeRwLock::new(mac), pending_packets: IrqSafeRwLock::new(BTreeMap::new()), - vector_table: IrqSafeRwLock::new(vector_table), + + pci_device_info, } } @@ -108,6 +104,9 @@ impl VirtioNet { let token = unsafe { queue.add(&[&mut packet], &[]).unwrap() }; packets.insert(token, packet); } + + let mut transport = self.transport.lock(); + transport.notify(0); } fn handle_receive_interrupt(&self, queue: usize) -> bool { @@ -130,7 +129,7 @@ impl VirtioNet { } if count != 0 { - self.transport.lock().notify(1); + self.transport.lock().notify(0); } count != 0 @@ -177,35 +176,34 @@ impl VirtioNet { receive_count: usize, transmit_count: usize, ) -> Result<(), Error> { + let receive_vector = if let Some(pci) = self.pci_device_info.as_ref() { + pci.init_interrupts(PreferredInterruptMode::Msi)?; + let info = pci.map_interrupt(InterruptAffinity::Any, self)?; + if let Some(info) = info { + Some(info.vector as u16) + } else { + None + } + } else { + None + }; + // TODO multiqueue capability assert_eq!(receive_count, 1); assert_eq!(transmit_count, 1); let mut transport = self.transport.lock(); - let mut vt = self.vector_table.write(); - - let msix_range = vt.register_range( - 0, - 1 + receive_count, - message_interrupt_controller(), - InterruptAffinity::Any, - self, - )?; - - // TODO set the configuration vector in virtio common cfg - let receive_vector: u16 = msix_range[1].vector.try_into().unwrap(); // Setup the virtqs - let rx = VirtQueue::with_max_capacity(&mut *transport, 0, 128, Some(receive_vector)) + let rx = VirtQueue::with_max_capacity(&mut *transport, 0, 128, receive_vector, false) .map_err(cvt_error)?; - let tx = VirtQueue::with_max_capacity(&mut *transport, 1, 128, None).map_err(cvt_error)?; + let tx = + VirtQueue::with_max_capacity(&mut *transport, 1, 128, None, true).map_err(cvt_error)?; self.queues.init(Queues { receive: IrqSafeSpinlock::new(rx), transmit: IrqSafeSpinlock::new(tx), - - configuration_vector: msix_range[0].vector, - receive_vector: msix_range[1].vector, + receive_vector, }); Ok(()) @@ -234,20 +232,40 @@ impl NetworkDevice for VirtioNet { } } -impl MsiHandler for VirtioNet { - fn handle_msi(&self, vector: usize) -> bool { - let Some(queues) = self.queues.try_get() else { - return false; - }; +impl InterruptHandler for VirtioNet { + fn handle_irq(&self, vector: Option) -> bool { + if let Some(_) = vector { + // MSI/MSI-X + let Some(queues) = self.queues.try_get() else { + return false; + }; - if vector == queues.receive_vector { - self.handle_receive_interrupt(0) + if vector == queues.receive_vector.map(Into::into) { + self.handle_receive_interrupt(0) + } else { + false + } } else { - false + // Legacy IRQ + let (queue_irq, config_irq) = self.transport.lock().read_interrupt_status(); + + if queue_irq { + log::debug!("Handle IRQ"); + self.handle_receive_interrupt(0); + } + + queue_irq || config_irq } } } +// impl MsiHandler for VirtioNet { +// fn handle_msi(&self, vector: usize) -> bool { +// +// todo!() +// } +// } + impl Device for VirtioNet { fn display_name(&self) -> &'static str { "VirtIO Network Device" @@ -284,17 +302,8 @@ fn cvt_error(error: ygg_driver_virtio_core::error::Error) -> Error { pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { let space = &info.config_space; - let mut msix = space.capability::().unwrap(); - let mut vt = msix.vector_table()?; - - // TODO is this really needed? PCI spec says this is masked on reset, though I'm not sure if - // firmware puts it back in masked state after loading the kernel - vt.mask_all(); - msix.set_function_mask(false); - msix.set_enabled(true); - let transport = PciTransport::from_config_space(space).unwrap(); - let device = VirtioNet::new(transport, vt); + let device = VirtioNet::new(transport, Some(info.clone())); let device = Box::leak(Box::new(device)); diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index 03d85f6b..bee36904 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -56,7 +56,7 @@ pub trait MessageInterruptController { fn register_msi( &self, affinity: InterruptAffinity, - handler: &'static dyn MsiHandler, + handler: &'static dyn InterruptHandler, ) -> Result { let mut range = [MsiInfo { affinity, @@ -70,7 +70,7 @@ pub trait MessageInterruptController { fn register_msi_range( &self, range: &mut [MsiInfo], - handler: &'static dyn MsiHandler, + handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { Err(Error::NotImplemented) } @@ -118,11 +118,7 @@ pub trait LocalInterruptController { } pub trait InterruptHandler: Device { - fn handle_irq(&self) -> bool; -} - -pub trait MsiHandler: Device { - fn handle_msi(&self, vector: usize) -> bool; + fn handle_irq(&self, vector: Option) -> bool; } pub struct FixedInterruptTable { diff --git a/lib/kernel-util/src/api.rs b/lib/kernel-util/src/api.rs index fae48fe4..d3beedf2 100644 --- a/lib/kernel-util/src/api.rs +++ b/lib/kernel-util/src/api.rs @@ -1,14 +1,17 @@ use core::time::Duration; use alloc::{string::String, sync::Arc}; -use device_api::interrupt::MessageInterruptController; +use device_api::interrupt::{InterruptHandler, IrqOptions, MessageInterruptController}; use yggdrasil_abi::{ error::Error, process::{ExitCode, Signal}, }; use crate::{ - mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping}, + mem::{ + address::PhysicalAddress, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + }, thread::{CurrentThread, Thread}, }; @@ -30,6 +33,7 @@ extern "Rust" { pub fn __map_device_pages( base: PhysicalAddress, count: usize, + attrs: DeviceMemoryAttributes, ) -> Result; pub fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping); @@ -49,6 +53,11 @@ extern "Rust" { pub fn __monotonic_timestamp() -> Result; pub fn __message_interrupt_controller() -> &'static dyn MessageInterruptController; + pub fn __register_global_interrupt( + irq: u32, + options: IrqOptions, + handler: &'static dyn InterruptHandler, + ) -> Result<(), Error>; pub fn __signal_process_group(group_id: u32, signal: Signal); } diff --git a/lib/kernel-util/src/lib.rs b/lib/kernel-util/src/lib.rs index 98403a48..87a53add 100644 --- a/lib/kernel-util/src/lib.rs +++ b/lib/kernel-util/src/lib.rs @@ -17,7 +17,7 @@ use core::time::Duration; -use device_api::interrupt::MessageInterruptController; +use device_api::interrupt::{InterruptHandler, IrqOptions, MessageInterruptController}; use yggdrasil_abi::{error::Error, process::Signal}; extern crate alloc; @@ -35,6 +35,15 @@ pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController unsafe { api::__message_interrupt_controller() } } +#[inline] +pub fn register_global_interrupt( + irq: u32, + options: IrqOptions, + handler: &'static dyn InterruptHandler, +) -> Result<(), Error> { + unsafe { api::__register_global_interrupt(irq, options, handler) } +} + #[inline] pub fn cpu_index() -> usize { unsafe { api::__cpu_index() } diff --git a/lib/kernel-util/src/mem/device.rs b/lib/kernel-util/src/mem/device.rs index a39c073a..45647cd9 100644 --- a/lib/kernel-util/src/mem/device.rs +++ b/lib/kernel-util/src/mem/device.rs @@ -25,6 +25,18 @@ pub struct RawDeviceMemoryMapping { pub page_count: usize, } +#[derive(Debug, Default, Clone, Copy)] +pub enum DeviceMemoryCaching { + #[default] + None, + Cacheable, +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct DeviceMemoryAttributes { + pub caching: DeviceMemoryCaching, +} + /// Describes a single untyped device memory mapping #[derive(Clone, Debug)] pub struct DeviceMemoryMapping { @@ -56,8 +68,12 @@ impl RawDeviceMemoryMapping { /// /// The caller must ensure proper access synchronization, as well as the address' origin. #[inline] - pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { - __map_device_pages(base, size) + pub unsafe fn map( + base: PhysicalAddress, + size: usize, + attrs: DeviceMemoryAttributes, + ) -> Result { + __map_device_pages(base, size, attrs) } /// Consumes the device mapping, leaking its address without deallocating the translation @@ -118,8 +134,12 @@ impl DeviceMemoryMapping { /// # Safety /// /// The caller must ensure proper access synchronization, as well as the address' origin. - pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result { - let inner = RawDeviceMemoryMapping::map(base, size)?; + pub unsafe fn map( + base: PhysicalAddress, + size: usize, + attrs: DeviceMemoryAttributes, + ) -> Result { + let inner = RawDeviceMemoryMapping::map(base, size, attrs)?; let address = inner.address; Ok(Self { inner: Arc::new(inner), @@ -160,9 +180,10 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { pub unsafe fn map_slice( base: PhysicalAddress, count: usize, + attrs: DeviceMemoryAttributes, ) -> Result, Error> { let layout = Layout::array::(count).unwrap(); - let inner = RawDeviceMemoryMapping::map(base, layout.size())?; + let inner = RawDeviceMemoryMapping::map(base, layout.size(), attrs)?; let value = core::slice::from_raw_parts(inner.address as *mut T, count); Ok(DeviceMemoryIo { @@ -177,8 +198,11 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { /// /// The caller must ensure the address actually points to a value of type `T`, as well as /// proper access synchronization. - pub unsafe fn map(base: PhysicalAddress) -> Result, Error> { - let inner = RawDeviceMemoryMapping::map(base, size_of::())?; + pub unsafe fn map( + base: PhysicalAddress, + attrs: DeviceMemoryAttributes, + ) -> Result, Error> { + let inner = RawDeviceMemoryMapping::map(base, size_of::(), attrs)?; let value = &*(inner.address as *const T); Ok(DeviceMemoryIo { @@ -235,9 +259,10 @@ impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> { pub unsafe fn map_slice( base: PhysicalAddress, len: usize, + attrs: DeviceMemoryAttributes, ) -> Result, Error> { let layout = Layout::array::(len).unwrap(); - let inner = RawDeviceMemoryMapping::map(base, layout.size())?; + let inner = RawDeviceMemoryMapping::map(base, layout.size(), attrs)?; let value = core::slice::from_raw_parts_mut(inner.address as *mut T, len); Ok(DeviceMemoryIoMut { inner, value }) diff --git a/lib/kernel-util/src/util/mod.rs b/lib/kernel-util/src/util/mod.rs index 2f306efb..50a1cf9e 100644 --- a/lib/kernel-util/src/util/mod.rs +++ b/lib/kernel-util/src/util/mod.rs @@ -51,6 +51,35 @@ impl OneTimeInit { self.state.load(Ordering::Acquire) == Self::STATE_INITIALIZED } + pub fn try_init_with T>(&self, f: F) -> Option<&T> { + if self + .state + .compare_exchange( + Self::STATE_UNINITIALIZED, + Self::STATE_INITIALIZING, + Ordering::Release, + Ordering::Relaxed, + ) + .is_err() + { + // Already initialized + return None; + } + + let value = unsafe { (*self.value.get()).write(f()) }; + + self.state + .compare_exchange( + Self::STATE_INITIALIZING, + Self::STATE_INITIALIZED, + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); + + Some(value) + } + /// Sets the underlying value of the [OneTimeInit]. If already initialized, panics. #[track_caller] pub fn init(&self, value: T) -> &T { diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 70c46a8c..3e8f97ab 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -18,7 +18,7 @@ use crate::{ task::{sched::CpuQueue, thread::ThreadId}, }; -use super::smp::CPU_COUNT; +use super::{gic::GicPerCpu, smp::CPU_COUNT}; // use super::smp::CPU_COUNT; @@ -29,6 +29,8 @@ pub struct Cpu { queue: OneTimeInit<&'static CpuQueue>, thread_id: Option, + + pub(super) gic_per_cpu: GicPerCpu, } /// Handle allowing safe access to the local CPU. @@ -83,6 +85,7 @@ impl Cpu { id: Self::local_id(), queue: OneTimeInit::new(), thread_id: None, + gic_per_cpu: GicPerCpu::new(), }); TPIDR_EL1.set(Box::into_raw(this) as _); } diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index dc1aaf2c..ff69a78c 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -1,5 +1,5 @@ //! ARM GICv2 Distributor registers -use device_api::interrupt::IpiDeliveryTarget; +use device_api::interrupt::{IpiDeliveryTarget, IrqLevel, IrqOptions, IrqTrigger}; use kernel_util::mem::device::DeviceMemoryIo; use spinning_top::Spinlock; use tock_registers::{ @@ -113,6 +113,34 @@ impl Gicd { self.banked_regs.ITARGETSR[0].read(ITARGETSR::Offset0) } + pub fn configure_irq(&self, irq: usize, options: IrqOptions) { + // TODO configure trigger level + // 2 bits per IRQ, 16 entries per register + let reg = irq / 16; + let shift = (irq % 16) * 2; + let cfgr_value = match (options.trigger, options.level) { + (IrqTrigger::Level, IrqLevel::ActiveLow) => 0, + (IrqTrigger::Level, _) => 0, + (_, IrqLevel::ActiveLow) => 1, + (_, _) => 1, + }; + + match reg { + // Private IRQs + 0..=1 => { + todo!(); + } + // Shared IRQs + _ => { + let regs = self.shared_regs.lock(); + let reg = ®s.ICFGR[reg - 2]; + + let v = reg.get() & !(0x3 << shift); + reg.set(v | (cfgr_value << shift)); + } + } + } + pub fn enable_irq(&self, irq: usize) { let reg = irq >> 5; let bit = 1u32 << (irq & 0x1F); diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index bad6b6b9..bddf7500 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -8,7 +8,8 @@ use alloc::{boxed::Box, sync::Arc}; use device_api::{ interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, - IpiDeliveryTarget, IrqOptions, LocalInterruptController, + IpiDeliveryTarget, IrqOptions, LocalInterruptController, MessageInterruptController, + MsiInfo, }, Device, }; @@ -46,16 +47,23 @@ pub struct Gic { table: IrqSafeSpinlock>, } +/// Per-CPU GIC information +pub struct GicPerCpu {} + impl Device for Gic { fn display_name(&self) -> &'static str { "ARM Generic Interrupt Controller v2" } unsafe fn init(&'static self) -> Result<(), Error> { - let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map(self.gicd_base, 0x1000)?); + let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map( + self.gicd_base, + 0x1000, + Default::default(), + )?); let gicd_mmio_shared = DeviceMemoryIo::from_raw(gicd_mmio.clone())?; let gicd_mmio_banked = DeviceMemoryIo::from_raw(gicd_mmio)?; - let gicc_mmio = DeviceMemoryIo::map(self.gicc_base)?; + let gicc_mmio = DeviceMemoryIo::map(self.gicc_base, Default::default())?; let gicd = Gicd::new(gicd_mmio_shared, gicd_mmio_banked); let gicc = Gicc::new(gicc_mmio); @@ -68,6 +76,7 @@ impl Device for Gic { ARCHITECTURE.register_external_interrupt_controller(self)?; ARCHITECTURE.register_local_interrupt_controller(self)?; + ARCHITECTURE.register_message_interrupt_controller(self)?; Ok(()) } @@ -79,17 +88,27 @@ impl ExternalInterruptController for Gic { fn register_irq( &self, irq: IrqNumber, - _options: IrqOptions, + options: IrqOptions, handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { let mut table = self.table.lock(); + let gicd = self.gicd.get(); let index = match irq { IrqNumber::Shared(i) => i + 32, IrqNumber::Private(i) => i + 16, } as usize; - debugln!("Bound irq{} to {:?}", index, handler.display_name()); + debugln!( + "Bound irq{} to {:?} {:?} {:?}", + index, + handler.display_name(), + options.trigger, + options.level + ); + if index >= 32 { + gicd.configure_irq(index, options); + } table.insert(index, handler)?; Ok(()) @@ -125,7 +144,7 @@ impl ExternalInterruptController for Gic { match table.handler(irq_number) { Some(handler) => { drop(table); - handler.handle_irq(); + handler.handle_irq(None); } None => warnln!("No handler for irq{}", irq_number), } @@ -133,6 +152,20 @@ impl ExternalInterruptController for Gic { } } +impl MessageInterruptController for Gic { + fn handle_msi(&self, _vector: usize) { + todo!() + } + + fn register_msi_range( + &self, + _range: &mut [MsiInfo], + _handler: &'static dyn InterruptHandler, + ) -> Result<(), Error> { + todo!() + } +} + impl LocalInterruptController for Gic { type IpiMessage = CpuMessage; @@ -174,7 +207,7 @@ impl Gic { /// # Safety /// /// The caller must ensure the addresses actually point to the GIC components. - pub const unsafe fn new(gicd_base: PhysicalAddress, gicc_base: PhysicalAddress) -> Self { + pub unsafe fn new(gicd_base: PhysicalAddress, gicc_base: PhysicalAddress) -> Self { Self { gicc: OneTimeInit::new(), gicd: OneTimeInit::new(), @@ -185,6 +218,13 @@ impl Gic { } } +impl GicPerCpu { + /// Constructs per-CPU GIC data structure + pub fn new() -> Self { + Self {} + } +} + device_tree_driver! { compatible: ["arm,cortex-a15-gic", "arm,gic-400"], probe(dt) => { diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index 9c12ef90..650a6075 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -10,7 +10,7 @@ use cfg_if::cfg_if; use kernel_util::{ mem::{ address::{FromRaw, PhysicalAddress}, - device::RawDeviceMemoryMapping, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, table::{EntryLevel, EntryLevelExt}, }, util::OneTimeInit, @@ -208,7 +208,11 @@ pub(super) unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) { } // Device mappings -unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result { +unsafe fn map_device_memory_l3( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { // TODO don't map pages if already mapped 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { @@ -235,7 +239,11 @@ unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result Result { +unsafe fn map_device_memory_l2( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { for j in 0..count { if DEVICE_MAPPING_L2[i + j].is_present() { @@ -262,6 +270,7 @@ unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result Result { // debugln!("Map {}B @ {:#x}", size, base); let l3_aligned = base.page_align_down::(); @@ -274,7 +283,7 @@ pub(super) unsafe fn map_device_memory( let l2_offset = base.page_offset::(); let page_count = (l2_offset + size).page_count::(); - let base_address = map_device_memory_l2(l2_aligned, page_count)?; + let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; let address = base_address + l2_offset; Ok(RawDeviceMemoryMapping { @@ -285,7 +294,7 @@ pub(super) unsafe fn map_device_memory( }) } else { // Just map the pages directly - let base_address = map_device_memory_l3(l3_aligned, page_count)?; + let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; let address = base_address + l3_offset; Ok(RawDeviceMemoryMapping { @@ -312,8 +321,7 @@ pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; - todo!(); - // intrinsics::flush_tlb_entry(page); + tlb_flush_vaae1(page); } } L2::SIZE => todo!(), @@ -321,6 +329,14 @@ pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { } } +#[inline] +fn tlb_flush_vaae1(mut page: usize) { + page >>= 12; + unsafe { + core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); + } +} + /// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half" /// /// # Safety diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index afdbb71c..f7372ca4 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -14,7 +14,10 @@ use crate::mem::{ table::{EntryLevelDrop, MapAttributes, NextPageTable}, }; -use super::table::{PageEntry, PageTable, L1, L2, L3}; +use super::{ + table::{PageEntry, PageTable, L1, L2, L3}, + tlb_flush_vaae1, +}; /// AArch64 implementation of a process address space table #[repr(C)] @@ -141,11 +144,3 @@ impl Drop for ProcessAddressSpaceImpl { } } } - -#[inline] -fn tlb_flush_vaae1(mut page: usize) { - page >>= 12; - unsafe { - core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); - } -} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index d76beadd..4df427c6 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -7,7 +7,10 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF}; use abi::error::Error; use device_api::{ - interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, + interrupt::{ + ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController, + MessageInterruptController, + }, timer::MonotonicTimestampProviderDevice, ResetDevice, }; @@ -16,13 +19,14 @@ use git_version::git_version; use kernel_util::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, - device::RawDeviceMemoryMapping, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, pointer::PhysicalRef, table::{EntryLevel, EntryLevelExt}, }, util::OneTimeInit, }; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use ygg_driver_pci::PciBusManager; use crate::{ arch::aarch64::{ @@ -72,6 +76,7 @@ pub struct AArch64 { lintc: OneTimeInit<&'static dyn LocalInterruptController>, xintc: OneTimeInit<&'static dyn ExternalInterruptController>, + msi_intc: OneTimeInit<&'static dyn MessageInterruptController>, mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, @@ -116,8 +121,9 @@ impl Architecture for AArch64 { &self, base: PhysicalAddress, size: usize, + attrs: DeviceMemoryAttributes, ) -> Result { - mem::map_device_memory(base, size) + mem::map_device_memory(base, size, attrs) } unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) { @@ -178,6 +184,10 @@ impl Architecture for AArch64 { *self.xintc.get() } + fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { + *self.msi_intc.get() + } + fn register_local_interrupt_controller( &self, intc: &'static dyn LocalInterruptController, @@ -194,6 +204,14 @@ impl Architecture for AArch64 { Ok(()) } + fn register_message_interrupt_controller( + &self, + intc: &'static dyn MessageInterruptController, + ) -> Result<(), Error> { + self.msi_intc.init(intc); + Ok(()) + } + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { *self.mtimer.get() } @@ -334,6 +352,13 @@ impl AArch64 { Cpu::init_local(); if is_bsp { + ygg_driver_pci::register_vendor_driver( + "Virtio PCI Network Device", + 0x1AF4, + 0x1000, + ygg_driver_virtio_net::probe, + ); + let dt = self.dt.get(); let address_cells = dt.address_cells(); @@ -364,6 +389,8 @@ impl AArch64 { ); infoln!("Initializing aarch64 platform"); + dt.dump(crate::debug::LogLevel::Debug); + let nodes = dt.root().children(); if let Err(error) = devtree::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { @@ -402,6 +429,8 @@ impl AArch64 { device::manager_lock().devices().for_each(|dev| { infoln!("* {:?}", dev.display_name()); }); + + PciBusManager::setup_bus_devices()?; } else { // BSP already initialized everything needed // Setup timer and local interrupt controller @@ -431,6 +460,7 @@ pub static ARCHITECTURE: AArch64 = AArch64 { lintc: OneTimeInit::new(), xintc: OneTimeInit::new(), + msi_intc: OneTimeInit::new(), mtimer: OneTimeInit::new(), }; diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 6c463b53..c3625c99 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -25,7 +25,7 @@ pub struct ArmTimer { pub const TICK_INTERVAL: u64 = 1000000; impl InterruptHandler for ArmTimer { - fn handle_irq(&self) -> bool { + fn handle_irq(&self, _vector: Option) -> bool { CNTP_TVAL_EL0.set(TICK_INTERVAL); let now = self.monotonic_timestamp().unwrap(); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 2332174f..e45d41ca 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -24,14 +24,18 @@ macro_rules! absolute_address { use cfg_if::cfg_if; use device_api::{ interrupt::{ - ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController, - MessageInterruptController, + ExternalInterruptController, InterruptHandler, IpiDeliveryTarget, IrqOptions, + LocalInterruptController, MessageInterruptController, }, timer::MonotonicTimestampProviderDevice, ResetDevice, }; use kernel_util::{ - mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel}, + mem::{ + address::PhysicalAddress, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + table::EntryLevel, + }, sync::IrqGuard, }; @@ -98,6 +102,7 @@ pub trait Architecture { &self, base: PhysicalAddress, size: usize, + attrs: DeviceMemoryAttributes, ) -> Result; /// Removes the provided mapping from the kernel's translation tables. @@ -339,8 +344,9 @@ fn __physicalize(addr: usize) -> u64 { fn __map_device_pages( base: PhysicalAddress, count: usize, + attrs: DeviceMemoryAttributes, ) -> Result { - unsafe { ARCHITECTURE.map_device_memory(base, count) } + unsafe { ARCHITECTURE.map_device_memory(base, count, attrs) } } #[no_mangle] fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping) { @@ -356,3 +362,28 @@ fn __monotonic_timestamp() -> Result { fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { ARCHITECTURE.message_interrupt_controller() } + +#[no_mangle] +fn __register_global_interrupt( + irq: u32, + options: IrqOptions, + handler: &'static dyn InterruptHandler, +) -> Result<(), Error> { + let intc = ARCHITECTURE.external_interrupt_controller(); + + let irq = { + #[cfg(target_arch = "aarch64")] + { + aarch64::IrqNumber::Shared(irq) + } + #[cfg(target_arch = "x86_64")] + { + x86_64::IrqNumber::Gsi(irq.try_into().unwrap()) + } + }; + + intc.register_irq(irq, options, handler)?; + intc.enable_irq(irq)?; + + Ok(()) +} diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index e0d2ef95..b5a268ab 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -52,7 +52,7 @@ impl Device for SciHandler { } impl InterruptHandler for SciHandler { - fn handle_irq(&self) -> bool { + fn handle_irq(&self, _vector: Option) -> bool { log::trace!("ACPI SCI received"); ACPI_SYSTEM.get().lock().handle_sci(); true diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index a0bd6fe8..585adcac 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -222,7 +222,7 @@ impl ExternalInterruptController for IoApic { let table = self.table.lock(); if let Some(handler) = table.handler(gsi) { - handler.handle_irq(); + handler.handle_irq(None); } else { warnln!("No handler set for GSI #{}", gsi); } @@ -280,7 +280,10 @@ impl IoApic { // }; // let mapping = unsafe { DeviceMemoryMapping::map(base, size) }; let regs = unsafe { - DeviceMemoryIo::<'_, Regs>::map(PhysicalAddress::from_raw(ioapic.address as u64))? + DeviceMemoryIo::<'_, Regs>::map( + PhysicalAddress::from_raw(ioapic.address as u64), + Default::default(), + )? }; let max_gsi = (regs.read(REG_IOAPIC_VERSION) >> 16) & 0xFF; diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index b951b1eb..4f81958e 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -5,8 +5,8 @@ use abi::error::Error; use alloc::{vec, vec::Vec}; use device_api::{ interrupt::{ - InterruptAffinity, IpiDeliveryTarget, LocalInterruptController, MessageInterruptController, - MsiHandler, MsiInfo, + InterruptAffinity, InterruptHandler, IpiDeliveryTarget, LocalInterruptController, + MessageInterruptController, MsiInfo, }, Device, }; @@ -147,7 +147,7 @@ register_structs! { pub struct LocalApic { regs: DeviceMemoryIo<'static, Regs>, id: u32, - msi_vectors: Vec>>, + msi_vectors: Vec>>, } unsafe impl Send for LocalApic {} @@ -172,7 +172,7 @@ impl MessageInterruptController for LocalApic { }; drop(table); - if handler.handle_msi(vector) { + if handler.handle_irq(Some(vector)) { break; } @@ -183,7 +183,7 @@ impl MessageInterruptController for LocalApic { fn register_msi_range( &self, range: &mut [MsiInfo], - handler: &'static dyn MsiHandler, + handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { let _guard = IrqGuard::acquire(); @@ -265,7 +265,7 @@ impl LocalApic { /// /// Only meant to be called once per processor during their init. pub unsafe fn new() -> Self { - let regs = DeviceMemoryIo::::map(Self::base()).unwrap(); + let regs = DeviceMemoryIo::::map(Self::base(), Default::default()).unwrap(); let id = regs.Id.read(Id::ApicId); diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index e131408e..0b4c39c8 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -9,7 +9,7 @@ use abi::error::Error; use kernel_util::{ mem::{ address::{FromRaw, PhysicalAddress}, - device::RawDeviceMemoryMapping, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, table::EntryLevelExt, }, util::OneTimeInit, @@ -114,7 +114,11 @@ unsafe fn unmap_early_page(address: usize) { } // Device mappings -unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result { +unsafe fn map_device_memory_l3( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { // TODO don't map pages if already mapped 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { @@ -142,7 +146,11 @@ unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result Result { +unsafe fn map_device_memory_l2( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { for j in 0..count { if DEVICE_MAPPING_L2[i + j].is_present() { @@ -170,6 +178,7 @@ unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result Result { // debugln!("Map {}B @ {:#x}", size, base); let l3_aligned = base.page_align_down::(); @@ -182,7 +191,7 @@ pub(super) unsafe fn map_device_memory( let l2_offset = base.page_offset::(); let page_count = (l2_offset + size).page_count::(); - let base_address = map_device_memory_l2(l2_aligned, page_count)?; + let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; let address = base_address + l2_offset; Ok(RawDeviceMemoryMapping { @@ -193,7 +202,7 @@ pub(super) unsafe fn map_device_memory( }) } else { // Just map the pages directly - let base_address = map_device_memory_l3(l3_aligned, page_count)?; + let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; let address = base_address + l3_offset; Ok(RawDeviceMemoryMapping { @@ -311,23 +320,6 @@ fn clone_kernel_tables(dst: &mut PageTable) { } } -pub(super) fn translate_kernel_address(address: usize) -> Option { - let l0i = address.page_index::(); - let l1i = address.page_index::(); - // let l2i = address.page_index::(); - // let l3i = address.page_index::(); - - match l0i { - KERNEL_L0_INDEX => match l1i { - HEAP_MAPPING_L1I => Some(PhysicalAddress::from_raw(address - HEAP_MAPPING_OFFSET)), - DEVICE_MAPPING_L1I => todo!(), - _ => todo!(), - }, - RAM_MAPPING_L0I => todo!(), - _ => None, - } -} - /// Sets up the following memory map: /// ...: KERNEL_TABLES.l0: /// * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000 : RAM_MAPPING_L1 diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 049a741d..1a61f38c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -14,7 +14,7 @@ use kernel_fs::devfs; use kernel_util::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, - device::RawDeviceMemoryMapping, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, table::EntryLevelExt, }, sync::SpinFence, @@ -164,8 +164,9 @@ impl Architecture for X86_64 { &self, base: PhysicalAddress, size: usize, + attrs: DeviceMemoryAttributes, ) -> Result { - mem::map_device_memory(base, size) + mem::map_device_memory(base, size, attrs) } #[inline] diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index 3a94e1cc..62530857 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -42,7 +42,7 @@ impl MonotonicTimestampProviderDevice for I8253 { } impl InterruptHandler for I8253 { - fn handle_irq(&self) -> bool { + fn handle_irq(&self, _vector: Option) -> bool { let mut inner = self.inner.lock(); inner.tick += 1; diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index bb650446..ca2b46b9 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -77,7 +77,7 @@ impl Inner { } impl InterruptHandler for PS2Controller { - fn handle_irq(&self) -> bool { + fn handle_irq(&self, _vector: Option) -> bool { let mut count = 0; let mut inner = self.inner.lock(); diff --git a/src/device/bus/dt_pci.rs b/src/device/bus/dt_pci.rs new file mode 100644 index 00000000..4a0da378 --- /dev/null +++ b/src/device/bus/dt_pci.rs @@ -0,0 +1,189 @@ +//! PCI bus device wrapper for device tree + +use alloc::{collections::BTreeMap, vec::Vec}; +use device_api::interrupt::{IrqLevel, IrqOptions, IrqTrigger}; +use kernel_util::mem::address::{FromRaw, PhysicalAddress}; +use ygg_driver_pci::{ + device::{PciInterrupt, PciInterruptPin, PciInterruptRoute}, + PciAddress, PciAddressRange, PciBusManager, PciRangeType, +}; + +use crate::{ + device::devtree::{self, DevTreeIndexNodeExt, DevTreeIndexPropExt, DevTreeNodeInfo}, + device_tree_driver, +}; + +fn extract_ranges(dt: &DevTreeNodeInfo) -> Vec { + let Some(ranges) = devtree::find_prop(&dt.node, "ranges") else { + return Vec::new(); + }; + let pci_address_cells = dt.node.address_cells(); + let pci_size_cells = dt.node.size_cells(); + + let cells_per_range = dt.address_cells + pci_address_cells + pci_size_cells; + + assert_eq!(ranges.len() % cells_per_range, 0); + + let range_count = ranges.len() / (cells_per_range * 4); + + let mut result = Vec::new(); + + for i in 0..range_count { + let ty_bits = ranges.cell1_array_item(i * cells_per_range, 1).unwrap(); + let ty = match (ty_bits >> 24) & 0x3 { + 0 => PciRangeType::Configuration, + 1 => PciRangeType::Io, + 2 => PciRangeType::Memory32, + 3 => PciRangeType::Memory64, + _ => unreachable!(), + }; + let bus_number = (ty_bits >> 16) as u8; + + let pci_base = match pci_address_cells { + 3 => { + let hi = ranges.cell1_array_item(i * cells_per_range + 1, 1).unwrap(); + let lo = ranges.cell1_array_item(i * cells_per_range + 2, 1).unwrap(); + + (hi << 32) | lo + } + _ => unimplemented!(), + }; + + let host_base = PhysicalAddress::from_raw(match dt.address_cells { + 2 => { + let hi = ranges + .cell1_array_item(i * cells_per_range + pci_address_cells, 1) + .unwrap(); + let lo = ranges + .cell1_array_item(i * cells_per_range + pci_address_cells + 1, 1) + .unwrap(); + + (hi << 32) | lo + } + _ => unimplemented!(), + }); + + let size = match pci_size_cells { + 2 => { + let hi = ranges + .cell1_array_item( + i * cells_per_range + pci_address_cells + dt.address_cells, + 1, + ) + .unwrap(); + let lo = ranges + .cell1_array_item( + i * cells_per_range + pci_address_cells + dt.address_cells + 1, + 1, + ) + .unwrap(); + + (hi << 32) | lo + } + _ => unimplemented!(), + } as usize; + + result.push(PciAddressRange { + ty, + bus_number, + host_base, + pci_base, + size, + }); + } + + result +} + +fn extract_interrupt_map(dt: &DevTreeNodeInfo) -> BTreeMap { + // let interrupt_map_mask = devtree::find_prop(&dt.node, "interrupt-map").unwrap(); + let interrupt_map = devtree::find_prop(&dt.node, "interrupt-map").unwrap(); + let pci_address_cells = dt.node.address_cells(); + + // TODO replace 3 with interrupt-cells in interrupt-controller + let cells_per_imap = pci_address_cells + /* Pin */ 1 + /* #interrupt-cells in interrupt-controller */ 3 + /* Interrupt Controller Data */ 3; + + assert_eq!(interrupt_map.len() % (4 * cells_per_imap), 0); + + let mut imap = BTreeMap::new(); + + for i in 0..interrupt_map.len() / (4 * cells_per_imap) { + let pci_address_0 = interrupt_map + .cell1_array_item(i * cells_per_imap, 1) + .unwrap(); + + let bus = (pci_address_0 >> 24) as u8; + let device = ((pci_address_0 >> 11) & 0x1F) as u8; + let function = ((pci_address_0 >> 8) & 0x7) as u8; + + let address = PciAddress::for_function(0, bus, device, function); + + let pin = interrupt_map + .cell1_array_item(i * cells_per_imap + pci_address_cells, 1) + .unwrap() as u32; + + let Ok(pin) = PciInterruptPin::try_from(pin) else { + continue; + }; + + let _interrupt_ty = interrupt_map + .cell1_array_item(i * cells_per_imap + pci_address_cells + 4, 1) + .unwrap(); + let interrupt_number = interrupt_map + .cell1_array_item(i * cells_per_imap + pci_address_cells + 5, 1) + .unwrap(); + let interrupt_mode = interrupt_map + .cell1_array_item(i * cells_per_imap + pci_address_cells + 6, 1) + .unwrap(); + + let (trigger, level) = match interrupt_mode { + 0x04 => (IrqTrigger::Level, IrqLevel::ActiveHigh), + _ => todo!(), + }; + + let src = PciInterrupt { address, pin }; + let dst = PciInterruptRoute { + number: interrupt_number as _, + options: IrqOptions { trigger, level }, + }; + + // TODO use phandle for interrupt-controller + // TODO interrupt-controller-specific decoding of idata + // TODO don't ignore interrupt_ty, don't assume they're all SPIs + imap.insert(src, dst); + } + + imap +} + +device_tree_driver! { + compatible: ["pci-host-ecam-generic"], + probe(dt) => { + let reg = devtree::find_prop(&dt.node, "reg")?; + let bus_range = devtree::find_prop(&dt.node, "bus-range")?; + + let (cfg_space_base, _) = reg + .cell2_array_item(0, dt.address_cells, dt.size_cells) + .unwrap(); + let cfg_space_base = PhysicalAddress::from_raw(cfg_space_base); + + let bus_start = bus_range.cell1_array_item(0, 1)? as u8; + let bus_end = bus_range.cell1_array_item(1, 1)? as u8; + + let ranges = extract_ranges(dt); + let interrupt_map = extract_interrupt_map(dt); + + if ranges.is_empty() { + return None; + } + + PciBusManager::add_segment_from_device_tree( + cfg_space_base, + bus_start..bus_end, + ranges, + interrupt_map + ).ok(); + + None + } +} diff --git a/src/device/bus/mod.rs b/src/device/bus/mod.rs index 42007f9c..cc4d12e2 100644 --- a/src/device/bus/mod.rs +++ b/src/device/bus/mod.rs @@ -1,4 +1,6 @@ //! Bus devices +#[cfg(target_arch = "aarch64")] +pub mod dt_pci; #[cfg(feature = "device-tree")] pub mod simple_bus; diff --git a/src/device/devtree.rs b/src/device/devtree.rs index 19733857..b1fbd89f 100644 --- a/src/device/devtree.rs +++ b/src/device/devtree.rs @@ -88,6 +88,9 @@ pub trait DevTreeIndexPropExt { /// Reads a cell value from the property at given offset fn read_cell(&self, u32_offset: usize, cell_size: usize) -> Option; + + /// Returns the length in bytes + fn len(&self) -> usize; } /// Helper trait to provide extra functionality for [DevTreeIndexNode] @@ -248,6 +251,19 @@ impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { "compatible" | "stdout-path" => { log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) } + "#interrupt-cells" | "#size-cells" | "#address-cells" => { + log_print_raw!(level, "{:#x}", prop.u32(0).unwrap_or(0xFFFFFFFF)); + } + _ if prop.length() % 4 == 0 => { + log_print_raw!(level, "["); + for i in 0..prop.length() / 4 { + if i != 0 { + log_print_raw!(level, ", "); + } + log_print_raw!(level, "{:#010x}", prop.u32(i).unwrap()); + } + log_print_raw!(level, "]"); + } _ => log_print_raw!(level, "{:x?}", prop.raw()), } @@ -287,6 +303,10 @@ impl<'a, 'i, 'dt> DevTreeIndexPropExt for DevTreeIndexProp<'a, 'i, 'dt> { let cell1 = self.read_cell(u32_index + cells0, cells1)?; Some((cell0, cell1)) } + + fn len(&self) -> usize { + self.length() + } } impl<'a> FdtMemoryRegionIter<'a> { diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 0dfe5f41..1b6c67a7 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -9,7 +9,10 @@ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; use kernel_util::{ mem::{ - address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, PageProvider, + address::PhysicalAddress, + device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, + table::EntryLevel, + PageProvider, }, sync::IrqSafeSpinlock, }; @@ -56,7 +59,16 @@ impl LinearFramebuffer { width: u32, height: u32, ) -> Result { - let base = unsafe { RawDeviceMemoryMapping::map(phys_base, size) }?.leak(); + let base = unsafe { + RawDeviceMemoryMapping::map( + phys_base, + size, + DeviceMemoryAttributes { + caching: DeviceMemoryCaching::Cacheable, + }, + ) + }? + .leak(); let inner = Inner { base, diff --git a/src/device/mod.rs b/src/device/mod.rs index 79bccf41..4ba59ed0 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -6,8 +6,6 @@ use kernel_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; #[cfg(target_arch = "aarch64")] pub mod devtree; -// TODO bus device support on aarch64 -#[cfg(not(target_arch = "aarch64"))] pub mod bus; pub mod display; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 4114bb20..262f0e17 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -176,7 +176,7 @@ impl SerialDevice for Pl011 { } impl InterruptHandler for Pl011 { - fn handle_irq(&self) -> bool { + fn handle_irq(&self, _vector: Option) -> bool { let inner = self.inner.get().lock(); inner.regs.ICR.write(ICR::ALL::CLEAR); @@ -214,7 +214,7 @@ impl Device for Pl011 { unsafe fn init(&'static self) -> Result<(), Error> { let mut inner = Pl011Inner { - regs: DeviceMemoryIo::map(self.base)?, + regs: DeviceMemoryIo::map(self.base, Default::default())?, }; inner.init(); diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 307c9cb5..aeb48ead 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -29,7 +29,7 @@ pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; /// The caller must ensure the correct origin of the address, its alignment and that the access is /// properly synchronized. pub unsafe fn read_memory(address: PhysicalAddress) -> T { - let io = DeviceMemoryMapping::map(address, size_of::()).unwrap(); + let io = DeviceMemoryMapping::map(address, size_of::(), Default::default()).unwrap(); let address = io.address(); if address % align_of::() == 0 { @@ -46,7 +46,7 @@ pub unsafe fn read_memory(address: PhysicalAddress) -> T { /// The caller must ensure the correct origin of the address, its alignment and that the access is /// properly synchronized. pub unsafe fn write_memory(address: PhysicalAddress, value: T) { - let io = DeviceMemoryMapping::map(address, size_of::()).unwrap(); + let io = DeviceMemoryMapping::map(address, size_of::(), Default::default()).unwrap(); let address = io.address(); if address % align_of::() == 0 { diff --git a/src/mem/process.rs b/src/mem/process.rs index e238d1e7..474d580a 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -159,7 +159,7 @@ impl Inner { &mut self, address: usize, page_count: usize, - backing: VirtualRangeBacking, + backing: &VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), (usize, Error)> { for i in 0..page_count { @@ -173,6 +173,7 @@ impl Inner { }; if let Err(err) = unsafe { self.table.map_page(virt, phys, attributes) } { + backing.release_page(offset, phys).unwrap(); return Err((i, err)); } } @@ -180,6 +181,31 @@ impl Inner { Ok(()) } + unsafe fn rollback_allocation( + &mut self, + start_pfn: usize, + pages_mapped: usize, + region_size: usize, + ) { + let unmap_range = start_pfn..start_pfn + pages_mapped; + self.allocator + .free(start_pfn, region_size, |origin_pfn, pfn_range, backing| { + for pfn in pfn_range { + if unmap_range.contains(&pfn) { + let offset = (pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE; + let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; + + let phys = self.table.unmap_page(virt)?; + + backing.release_page(offset as u64, phys)?; + } + } + + Ok(()) + }) + .unwrap(); + } + fn map_range( &mut self, address: usize, @@ -192,9 +218,13 @@ impl Inner { self.allocator .insert(start_pfn, page_count, backing.clone())?; - if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { - // TODO rollback & remove the range - todo!(); + if let Err((mapped, error)) = self.try_map_pages(address, page_count, &backing, attributes) + { + debug_assert!(mapped < page_count); + unsafe { + self.rollback_allocation(start_pfn, mapped, page_count); + } + return Err(error); }; Ok(()) @@ -211,15 +241,24 @@ impl Inner { let phys = match backing.get_page(0) { Ok(page) => page, - Err(_err) => { - // TODO rollback - todo!(); + Err(err) => { + // Do nothing, as the page has not been allocated to this range yet + self.allocator.free(start_pfn, 1, |_, _, _| Ok(())).unwrap(); + return Err(err); } }; - if let Err(_err) = unsafe { self.table.map_page(address, phys, attributes) } { - // TODO rollback - todo!(); + if let Err(err) = unsafe { self.table.map_page(address, phys, attributes) } { + self.allocator + .free(start_pfn, 1, |_, _, _| { + // Deallocate the page, but do not unmap, as the mapping failed + unsafe { + phys::free_page(phys); + } + Ok(()) + }) + .unwrap(); + return Err(err); } Ok(phys) @@ -234,9 +273,13 @@ impl Inner { let start_pfn = self.allocator.allocate(page_count, backing.clone())?; let address = start_pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { - // TODO rollback - todo!("alloc_range({})", page_count); + if let Err((mapped, error)) = self.try_map_pages(address, page_count, &backing, attributes) + { + debug_assert!(mapped < page_count); + unsafe { + self.rollback_allocation(start_pfn, mapped, page_count); + } + return Err(error); }; Ok(address) From 7d7bba78b7de6d7feffa7f3febd7a8d7fd8bee62 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 4 Feb 2024 13:15:18 +0200 Subject: [PATCH 174/211] block/ahci: proper AHCI port init --- Cargo.toml | 3 ++- driver/block/ahci/src/lib.rs | 39 +++++++++++++++++++++++++++++------ driver/block/ahci/src/port.rs | 2 ++ src/arch/aarch64/mod.rs | 9 ++++++-- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5433a7a9..ee0702c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,8 @@ ygg_driver_block = { path = "driver/block/core" } ygg_driver_net_core = { path = "driver/net/core" } ygg_driver_net_loopback = { path = "driver/net/loopback" } ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] } +ygg_driver_ahci = { path = "driver/block/ahci" } + kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } @@ -58,7 +60,6 @@ acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } # TODO currently only supported here xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } ygg_driver_nvme = { path = "driver/block/nvme" } -ygg_driver_ahci = { path = "driver/block/ahci" } [features] default = ["fb_console"] diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 53e9cc2a..795aa636 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -5,13 +5,20 @@ extern crate alloc; use alloc::{boxed::Box, format, vec, vec::Vec}; +use bytemuck::Zeroable; +use data::ReceivedFis; use device_api::{ interrupt::{InterruptAffinity, InterruptHandler}, Device, }; use error::AhciError; use kernel_fs::devfs; -use kernel_util::{mem::device::DeviceMemoryIo, runtime, sync::IrqSafeSpinlock, util::OneTimeInit}; +use kernel_util::{ + mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, + runtime, + sync::IrqSafeSpinlock, + util::OneTimeInit, +}; use port::AhciPort; use regs::{PortRegs, Regs}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; @@ -38,6 +45,7 @@ const MAX_DRIVES: usize = (b'z' - b'a') as usize; pub struct AhciController { regs: IrqSafeSpinlock>, ports: OneTimeInit>, + received_fis_buffers: OneTimeInit<[Option>; 16]>, version: Version, max_port_count: usize, @@ -67,6 +75,23 @@ impl AhciController { drop(regs); + let mut fis_buffers = [const { None }; 16]; + // Allocate FIS receive buffers for the ports + for i in 0..self.max_port_count { + if pi & (1 << i) == 0 { + continue; + } + + let regs = self.regs.lock(); + let port = ®s.PORTS[i]; + + let buffer = PageBox::new(ReceivedFis::zeroed()).map_err(AhciError::MemoryError)?; + port.set_received_fis_address_64(unsafe { buffer.as_physical_address() }); + fis_buffers[i] = Some(buffer); + } + + self.received_fis_buffers.init(fis_buffers); + for i in 0..self.max_port_count { if pi & (1 << i) == 0 { continue; @@ -192,6 +217,11 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { let bar5 = info.config_space.bar(5).ok_or(Error::InvalidOperation)?; let bar5 = bar5.as_memory().ok_or(Error::InvalidOperation)?; + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); + cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); + cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; + info.config_space.set_command(cmd.bits()); + info.init_interrupts(PreferredInterruptMode::Msi)?; // // TODO support regular PCI interrupts (ACPI dependency) @@ -202,21 +232,18 @@ pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { // Map the registers let regs = unsafe { DeviceMemoryIo::::map(bar5, Default::default()) }?; + let version = Version::try_from(regs.VS.get())?; let ahci_only = regs.CAP.matches_all(CAP::SAM::SET); let max_port_count = regs.CAP.read(CAP::NP) as usize; let has_64_bit = regs.CAP.matches_all(CAP::S64A::SET); - let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); - cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); - cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; - info.config_space.set_command(cmd.bits()); - // TODO extract Number of Command Slots let ahci = Box::leak(Box::new(AhciController { regs: IrqSafeSpinlock::new(regs), ports: OneTimeInit::new(), + received_fis_buffers: OneTimeInit::new(), version, max_port_count, ahci_only, diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index 8b1da03c..0333faa7 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -75,6 +75,8 @@ impl PortInner { )?; // Sync before send + // XXX do this properly + #[cfg(target_arch = "x86_64")] unsafe { core::arch::asm!("wbinvd"); } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 4df427c6..59ba329a 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -358,6 +358,13 @@ impl AArch64 { 0x1000, ygg_driver_virtio_net::probe, ); + ygg_driver_pci::register_class_driver( + "AHCI SATA Controller", + 0x01, + Some(0x06), + Some(0x01), + ygg_driver_ahci::probe, + ); let dt = self.dt.get(); @@ -389,8 +396,6 @@ impl AArch64 { ); infoln!("Initializing aarch64 platform"); - dt.dump(crate::debug::LogLevel::Debug); - let nodes = dt.root().children(); if let Err(error) = devtree::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { From 335741722968c18116a2c631ac77af6c635a47a7 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 4 Feb 2024 15:18:34 +0200 Subject: [PATCH 175/211] lib/fdt: move device-tree into its own crate --- Cargo.toml | 5 +- lib/device-tree/Cargo.toml | 14 + lib/device-tree/src/driver.rs | 151 +++++++++ .../devtree.rs => lib/device-tree/src/dt.rs | 287 +++--------------- lib/device-tree/src/lib.rs | 10 + lib/kernel-util/src/mem/mod.rs | 1 + lib/kernel-util/src/mem/phys.rs | 36 +++ src/arch/aarch64/gic/mod.rs | 9 +- src/arch/aarch64/mod.rs | 29 +- src/arch/aarch64/smp.rs | 8 +- src/arch/aarch64/timer.rs | 6 +- src/arch/mod.rs | 6 +- src/device/bus/dt_pci.rs | 17 +- src/device/bus/mod.rs | 2 +- src/device/bus/simple_bus.rs | 8 +- src/device/mod.rs | 3 - src/device/power/arm_psci.rs | 16 +- src/device/serial/pl011.rs | 9 +- src/mem/phys/mod.rs | 46 +-- 19 files changed, 314 insertions(+), 349 deletions(-) create mode 100644 lib/device-tree/Cargo.toml create mode 100644 lib/device-tree/src/driver.rs rename src/device/devtree.rs => lib/device-tree/src/dt.rs (58%) create mode 100644 lib/device-tree/src/lib.rs create mode 100644 lib/kernel-util/src/mem/phys.rs diff --git a/Cargo.toml b/Cargo.toml index ee0702c7..ad54aed7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ features = ["no_std_stream"] [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.3.1" -fdt-rs = { version = "0.4.3", default-features = false } +device-tree = { path = "lib/device-tree" } [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } @@ -64,6 +64,3 @@ ygg_driver_nvme = { path = "driver/block/nvme" } [features] default = ["fb_console"] fb_console = [] -aarch64_qemu = [] -aarch64_orange_pi3 = [] -aarch64_rpi3b = [] diff --git a/lib/device-tree/Cargo.toml b/lib/device-tree/Cargo.toml new file mode 100644 index 00000000..322112ce --- /dev/null +++ b/lib/device-tree/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "device-tree" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../device-api", features = ["derive"] } +kernel-util = { path = "../kernel-util" } + +fdt-rs = { version = "0.4.3", default-features = false } +log = "0.4.20" diff --git a/lib/device-tree/src/driver.rs b/lib/device-tree/src/driver.rs new file mode 100644 index 00000000..40f81f37 --- /dev/null +++ b/lib/device-tree/src/driver.rs @@ -0,0 +1,151 @@ +//! Device tree-based driver definitions + +use core::mem::size_of; + +use alloc::boxed::Box; +use device_api::{Device, DeviceId}; +use fdt_rs::index::DevTreeIndexNode; +use yggdrasil_abi::error::Error; + +use crate::dt::{DevTreeIndexNodePropGet, DevTreeNodeInfo}; + +/// Helper macro to return the count of expressions supplied to it +#[macro_export] +macro_rules! count { + () => (0usize); + ($x:tt $($xs:tt)*) => (1usize + $crate::count!($($xs)*)); +} + +/// Registers a device driver for compatible device tree nodes +/// +/// # Usage example +/// +/// ``` +/// device_tree_driver! { +/// compatible: ["arm,pl011"], +/// probe(of) => { +/// let my_device = ...; // ... extract some info about the device ... +/// Some(Box::new(my_device)) +/// } +/// } +/// ``` +#[macro_export] +macro_rules! device_tree_driver { + ( + compatible: [$($compatible:literal),+], + probe ($node:ident) => $probe_body:block $(,)? + ) => { + const __COMPATIBLE_LEN: usize = $crate::count!($($compatible )+); + static __COMPATIBLE: [&str; __COMPATIBLE_LEN] = [$($compatible),+]; + + fn __probe($node: &$crate::dt::DevTreeNodeInfo) -> + Option> $probe_body + + core::arch::global_asm!(r#" + .pushsection .dt_probes, "a" + .quad {compatible} + .quad {compatible_len} + .quad {probe_func} + .popsection + "#, + compatible = sym __COMPATIBLE, + compatible_len = const __COMPATIBLE_LEN, + probe_func = sym __probe + ); + }; +} + +struct DevTreeProbe<'a> { + compatible: &'static [&'static str], + probe_func: fn(&'a DevTreeNodeInfo<'a, 'a, 'a>) -> Option>, +} + +fn iter_dt_probes<'a>() -> impl Iterator> { + extern "C" { + static __dt_probes_start: u64; + static __dt_probes_end: u64; + } + + unsafe { + let base = &__dt_probes_start as *const u64; + let end = &__dt_probes_end as *const u64; + let len = (end as usize - base as usize) / (size_of::() * 3); + + (0..len).map(move |i| { + let compatible_ptr = *base.add(i * 3); + let compatible_len = *base.add(i * 3 + 1); + let probe_func_ptr = *base.add(i * 3 + 2); + + let compatible = + core::slice::from_raw_parts(compatible_ptr as *const &str, compatible_len as usize); + let probe_func = core::mem::transmute(probe_func_ptr); + + DevTreeProbe { + compatible, + probe_func, + } + }) + } +} + +fn dt_match_compatible(compatible: &str) -> Option { + iter_dt_probes().find(|probe| probe.compatible.contains(&compatible)) +} + +/// "Probes" a device tree node for any matching device, registering it if a compatible driver is +/// found +pub fn probe_dt_node DeviceId>( + dt: &DevTreeNodeInfo, + register: F, +) -> Option<(&'static dyn Device, DeviceId)> { + // TODO use list, not just the first item + let Some(compatible) = dt.node.prop("compatible") else { + return None; + }; + + let probe = dt_match_compatible(compatible)?; + let device = Box::leak((probe.probe_func)(dt)?); + let id = register(device); + Some((device, id)) +} + +/// Performs shallow walk of a device tree node and executes the visitor function on each node +pub fn enumerate_dt< + 'a, + I: Iterator>, + F: Fn(&str, DevTreeNodeInfo) -> Result<(), Error>, +>( + address_cells: usize, + size_cells: usize, + nodes: I, + f: F, +) -> Result<(), usize> { + let mut failed_count = 0; + + for node in nodes { + // Skip /cpus and /memory* + let probe = DevTreeNodeInfo { + address_cells, + size_cells, + node, + }; + + let Ok(name) = probe.node.name() else { + continue; + }; + let Some(compatible) = probe.node.prop("compatible") else { + continue; + }; + + if let Err(error) = f(compatible, probe) { + log::warn!("{}: {:?}", name, error); + failed_count += 1; + } + } + + if failed_count == 0 { + Ok(()) + } else { + Err(failed_count) + } +} diff --git a/src/device/devtree.rs b/lib/device-tree/src/dt.rs similarity index 58% rename from src/device/devtree.rs rename to lib/device-tree/src/dt.rs index b1fbd89f..dc96b311 100644 --- a/src/device/devtree.rs +++ b/lib/device-tree/src/dt.rs @@ -1,65 +1,15 @@ //! ARM device tree utlities -use core::mem::size_of; -use abi::error::Error; -use alloc::boxed::Box; -use device_api::{Device, DeviceId}; use fdt_rs::{ base::DevTree, index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, prelude::PropReader, }; -use kernel_util::mem::address::{FromRaw, PhysicalAddress}; - -use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; - -use super::register_device; - -/// Helper macro to return the count of expressions supplied to it -#[macro_export] -macro_rules! count { - () => (0usize); - ($x:tt $($xs:tt)*) => (1usize + $crate::count!($($xs)*)); -} - -/// Registers a device driver for compatible device tree nodes -/// -/// # Usage example -/// -/// ``` -/// device_tree_driver! { -/// compatible: ["arm,pl011"], -/// probe(of) => { -/// let my_device = ...; // ... extract some info about the device ... -/// Some(Box::new(my_device)) -/// } -/// } -/// ``` -#[macro_export] -macro_rules! device_tree_driver { - ( - compatible: [$($compatible:literal),+], - probe ($node:ident) => $probe_body:block $(,)? - ) => { - const __COMPATIBLE_LEN: usize = $crate::count!($($compatible )+); - static __COMPATIBLE: [&str; __COMPATIBLE_LEN] = [$($compatible),+]; - - fn __probe($node: &$crate::device::devtree::DevTreeNodeInfo) -> - Option> $probe_body - - core::arch::global_asm!(r#" - .pushsection .dt_probes, "a" - .quad {compatible} - .quad {compatible_len} - .quad {probe_func} - .popsection - "#, - compatible = sym __COMPATIBLE, - compatible_len = const __COMPATIBLE_LEN, - probe_func = sym __probe - ); - }; -} +use kernel_util::mem::{ + address::{FromRaw, PhysicalAddress}, + phys::PhysicalMemoryRegion, +}; +use yggdrasil_abi::error::Error; const INDEX_BUFFER_SIZE: usize = 65536; @@ -110,9 +60,6 @@ pub trait DevTreeIndexNodeExt { fn get_address_cells(&self) -> Option; /// Returns the #size-cells property of the node, if there is one fn get_size_cells(&self) -> Option; - - /// Prints the node contents to debug output - fn dump(&self, level: LogLevel, depth: usize); } /// Extension trait for [DevTreeIndexNode] to obtain typed property values @@ -121,30 +68,6 @@ pub trait DevTreeIndexNodePropGet { fn prop(&self, name: &str) -> Option; } -impl<'a, 'i, 'dt> DevTreeIndexNodePropGet for DevTreeIndexNode<'a, 'i, 'dt> { - fn prop(&self, name: &str) -> Option { - self.props().find_map(|prop| { - if prop.name().ok()? == name { - prop.u32(0).ok() - } else { - None - } - }) - } -} - -impl<'a, 'i, 'dt> DevTreeIndexNodePropGet<&'a str> for DevTreeIndexNode<'a, 'i, 'dt> { - fn prop(&self, name: &str) -> Option<&'a str> { - self.props().find_map(|prop| { - if prop.name().ok()? == name { - prop.str().ok() - } else { - None - } - }) - } -} - /// Iterator for physical memory regions present in the device tree #[derive(Clone)] pub struct FdtMemoryRegionIter<'a> { @@ -159,7 +82,19 @@ pub struct DeviceTree<'a> { index: DevTreeIndex<'a, 'a>, } +/// Provides information about a device tree node to device driver's "probe" function +pub struct DevTreeNodeInfo<'a, 'i, 'dt> { + /// #address-cells property of the parent bus/system + pub address_cells: usize, + /// #size-cells property of the parent bus/system + pub size_cells: usize, + /// Device tree node being probed + pub node: DevTreeIndexNode<'a, 'i, 'dt>, +} + impl<'a> DeviceTree<'a> { + pub const MIN_HEADER_SIZE: usize = DevTree::MIN_HEADER_SIZE; + /// Constructs a device tree wrapper from the DTB virtual address. /// /// # Safety @@ -177,11 +112,6 @@ impl<'a> DeviceTree<'a> { find_node(self.index.root(), path.trim_start_matches('/')) } - /// Prints the device tree to log output - pub fn dump(&self, level: LogLevel) { - self.index.root().dump(level, 0) - } - /// Returns the total size of the device tree in memory pub fn size(&self) -> usize { self.tree.totalsize() @@ -211,6 +141,11 @@ impl<'a> DeviceTree<'a> { let chosen = self.node_by_path("/chosen")?; chosen.prop("stdout-path") } + + /// Returns the length of the header provided as a slice of bytes + pub unsafe fn read_totalsize(header: &[u8]) -> Result { + DevTree::read_totalsize(header).map_err(|_| Error::InvalidArgument) + } } impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { @@ -225,58 +160,6 @@ impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { .find(|p| p.name().unwrap_or("") == "#size-cells") .map(|p| p.u32(0).unwrap() as usize) } - - fn dump(&self, level: LogLevel, depth: usize) { - fn indent(level: LogLevel, depth: usize) { - for _ in 0..depth { - log_print_raw!(level, " "); - } - } - - let node_name = self.name().unwrap(); - - // Don't dump these - if node_name.starts_with("virtio_mmio@") { - return; - } - - indent(level, depth); - log_print_raw!(level, "{:?} {{\n", node_name); - for prop in self.props() { - indent(level, depth + 1); - let name = prop.name().unwrap_or(""); - log_print_raw!(level, "{name:?} = "); - - match name { - "compatible" | "stdout-path" => { - log_print_raw!(level, "{:?}", prop.str().unwrap_or("")) - } - "#interrupt-cells" | "#size-cells" | "#address-cells" => { - log_print_raw!(level, "{:#x}", prop.u32(0).unwrap_or(0xFFFFFFFF)); - } - _ if prop.length() % 4 == 0 => { - log_print_raw!(level, "["); - for i in 0..prop.length() / 4 { - if i != 0 { - log_print_raw!(level, ", "); - } - log_print_raw!(level, "{:#010x}", prop.u32(i).unwrap()); - } - log_print_raw!(level, "]"); - } - _ => log_print_raw!(level, "{:x?}", prop.raw()), - } - - log_print_raw!(level, "\n"); - } - - for child in self.children() { - child.dump(level, depth + 1); - } - - indent(level, depth); - log_print_raw!(level, "}}\n"); - } } impl<'a, 'i, 'dt> DevTreeIndexPropExt for DevTreeIndexProp<'a, 'i, 'dt> { @@ -353,6 +236,30 @@ impl Iterator for FdtMemoryRegionIter<'_> { } } +impl<'a, 'i, 'dt> DevTreeIndexNodePropGet for DevTreeIndexNode<'a, 'i, 'dt> { + fn prop(&self, name: &str) -> Option { + self.props().find_map(|prop| { + if prop.name().ok()? == name { + prop.u32(0).ok() + } else { + None + } + }) + } +} + +impl<'a, 'i, 'dt> DevTreeIndexNodePropGet<&'a str> for DevTreeIndexNode<'a, 'i, 'dt> { + fn prop(&self, name: &str) -> Option<&'a str> { + self.props().find_map(|prop| { + if prop.name().ok()? == name { + prop.str().ok() + } else { + None + } + }) + } +} + /// Looks up a property with given name in the node pub fn find_prop<'a>(node: &TNode<'a>, name: &str) -> Option> { node.props().find(|p| p.name().unwrap_or("") == name) @@ -380,105 +287,3 @@ fn find_node<'a>(at: TNode<'a>, path: &str) -> Option> { } } } - -struct DevTreeProbe<'a> { - compatible: &'static [&'static str], - probe_func: fn(&'a DevTreeNodeInfo<'a, 'a, 'a>) -> Option>, -} - -/// Provides information about a device tree node to device driver's "probe" function -pub struct DevTreeNodeInfo<'a, 'i, 'dt> { - /// #address-cells property of the parent bus/system - pub address_cells: usize, - /// #size-cells property of the parent bus/system - pub size_cells: usize, - /// Device tree node being probed - pub node: DevTreeIndexNode<'a, 'i, 'dt>, -} - -fn iter_dt_probes<'a>() -> impl Iterator> { - extern "C" { - static __dt_probes_start: u64; - static __dt_probes_end: u64; - } - - unsafe { - let base = &__dt_probes_start as *const u64; - let end = &__dt_probes_end as *const u64; - let len = (end as usize - base as usize) / (size_of::() * 3); - - (0..len).map(move |i| { - let compatible_ptr = *base.add(i * 3); - let compatible_len = *base.add(i * 3 + 1); - let probe_func_ptr = *base.add(i * 3 + 2); - - let compatible = - core::slice::from_raw_parts(compatible_ptr as *const &str, compatible_len as usize); - let probe_func = core::mem::transmute(probe_func_ptr); - - DevTreeProbe { - compatible, - probe_func, - } - }) - } -} - -fn dt_match_compatible(compatible: &str) -> Option { - iter_dt_probes().find(|probe| probe.compatible.contains(&compatible)) -} - -/// "Probes" a device tree node for any matching device, registering it if a compatible driver is -/// found -pub fn probe_dt_node(dt: &DevTreeNodeInfo) -> Option<(&'static dyn Device, DeviceId)> { - // TODO use list, not just the first item - let Some(compatible) = dt.node.prop("compatible") else { - return None; - }; - - let probe = dt_match_compatible(compatible)?; - let device = Box::leak((probe.probe_func)(dt)?); - let id = register_device(device); - Some((device, id)) -} - -/// Performs shallow walk of a device tree node and executes the visitor function on each node -pub fn enumerate_dt< - 'a, - I: Iterator>, - F: Fn(&str, DevTreeNodeInfo) -> Result<(), Error>, ->( - address_cells: usize, - size_cells: usize, - nodes: I, - f: F, -) -> Result<(), usize> { - let mut failed_count = 0; - - for node in nodes { - // Skip /cpus and /memory* - let probe = DevTreeNodeInfo { - address_cells, - size_cells, - node, - }; - - let Ok(name) = probe.node.name() else { - continue; - }; - let Some(compatible) = probe.node.prop("compatible") else { - continue; - }; - - if let Err(error) = f(compatible, probe) { - warnln!("{}: {:?}", name, error); - failed_count += 1; - } - } - - if failed_count == 0 { - Ok(()) - } else { - Err(failed_count) - } -} diff --git a/lib/device-tree/src/lib.rs b/lib/device-tree/src/lib.rs new file mode 100644 index 00000000..6d90ee0c --- /dev/null +++ b/lib/device-tree/src/lib.rs @@ -0,0 +1,10 @@ +#![no_std] + +extern crate alloc; + +#[macro_use] +pub mod driver; + +pub mod dt; + +pub use dt::find_prop; diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index e43a63c1..3f439981 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -13,6 +13,7 @@ use self::address::{AsPhysicalAddress, PhysicalAddress}; pub mod address; pub mod device; +pub mod phys; pub mod pointer; pub mod table; diff --git a/lib/kernel-util/src/mem/phys.rs b/lib/kernel-util/src/mem/phys.rs new file mode 100644 index 00000000..8177323a --- /dev/null +++ b/lib/kernel-util/src/mem/phys.rs @@ -0,0 +1,36 @@ +use core::ops::Range; + +use super::address::PhysicalAddress; + +/// Defines an usable memory region +#[derive(Clone, Copy, Debug)] +pub struct PhysicalMemoryRegion { + /// Start of the region + pub base: PhysicalAddress, + /// Length of the region + pub size: usize, +} + +impl PhysicalMemoryRegion { + /// Returns the end address of the region + pub const fn end(&self) -> PhysicalAddress { + self.base.add(self.size) + } + + /// Returns an address range covered by the region + pub fn range(&self) -> Range { + self.base..self.end() + } + + /// Constrains the [PhysicalMemoryRegion] to global memory limits set in the kernel + pub fn clamp(self, limit: PhysicalAddress) -> Option<(PhysicalAddress, PhysicalAddress)> { + let start = self.base.min(limit); + let end = self.end().min(limit); + + if start < end { + Some((start, end)) + } else { + None + } + } +} diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index bddf7500..4c524dfe 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -13,6 +13,7 @@ use device_api::{ }, Device, }; +use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use kernel_util::{ mem::{ address::{FromRaw, PhysicalAddress}, @@ -22,11 +23,7 @@ use kernel_util::{ util::OneTimeInit, }; -use crate::{ - arch::{aarch64::IrqNumber, Architecture, CpuAccess, CpuMessage}, - device::devtree::{self, DevTreeIndexPropExt}, - device_tree_driver, -}; +use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, CpuMessage}; use self::{gicc::Gicc, gicd::Gicd}; @@ -228,7 +225,7 @@ impl GicPerCpu { device_tree_driver! { compatible: ["arm,cortex-a15-gic", "arm,gic-400"], probe(dt) => { - let reg = devtree::find_prop(&dt.node, "reg")?; + let reg = device_tree::find_prop(&dt.node, "reg")?; let (gicc_base, _) = reg.cell2_array_item(0, dt.address_cells, dt.size_cells)?; let (gicd_base, _) = reg.cell2_array_item(1, dt.address_cells, dt.size_cells)?; diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 59ba329a..5ef05811 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -14,12 +14,13 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use fdt_rs::base::DevTree; +use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; use git_version::git_version; use kernel_util::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + phys::PhysicalMemoryRegion, pointer::PhysicalRef, table::{EntryLevel, EntryLevelExt}, }, @@ -34,15 +35,11 @@ use crate::{ mem::table::{L2, L3}, }, debug, - device::{ - self, - devtree::{self, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}, - power::arm_psci::Psci, - }, + device::{self, power::arm_psci::Psci}, fs::{Initrd, INITRD_DATA}, mem::{ heap, - phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, + phys::{self, reserved::reserve_region}, }, }; @@ -253,8 +250,8 @@ impl AArch64 { dt: &DeviceTree, ) -> Option<(PhysicalAddress, PhysicalAddress)> { let chosen = dt.node_by_path("/chosen")?; - let initrd_start = devtree::find_prop(&chosen, "linux,initrd-start")?; - let initrd_end = devtree::find_prop(&chosen, "linux,initrd-end")?; + let initrd_start = device_tree::find_prop(&chosen, "linux,initrd-start")?; + let initrd_end = device_tree::find_prop(&chosen, "linux,initrd-end")?; let address_cells = dt.address_cells(); @@ -276,8 +273,8 @@ impl AArch64 { // Extract the size of the device tree let dtb_size = { - let dtb_header = EarlyMapping::::map_slice(dtb, DevTree::MIN_HEADER_SIZE)?; - DevTree::read_totalsize(dtb_header.as_ref()).unwrap() + let dtb_header = EarlyMapping::::map_slice(dtb, DeviceTree::MIN_HEADER_SIZE)?; + DeviceTree::read_totalsize(dtb_header.as_ref()).unwrap() }; reserve_region( @@ -382,7 +379,9 @@ impl AArch64 { node, }; - if let Some((device, _)) = devtree::probe_dt_node(&probe) { + if let Some((device, _)) = + device_tree::driver::probe_dt_node(&probe, device::register_device) + { device.init()?; } } @@ -398,7 +397,7 @@ impl AArch64 { let nodes = dt.root().children(); if let Err(error) = - devtree::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { + device_tree::driver::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { // Skip chosen-stdout, already initialized if let Some(ref chosen_stdout) = chosen_stdout && chosen_stdout.name() == probe.node.name() @@ -406,7 +405,9 @@ impl AArch64 { return Ok(()); } - if let Some((device, _)) = devtree::probe_dt_node(&probe) { + if let Some((device, _)) = + device_tree::driver::probe_dt_node(&probe, device::register_device) + { device.init()?; } diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index 9196062f..f3144442 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -3,13 +3,11 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use abi::error::Error; use device_api::CpuBringupDevice; -use fdt_rs::prelude::PropReader; +use device_tree::dt::{DevTreeIndexNodePropGet, DeviceTree}; use crate::arch::ARCHITECTURE; use crate::mem::KERNEL_VIRT_OFFSET; -use crate::device::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; - use super::{BootStack, BOOT_STACK_SIZE}; #[derive(Debug)] @@ -32,8 +30,8 @@ fn enumerate_cpus<'a>(dt: &'a DeviceTree) -> impl Iterator> { let cpus = dt.node_by_path("/cpus").unwrap(); cpus.children().filter_map(|cpu_node| { - let compatible = devtree::find_prop(&cpu_node, "compatible").and_then(|p| p.str().ok())?; - let id = cpu_node.prop("reg")?; // devtree::find_prop(&cpu_node, "reg").and_then(|p| p.u32(0).ok())?; + let compatible = cpu_node.prop("compatible")?; + let id = cpu_node.prop("reg")?; let enable_method_str: &str = cpu_node.prop("enable-method")?; let enable_method = match enable_method_str { "psci" => CpuEnableMethod::Psci, diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index c3625c99..734f012f 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -6,13 +6,11 @@ use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0 use abi::error::Error; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; +use device_tree::device_tree_driver; use kernel_util::runtime; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use crate::{ - arch::{aarch64::IrqNumber, Architecture, CpuAccess, ARCHITECTURE}, - device_tree_driver, -}; +use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, ARCHITECTURE}; use super::cpu::Cpu; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index e45d41ca..2fc43bbc 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -34,15 +34,13 @@ use kernel_util::{ mem::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + phys::PhysicalMemoryRegion, table::EntryLevel, }, sync::IrqGuard, }; -use crate::{ - mem::phys::PhysicalMemoryRegion, - task::{sched::CpuQueue, thread::ThreadId, Cpu}, -}; +use crate::task::{sched::CpuQueue, thread::ThreadId, Cpu}; cfg_if! { if #[cfg(target_arch = "aarch64")] { diff --git a/src/device/bus/dt_pci.rs b/src/device/bus/dt_pci.rs index 4a0da378..9f042721 100644 --- a/src/device/bus/dt_pci.rs +++ b/src/device/bus/dt_pci.rs @@ -2,19 +2,18 @@ use alloc::{collections::BTreeMap, vec::Vec}; use device_api::interrupt::{IrqLevel, IrqOptions, IrqTrigger}; +use device_tree::{ + device_tree_driver, + dt::{self, DevTreeIndexNodeExt, DevTreeIndexPropExt, DevTreeNodeInfo}, +}; use kernel_util::mem::address::{FromRaw, PhysicalAddress}; use ygg_driver_pci::{ device::{PciInterrupt, PciInterruptPin, PciInterruptRoute}, PciAddress, PciAddressRange, PciBusManager, PciRangeType, }; -use crate::{ - device::devtree::{self, DevTreeIndexNodeExt, DevTreeIndexPropExt, DevTreeNodeInfo}, - device_tree_driver, -}; - fn extract_ranges(dt: &DevTreeNodeInfo) -> Vec { - let Some(ranges) = devtree::find_prop(&dt.node, "ranges") else { + let Some(ranges) = dt::find_prop(&dt.node, "ranges") else { return Vec::new(); }; let pci_address_cells = dt.node.address_cells(); @@ -97,7 +96,7 @@ fn extract_ranges(dt: &DevTreeNodeInfo) -> Vec { fn extract_interrupt_map(dt: &DevTreeNodeInfo) -> BTreeMap { // let interrupt_map_mask = devtree::find_prop(&dt.node, "interrupt-map").unwrap(); - let interrupt_map = devtree::find_prop(&dt.node, "interrupt-map").unwrap(); + let interrupt_map = dt::find_prop(&dt.node, "interrupt-map").unwrap(); let pci_address_cells = dt.node.address_cells(); // TODO replace 3 with interrupt-cells in interrupt-controller @@ -159,8 +158,8 @@ fn extract_interrupt_map(dt: &DevTreeNodeInfo) -> BTreeMap { - let reg = devtree::find_prop(&dt.node, "reg")?; - let bus_range = devtree::find_prop(&dt.node, "bus-range")?; + let reg = dt::find_prop(&dt.node, "reg")?; + let bus_range = dt::find_prop(&dt.node, "bus-range")?; let (cfg_space_base, _) = reg .cell2_array_item(0, dt.address_cells, dt.size_cells) diff --git a/src/device/bus/mod.rs b/src/device/bus/mod.rs index cc4d12e2..a71aedfb 100644 --- a/src/device/bus/mod.rs +++ b/src/device/bus/mod.rs @@ -2,5 +2,5 @@ #[cfg(target_arch = "aarch64")] pub mod dt_pci; -#[cfg(feature = "device-tree")] +#[cfg(target_arch = "aarch64")] pub mod simple_bus; diff --git a/src/device/bus/simple_bus.rs b/src/device/bus/simple_bus.rs index 9fb1574c..ce045bbf 100644 --- a/src/device/bus/simple_bus.rs +++ b/src/device/bus/simple_bus.rs @@ -1,6 +1,8 @@ //! Simple "passthrough" bus device -use crate::{device::devtree, device_tree_driver}; +use device_tree::{device_tree_driver, dt::DevTreeIndexNodeExt}; + +use crate::device; device_tree_driver! { compatible: ["simple-bus"], @@ -11,8 +13,8 @@ device_tree_driver! { let nodes = dt.node.children(); // Iterate devices on the bus - devtree::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { - if let Some((device, _)) = devtree::probe_dt_node(&probe) { + device_tree::driver::enumerate_dt(address_cells, size_cells, nodes, |_, probe| { + if let Some((device, _)) = device_tree::driver::probe_dt_node(&probe, device::register_device) { unsafe { device.init()?; } diff --git a/src/device/mod.rs b/src/device/mod.rs index 4ba59ed0..9e04c40b 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -3,9 +3,6 @@ use device_api::{manager::DeviceManager, Device, DeviceId}; use kernel_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; -#[cfg(target_arch = "aarch64")] -pub mod devtree; - pub mod bus; pub mod display; diff --git a/src/device/power/arm_psci.rs b/src/device/power/arm_psci.rs index 4dce90a3..ca578752 100644 --- a/src/device/power/arm_psci.rs +++ b/src/device/power/arm_psci.rs @@ -3,13 +3,9 @@ use abi::error::Error; use alloc::boxed::Box; use device_api::{CpuBringupDevice, Device, ResetDevice}; -use fdt_rs::prelude::PropReader; +use device_tree::{device_tree_driver, dt::DevTreeIndexNodePropGet}; -use crate::{ - arch::{Architecture, ArchitectureImpl, ARCHITECTURE}, - device::devtree, - device_tree_driver, -}; +use crate::arch::{Architecture, ArchitectureImpl, ARCHITECTURE}; enum CallMethod { Hvc, @@ -76,15 +72,15 @@ impl Psci { device_tree_driver! { compatible: ["arm,psci-1.0", "arm,psci"], probe(dt) => { - let method = devtree::find_prop(&dt.node, "method").and_then(|prop| prop.str().ok())?; + let method: &str = dt.node.prop("method")?; let method = match method { "hvc" => CallMethod::Hvc, "smc" => CallMethod::Smc, _ => panic!("Unknown PSCI call method: {:?}", method) }; - let cpu_on = devtree::find_prop(&dt.node, "cpu_on").and_then(|prop| prop.u32(0).ok())?; - let cpu_off = devtree::find_prop(&dt.node, "cpu_off").and_then(|prop| prop.u32(0).ok())?; - let cpu_suspend = devtree::find_prop(&dt.node, "cpu_suspend").and_then(|prop| prop.u32(0).ok())?; + let cpu_on = dt.node.prop("cpu_on")?; + let cpu_off = dt.node.prop("cpu_off")?; + let cpu_suspend = dt.node.prop("cpu_suspend")?; Some(Box::new(Psci { method, diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 262f0e17..b74b088c 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -2,6 +2,7 @@ use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; +use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use futures_util::task::{Context, Poll}; use kernel_fs::devfs::{self, CharDeviceType}; use kernel_util::{ @@ -23,11 +24,7 @@ use vfs::{CharDevice, FileReadiness}; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, debug::{self, DebugSink, LogLevel}, - device::{ - devtree::{self, DevTreeIndexPropExt}, - tty::{TtyContext, TtyDevice}, - }, - device_tree_driver, + device::tty::{TtyContext, TtyDevice}, task::process::ProcessId, }; @@ -240,7 +237,7 @@ impl Device for Pl011 { device_tree_driver! { compatible: ["arm,pl011"], probe(of) => { - let reg = devtree::find_prop(&of.node, "reg")?; + let reg = device_tree::find_prop(&of.node, "reg")?; let (base, _) = reg.cell2_array_item(0, of.address_cells, of.size_cells)?; Some(Box::new(Pl011 { diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 07046f77..ef6d62a5 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,10 +1,11 @@ //! Physical memory management utilities -use core::ops::Range; - use abi::{error::Error, system::SystemMemoryStats}; use kernel_util::{ - mem::address::{FromRaw, IntoRaw, PhysicalAddress}, + mem::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + phys::PhysicalMemoryRegion, + }, sync::IrqSafeSpinlock, util::OneTimeInit, }; @@ -25,39 +26,6 @@ const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PA mod manager; pub mod reserved; -/// Defines an usable memory region -#[derive(Clone, Copy, Debug)] -pub struct PhysicalMemoryRegion { - /// Start of the region - pub base: PhysicalAddress, - /// Length of the region - pub size: usize, -} - -impl PhysicalMemoryRegion { - /// Returns the end address of the region - pub const fn end(&self) -> PhysicalAddress { - self.base.add(self.size) - } - - /// Returns an address range covered by the region - pub fn range(&self) -> Range { - self.base..self.end() - } - - /// Constrains the [PhysicalMemoryRegion] to global memory limits set in the kernel - pub fn clamp(self) -> Option<(PhysicalAddress, PhysicalAddress)> { - let start = self.base.min(MEMORY_UPPER_LIMIT); - let end = self.end().min(MEMORY_UPPER_LIMIT); - - if start < end { - Some((start, end)) - } else { - None - } - } -} - /// Global physical memory manager pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); @@ -97,7 +65,7 @@ fn physical_memory_range>( let mut start = PhysicalAddress::MAX; let mut end = PhysicalAddress::MIN; - for (reg_start, reg_end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { + for (reg_start, reg_end) in it.into_iter().filter_map(|r| r.clamp(MEMORY_UPPER_LIMIT)) { if reg_start < start { start = reg_start; } @@ -118,7 +86,7 @@ pub fn find_contiguous_region>( it: I, count: usize, ) -> Option { - for (reg_start, reg_end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { + for (reg_start, reg_end) in it.into_iter().filter_map(|r| r.clamp(MEMORY_UPPER_LIMIT)) { let mut collected = 0; let mut base_addr = None; @@ -182,7 +150,7 @@ pub unsafe fn init_from_iter + Clone>( let mut collected = 0; const MAX_MEMORY: usize = 64 * 1024; - for (start, end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) { + for (start, end) in it.into_iter().filter_map(|r| r.clamp(MEMORY_UPPER_LIMIT)) { for page in (start..end).step_by(0x1000) { if collected >= MAX_MEMORY { break; From aa223a4c455617136e010fe41e16632f3fd2d5f9 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 4 Feb 2024 21:13:40 +0200 Subject: [PATCH 176/211] libk: rename kernel-util to libk --- Cargo.toml | 2 +- driver/block/ahci/Cargo.toml | 2 +- driver/block/ahci/src/command.rs | 2 +- driver/block/ahci/src/data.rs | 2 +- driver/block/ahci/src/lib.rs | 2 +- driver/block/ahci/src/port.rs | 2 +- driver/block/ahci/src/regs.rs | 2 +- driver/block/core/Cargo.toml | 2 +- driver/block/core/src/device.rs | 2 +- driver/block/core/src/lib.rs | 2 +- driver/block/core/src/partition.rs | 2 +- driver/block/core/src/request.rs | 2 +- driver/block/nvme/Cargo.toml | 2 +- driver/block/nvme/src/command.rs | 2 +- driver/block/nvme/src/drive.rs | 2 +- driver/block/nvme/src/lib.rs | 2 +- driver/block/nvme/src/queue.rs | 2 +- driver/bus/pci/Cargo.toml | 2 +- driver/bus/pci/src/capability.rs | 2 +- driver/bus/pci/src/device.rs | 2 +- driver/bus/pci/src/lib.rs | 2 +- driver/bus/pci/src/space/ecam.rs | 2 +- driver/fs/kernel-fs/Cargo.toml | 2 +- driver/fs/kernel-fs/src/devfs.rs | 2 +- driver/fs/memfs/Cargo.toml | 2 +- driver/fs/memfs/src/file.rs | 2 +- driver/net/core/Cargo.toml | 2 +- driver/net/core/src/ethernet.rs | 2 +- driver/net/core/src/interface.rs | 2 +- driver/net/core/src/l3/arp.rs | 2 +- driver/net/core/src/l3/mod.rs | 2 +- driver/net/core/src/l4/tcp.rs | 2 +- driver/net/core/src/lib.rs | 2 +- driver/net/core/src/socket.rs | 2 +- driver/net/loopback/Cargo.toml | 2 +- driver/net/loopback/src/lib.rs | 2 +- driver/virtio/core/Cargo.toml | 2 +- driver/virtio/core/src/queue.rs | 2 +- driver/virtio/core/src/transport/mod.rs | 2 +- driver/virtio/core/src/transport/pci.rs | 2 +- driver/virtio/net/Cargo.toml | 2 +- driver/virtio/net/src/lib.rs | 2 +- lib/device-tree/Cargo.toml | 2 +- lib/device-tree/src/dt.rs | 2 +- lib/vfs/Cargo.toml | 2 +- lib/vfs/src/channel.rs | 2 +- lib/vfs/src/file/device.rs | 2 +- lib/vfs/src/file/directory.rs | 2 +- lib/vfs/src/file/mod.rs | 4 ++-- lib/vfs/src/file/pipe.rs | 2 +- lib/vfs/src/file/regular.rs | 2 +- lib/vfs/src/node/impls.rs | 2 +- lib/vfs/src/node/mod.rs | 2 +- lib/vfs/src/poll.rs | 2 +- lib/vfs/src/pty.rs | 2 +- lib/vfs/src/shared_memory.rs | 2 +- lib/vfs/src/timer.rs | 2 +- {lib/kernel-util => libk}/Cargo.toml | 4 ++-- {lib/kernel-util => libk}/src/api.rs | 0 {lib/kernel-util => libk}/src/lib.rs | 0 {lib/kernel-util => libk}/src/mem/address.rs | 0 {lib/kernel-util => libk}/src/mem/device.rs | 0 {lib/kernel-util => libk}/src/mem/mod.rs | 0 {lib/kernel-util => libk}/src/mem/phys.rs | 0 {lib/kernel-util => libk}/src/mem/pointer.rs | 0 {lib/kernel-util => libk}/src/mem/table.rs | 0 {lib/kernel-util => libk}/src/runtime/executor.rs | 0 {lib/kernel-util => libk}/src/runtime/macros.rs | 0 {lib/kernel-util => libk}/src/runtime/mod.rs | 0 {lib/kernel-util => libk}/src/runtime/task.rs | 0 {lib/kernel-util => libk}/src/runtime/task_queue.rs | 0 {lib/kernel-util => libk}/src/runtime/timer.rs | 0 {lib/kernel-util => libk}/src/runtime/waker.rs | 0 {lib/kernel-util => libk}/src/sync/fence.rs | 0 {lib/kernel-util => libk}/src/sync/guard.rs | 0 {lib/kernel-util => libk}/src/sync/mod.rs | 0 {lib/kernel-util => libk}/src/sync/mutex.rs | 0 {lib/kernel-util => libk}/src/sync/spin_rwlock.rs | 0 {lib/kernel-util => libk}/src/sync/spinlock.rs | 0 {lib/kernel-util => libk}/src/thread.rs | 0 {lib/kernel-util => libk}/src/util/event.rs | 0 {lib/kernel-util => libk}/src/util/mod.rs | 0 {lib/kernel-util => libk}/src/util/queue.rs | 0 {lib/kernel-util => libk}/src/util/ring.rs | 0 src/arch/aarch64/boot/mod.rs | 2 +- src/arch/aarch64/context.rs | 2 +- src/arch/aarch64/cpu.rs | 2 +- src/arch/aarch64/exception.rs | 2 +- src/arch/aarch64/gic/gicc.rs | 2 +- src/arch/aarch64/gic/gicd.rs | 2 +- src/arch/aarch64/gic/mod.rs | 2 +- src/arch/aarch64/mem/mod.rs | 2 +- src/arch/aarch64/mem/process.rs | 2 +- src/arch/aarch64/mem/table.rs | 2 +- src/arch/aarch64/mod.rs | 2 +- src/arch/aarch64/timer.rs | 2 +- src/arch/mod.rs | 2 +- src/arch/x86_64/acpi.rs | 2 +- src/arch/x86_64/apic/ioapic.rs | 2 +- src/arch/x86_64/apic/local.rs | 2 +- src/arch/x86_64/boot/mod.rs | 2 +- src/arch/x86_64/context.rs | 2 +- src/arch/x86_64/cpu.rs | 2 +- src/arch/x86_64/cpuid.rs | 2 +- src/arch/x86_64/mem/mod.rs | 2 +- src/arch/x86_64/mem/process.rs | 2 +- src/arch/x86_64/mem/table.rs | 2 +- src/arch/x86_64/mod.rs | 2 +- src/arch/x86_64/peripherals/i8253.rs | 2 +- src/arch/x86_64/peripherals/ps2/mod.rs | 2 +- src/arch/x86_64/peripherals/serial.rs | 2 +- src/arch/x86_64/smp.rs | 2 +- src/debug.rs | 2 +- src/device/bus/dt_pci.rs | 2 +- src/device/display/console.rs | 2 +- src/device/display/fb_console.rs | 2 +- src/device/display/font.rs | 2 +- src/device/display/linear_fb.rs | 2 +- src/device/input.rs | 2 +- src/device/mod.rs | 2 +- src/device/power/sunxi_rwdog.rs | 2 +- src/device/serial/pl011.rs | 2 +- src/device/serial/sunxi_uart.rs | 2 +- src/device/tty.rs | 2 +- src/fs/mod.rs | 2 +- src/fs/sysfs.rs | 2 +- src/init.rs | 2 +- src/main.rs | 2 +- src/mem/address.rs | 2 +- src/mem/heap.rs | 2 +- src/mem/mod.rs | 2 +- src/mem/phys/manager.rs | 2 +- src/mem/phys/mod.rs | 2 +- src/mem/phys/reserved.rs | 2 +- src/mem/process.rs | 2 +- src/mem/table.rs | 2 +- src/panic.rs | 2 +- src/proc/elf.rs | 2 +- src/proc/exec.rs | 2 +- src/proc/random.rs | 2 +- src/syscall/mod.rs | 2 +- src/task/context.rs | 2 +- src/task/mod.rs | 2 +- src/task/process.rs | 2 +- src/task/sched.rs | 2 +- src/task/sync.rs | 2 +- src/task/thread.rs | 2 +- 147 files changed, 123 insertions(+), 123 deletions(-) rename {lib/kernel-util => libk}/Cargo.toml (85%) rename {lib/kernel-util => libk}/src/api.rs (100%) rename {lib/kernel-util => libk}/src/lib.rs (100%) rename {lib/kernel-util => libk}/src/mem/address.rs (100%) rename {lib/kernel-util => libk}/src/mem/device.rs (100%) rename {lib/kernel-util => libk}/src/mem/mod.rs (100%) rename {lib/kernel-util => libk}/src/mem/phys.rs (100%) rename {lib/kernel-util => libk}/src/mem/pointer.rs (100%) rename {lib/kernel-util => libk}/src/mem/table.rs (100%) rename {lib/kernel-util => libk}/src/runtime/executor.rs (100%) rename {lib/kernel-util => libk}/src/runtime/macros.rs (100%) rename {lib/kernel-util => libk}/src/runtime/mod.rs (100%) rename {lib/kernel-util => libk}/src/runtime/task.rs (100%) rename {lib/kernel-util => libk}/src/runtime/task_queue.rs (100%) rename {lib/kernel-util => libk}/src/runtime/timer.rs (100%) rename {lib/kernel-util => libk}/src/runtime/waker.rs (100%) rename {lib/kernel-util => libk}/src/sync/fence.rs (100%) rename {lib/kernel-util => libk}/src/sync/guard.rs (100%) rename {lib/kernel-util => libk}/src/sync/mod.rs (100%) rename {lib/kernel-util => libk}/src/sync/mutex.rs (100%) rename {lib/kernel-util => libk}/src/sync/spin_rwlock.rs (100%) rename {lib/kernel-util => libk}/src/sync/spinlock.rs (100%) rename {lib/kernel-util => libk}/src/thread.rs (100%) rename {lib/kernel-util => libk}/src/util/event.rs (100%) rename {lib/kernel-util => libk}/src/util/mod.rs (100%) rename {lib/kernel-util => libk}/src/util/queue.rs (100%) rename {lib/kernel-util => libk}/src/util/ring.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index ad54aed7..97729396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ opt-level = 3 yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } device-api = { path = "lib/device-api", features = ["derive"] } -kernel-util = { path = "lib/kernel-util" } +libk = { path = "libk" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } diff --git a/driver/block/ahci/Cargo.toml b/driver/block/ahci/Cargo.toml index a922f145..e31db109 100644 --- a/driver/block/ahci/Cargo.toml +++ b/driver/block/ahci/Cargo.toml @@ -8,7 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/ahci/src/command.rs b/driver/block/ahci/src/command.rs index 9665d152..bca757f4 100644 --- a/driver/block/ahci/src/command.rs +++ b/driver/block/ahci/src/command.rs @@ -1,6 +1,6 @@ use core::mem::{size_of, MaybeUninit}; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; diff --git a/driver/block/ahci/src/data.rs b/driver/block/ahci/src/data.rs index ff15f5ca..06cdd1c2 100644 --- a/driver/block/ahci/src/data.rs +++ b/driver/block/ahci/src/data.rs @@ -2,7 +2,7 @@ use core::mem::size_of; use alloc::string::String; use bytemuck::{Pod, Zeroable}; -use kernel_util::{ +use libk::{ mem::address::{IntoRaw, PhysicalAddress}, util::{ConstAssert, IsTrue}, }; diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 795aa636..c748fab6 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -13,7 +13,7 @@ use device_api::{ }; use error::AhciError; use kernel_fs::devfs; -use kernel_util::{ +use libk::{ mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, runtime, sync::IrqSafeSpinlock, diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index 0333faa7..02835c57 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -7,7 +7,7 @@ use core::{ use alloc::{boxed::Box, string::String}; use bytemuck::Zeroable; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{ +use libk::{ mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, runtime::QueueWaker, sync::IrqSafeSpinlock, diff --git a/driver/block/ahci/src/regs.rs b/driver/block/ahci/src/regs.rs index 8f24649c..a2b674b3 100644 --- a/driver/block/ahci/src/regs.rs +++ b/driver/block/ahci/src/regs.rs @@ -1,4 +1,4 @@ -use kernel_util::mem::address::{IntoRaw, PhysicalAddress}; +use libk::mem::address::{IntoRaw, PhysicalAddress}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, diff --git a/driver/block/core/Cargo.toml b/driver/block/core/Cargo.toml index cdcc3170..60988ab0 100644 --- a/driver/block/core/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -8,7 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index 7dab9f99..d7ef4d68 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -8,7 +8,7 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{ +use libk::{ mem::{address::PhysicalAddress, PageBox, PageProvider}, runtime::QueueWaker, }; diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index 5bed61ad..dfa6335a 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::task::{Context, Poll}; -use kernel_util::mem::PageProvider; +use libk::mem::PageProvider; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; diff --git a/driver/block/core/src/partition.rs b/driver/block/core/src/partition.rs index ee59aeb3..148d3ad7 100644 --- a/driver/block/core/src/partition.rs +++ b/driver/block/core/src/partition.rs @@ -2,7 +2,7 @@ use core::mem::{size_of, MaybeUninit}; use alloc::{vec, vec::Vec}; use bytemuck::{Pod, Zeroable}; -use kernel_util::mem::PageBox; +use libk::mem::PageBox; use static_assertions::const_assert_eq; use uuid::Uuid; use yggdrasil_abi::{error::Error, io::DeviceRequest}; diff --git a/driver/block/core/src/request.rs b/driver/block/core/src/request.rs index 2fbb634e..8d47c655 100644 --- a/driver/block/core/src/request.rs +++ b/driver/block/core/src/request.rs @@ -1,6 +1,6 @@ use core::mem::MaybeUninit; -use kernel_util::mem::PageBox; +use libk::mem::PageBox; pub enum IoOperation { Read { lba: u64, count: usize }, diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml index 467cc2d4..8034c6ec 100644 --- a/driver/block/nvme/Cargo.toml +++ b/driver/block/nvme/Cargo.toml @@ -8,7 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/nvme/src/command.rs b/driver/block/nvme/src/command.rs index 5b48e5b4..20473935 100644 --- a/driver/block/nvme/src/command.rs +++ b/driver/block/nvme/src/command.rs @@ -2,7 +2,7 @@ use core::fmt::{self, Write}; -use kernel_util::mem::address::PhysicalAddress; +use libk::mem::address::PhysicalAddress; use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike}; use crate::queue::PhysicalRegionPage; diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index 37fe394a..7a0c5209 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -2,7 +2,7 @@ use core::task::Poll; use alloc::{boxed::Box, format}; use kernel_fs::devfs; -use kernel_util::{cpu_index, mem::address::AsPhysicalAddress, runtime::QueueWaker}; +use libk::{cpu_index, mem::address::AsPhysicalAddress, runtime::QueueWaker}; use ygg_driver_block::{ probe_partitions, IoOperation, IoRequest, IoSubmissionId, NgBlockDevice, NgBlockDeviceWrapper, }; diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index 7c68d104..acada38c 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -17,7 +17,7 @@ use device_api::{ Device, }; use drive::NvmeDrive; -use kernel_util::{ +use libk::{ cpu_count, cpu_index, mem::{ address::{IntoRaw, PhysicalAddress}, diff --git a/driver/block/nvme/src/queue.rs b/driver/block/nvme/src/queue.rs index 91124583..38beb468 100644 --- a/driver/block/nvme/src/queue.rs +++ b/driver/block/nvme/src/queue.rs @@ -11,7 +11,7 @@ use alloc::{ }; use bytemuck::{Pod, Zeroable}; use futures_util::Future; -use kernel_util::{ +use libk::{ mem::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, PageBox, diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index d45c7865..6673d7ef 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../../../lib/device-api", features = ["derive"] } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } log = "0.4.20" bitflags = "2.3.3" diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index b8c3b92a..6b9a0efc 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -4,7 +4,7 @@ use alloc::{vec, vec::Vec}; use device_api::interrupt::{ InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo, }; -use kernel_util::mem::{address::PhysicalAddress, device::DeviceMemoryIoMut}; +use libk::mem::{address::PhysicalAddress, device::DeviceMemoryIoMut}; use tock_registers::{ interfaces::{Readable, Writeable}, registers::{ReadWrite, WriteOnly}, diff --git a/driver/bus/pci/src/device.rs b/driver/bus/pci/src/device.rs index 98548174..50a4cda6 100644 --- a/driver/bus/pci/src/device.rs +++ b/driver/bus/pci/src/device.rs @@ -5,7 +5,7 @@ use device_api::{ interrupt::{InterruptAffinity, InterruptHandler, IrqOptions, MsiInfo}, Device, }; -use kernel_util::{ +use libk::{ message_interrupt_controller, register_global_interrupt, sync::spin_rwlock::IrqSafeRwLock, util::OneTimeInit, }; diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index 823c7f9c..57c0b562 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -11,7 +11,7 @@ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use bitflags::bitflags; use device::{PciBusDevice, PciDeviceInfo, PciDriver, PciInterrupt, PciInterruptRoute, PciMatch}; use device_api::Device; -use kernel_util::{ +use libk::{ mem::address::{FromRaw, PhysicalAddress}, sync::IrqSafeSpinlock, util::OneTimeInit, diff --git a/driver/bus/pci/src/space/ecam.rs b/driver/bus/pci/src/space/ecam.rs index 06490553..86c3c7f6 100644 --- a/driver/bus/pci/src/space/ecam.rs +++ b/driver/bus/pci/src/space/ecam.rs @@ -1,5 +1,5 @@ //! PCI Express ECAM interface -use kernel_util::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; +use libk::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; use yggdrasil_abi::error::Error; use super::{PciAddress, PciConfigurationSpace}; diff --git a/driver/fs/kernel-fs/Cargo.toml b/driver/fs/kernel-fs/Cargo.toml index 7aa8d3b6..0156d07a 100644 --- a/driver/fs/kernel-fs/Cargo.toml +++ b/driver/fs/kernel-fs/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "../../../lib/vfs" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } ygg_driver_block = { path = "../../block/core" } diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 1adc7203..37285e83 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::{format, string::String}; -use kernel_util::util::OneTimeInit; +use libk::util::OneTimeInit; use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; diff --git a/driver/fs/memfs/Cargo.toml b/driver/fs/memfs/Cargo.toml index 444de5de..f8a6d023 100644 --- a/driver/fs/memfs/Cargo.toml +++ b/driver/fs/memfs/Cargo.toml @@ -8,7 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } vfs = { path = "../../../lib/vfs" } static_assertions = "1.1.0" diff --git a/driver/fs/memfs/src/file.rs b/driver/fs/memfs/src/file.rs index 800e71c5..afbe6f58 100644 --- a/driver/fs/memfs/src/file.rs +++ b/driver/fs/memfs/src/file.rs @@ -1,6 +1,6 @@ use core::any::Any; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use vfs::{CommonImpl, InstanceData, Node, NodeFlags, NodeRef, RegularImpl}; use yggdrasil_abi::{error::Error, io::OpenOptions}; diff --git a/driver/net/core/Cargo.toml b/driver/net/core/Cargo.toml index f8e674f9..d6f383d5 100644 --- a/driver/net/core/Cargo.toml +++ b/driver/net/core/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["serde_kernel", "bytemuck"] } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } vfs = { path = "../../../lib/vfs" } kernel-fs = { path = "../../fs/kernel-fs" } diff --git a/driver/net/core/src/ethernet.rs b/driver/net/core/src/ethernet.rs index a9db10ce..492453e3 100644 --- a/driver/net/core/src/ethernet.rs +++ b/driver/net/core/src/ethernet.rs @@ -2,7 +2,7 @@ use core::mem::size_of; use alloc::sync::Arc; use bytemuck::Pod; -use kernel_util::mem::PageBox; +use libk::mem::PageBox; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs index 02da9380..7a1fbe7a 100644 --- a/driver/net/core/src/interface.rs +++ b/driver/net/core/src/interface.rs @@ -5,7 +5,7 @@ use core::{ use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; // TODO: link state management? -use kernel_util::{ +use libk::{ mem::PageBox, sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, util::OneTimeInit, diff --git a/driver/net/core/src/l3/arp.rs b/driver/net/core/src/l3/arp.rs index 56399ab1..fe15b9c3 100644 --- a/driver/net/core/src/l3/arp.rs +++ b/driver/net/core/src/l3/arp.rs @@ -7,7 +7,7 @@ use core::{ }; use alloc::{boxed::Box, collections::BTreeMap}; -use kernel_util::{ +use libk::{ runtime::{self, QueueWaker}, sync::spin_rwlock::IrqSafeRwLock, }; diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 08657ab9..a124d3ba 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -2,7 +2,7 @@ use core::{fmt, mem::size_of}; use alloc::{sync::Arc, vec::Vec}; use bytemuck::{Pod, Zeroable}; -use kernel_util::{ +use libk::{ mem::PageBox, sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard}, }; diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index 31f39a4e..d0bb4b07 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -5,7 +5,7 @@ use core::{ use alloc::{vec, vec::Vec}; use bytemuck::Zeroable; -use kernel_util::runtime::QueueWaker; +use libk::runtime::QueueWaker; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index 3dd8cb8f..b4ae4b13 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -9,7 +9,7 @@ use core::mem::size_of; use alloc::sync::Arc; use bytemuck::Pod; use ethernet::L2Packet; -use kernel_util::{mem::PageBox, runtime, util::queue::UnboundedMpmcQueue}; +use libk::{mem::PageBox, runtime, util::queue::UnboundedMpmcQueue}; use l3::L3Packet; use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 6951b674..8f720a65 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -7,7 +7,7 @@ use core::{ }; use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; -use kernel_util::{ +use libk::{ block, mem::PageBox, monotonic_timestamp, diff --git a/driver/net/loopback/Cargo.toml b/driver/net/loopback/Cargo.toml index 8ab886ab..19c39d16 100644 --- a/driver/net/loopback/Cargo.toml +++ b/driver/net/loopback/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } ygg_driver_net_core = { path = "../../net/core" } diff --git a/driver/net/loopback/src/lib.rs b/driver/net/loopback/src/lib.rs index af3350a6..1ae87eff 100644 --- a/driver/net/loopback/src/lib.rs +++ b/driver/net/loopback/src/lib.rs @@ -1,6 +1,6 @@ #![no_std] -use kernel_util::{mem::PageBox, util::OneTimeInit}; +use libk::{mem::PageBox, util::OneTimeInit}; use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, Packet, diff --git a/driver/virtio/core/Cargo.toml b/driver/virtio/core/Cargo.toml index f2d0bc67..3dd59960 100644 --- a/driver/virtio/core/Cargo.toml +++ b/driver/virtio/core/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } device-api = { path = "../../../lib/device-api", features = ["derive"] } ygg_driver_pci = { path = "../../bus/pci", optional = true } diff --git a/driver/virtio/core/src/queue.rs b/driver/virtio/core/src/queue.rs index 0c738cbf..c96ca2a2 100644 --- a/driver/virtio/core/src/queue.rs +++ b/driver/virtio/core/src/queue.rs @@ -8,7 +8,7 @@ use core::{ sync::atomic::{fence, Ordering}, }; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, IntoRaw}, PageBox, }; diff --git a/driver/virtio/core/src/transport/mod.rs b/driver/virtio/core/src/transport/mod.rs index 1ae3aa00..0ea91b5e 100644 --- a/driver/virtio/core/src/transport/mod.rs +++ b/driver/virtio/core/src/transport/mod.rs @@ -1,6 +1,6 @@ use core::mem::size_of; -use kernel_util::mem::{ +use libk::mem::{ address::{IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, }; diff --git a/driver/virtio/core/src/transport/pci.rs b/driver/virtio/core/src/transport/pci.rs index 320730ee..b1456949 100644 --- a/driver/virtio/core/src/transport/pci.rs +++ b/driver/virtio/core/src/transport/pci.rs @@ -1,4 +1,4 @@ -use kernel_util::mem::device::DeviceMemoryIo; +use libk::mem::device::DeviceMemoryIo; use tock_registers::{ interfaces::Readable, registers::{ReadOnly, WriteOnly}, diff --git a/driver/virtio/net/Cargo.toml b/driver/virtio/net/Cargo.toml index b8008284..9a0c4ea2 100644 --- a/driver/virtio/net/Cargo.toml +++ b/driver/virtio/net/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-util = { path = "../../../lib/kernel-util" } +libk = { path = "../../../libk" } device-api = { path = "../../../lib/device-api", features = ["derive"] } ygg_driver_virtio_core = { path = "../core" } diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs index e3679eaa..717241df 100644 --- a/driver/virtio/net/src/lib.rs +++ b/driver/virtio/net/src/lib.rs @@ -12,7 +12,7 @@ use device_api::{ interrupt::{InterruptAffinity, InterruptHandler}, Device, }; -use kernel_util::{ +use libk::{ mem::PageBox, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, util::OneTimeInit, diff --git a/lib/device-tree/Cargo.toml b/lib/device-tree/Cargo.toml index 322112ce..d25c13dc 100644 --- a/lib/device-tree/Cargo.toml +++ b/lib/device-tree/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../device-api", features = ["derive"] } -kernel-util = { path = "../kernel-util" } +libk = { path = "../../libk" } fdt-rs = { version = "0.4.3", default-features = false } log = "0.4.20" diff --git a/lib/device-tree/src/dt.rs b/lib/device-tree/src/dt.rs index dc96b311..b4405619 100644 --- a/lib/device-tree/src/dt.rs +++ b/lib/device-tree/src/dt.rs @@ -5,7 +5,7 @@ use fdt_rs::{ index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, prelude::PropReader, }; -use kernel_util::mem::{ +use libk::mem::{ address::{FromRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, }; diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 0a292833..9e561d0c 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -8,7 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } -kernel-util = { path = "../kernel-util" } +libk = { path = "../../libk" } ygg_driver_block = { path = "../../driver/block/core" } diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index e83859f4..6743beb3 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -11,7 +11,7 @@ use alloc::{ sync::Arc, }; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{ +use libk::{ block, sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, }; diff --git a/lib/vfs/src/file/device.rs b/lib/vfs/src/file/device.rs index 1c639ea3..471e0882 100644 --- a/lib/vfs/src/file/device.rs +++ b/lib/vfs/src/file/device.rs @@ -1,4 +1,4 @@ -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ diff --git a/lib/vfs/src/file/directory.rs b/lib/vfs/src/file/directory.rs index 4dda854c..10e180dd 100644 --- a/lib/vfs/src/file/directory.rs +++ b/lib/vfs/src/file/directory.rs @@ -1,6 +1,6 @@ use core::{mem::MaybeUninit, str::FromStr}; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::DirectoryEntry, util::FixedString}; use crate::node::NodeRef; diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 14e4518e..aaf9eaf6 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -9,7 +9,7 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use kernel_util::{ +use libk::{ mem::{address::PhysicalAddress, PageProvider}, sync::IrqSafeSpinlock, }; @@ -551,7 +551,7 @@ mod tests { use core::{mem::MaybeUninit, str::FromStr}; use std::sync::{Arc, Mutex}; - use kernel_util::sync::IrqSafeSpinlock; + use libk::sync::IrqSafeSpinlock; use ygg_driver_block::BlockDevice; use yggdrasil_abi::{ error::Error, diff --git a/lib/vfs/src/file/pipe.rs b/lib/vfs/src/file/pipe.rs index 7e662344..aec3febe 100644 --- a/lib/vfs/src/file/pipe.rs +++ b/lib/vfs/src/file/pipe.rs @@ -6,7 +6,7 @@ use core::{ use alloc::{sync::Arc, vec, vec::Vec}; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{block, sync::IrqSafeSpinlock}; +use libk::{block, sync::IrqSafeSpinlock}; use yggdrasil_abi::error::Error; struct PipeInner { diff --git a/lib/vfs/src/file/regular.rs b/lib/vfs/src/file/regular.rs index f9638dbe..4600d17e 100644 --- a/lib/vfs/src/file/regular.rs +++ b/lib/vfs/src/file/regular.rs @@ -1,4 +1,4 @@ -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; use super::InstanceData; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index 21bfb1c5..d279a4de 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -7,7 +7,7 @@ use alloc::{ vec::Vec, }; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::OpenOptions}; use crate::{DirectoryOpenPosition, InstanceData}; diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index 2030bd52..65e9dcbc 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -1,7 +1,7 @@ use core::{any::Any, fmt}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use ygg_driver_block::BlockDevice; use yggdrasil_abi::{ bitflags, diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index ec61e220..55ca1c82 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -7,7 +7,7 @@ use core::{ use alloc::collections::BTreeMap; use futures_util::{future::BoxFuture, FutureExt}; -use kernel_util::{ +use libk::{ runtime, sync::{mutex::Mutex, LockMethod}, }; diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index bc762dbb..91c387a2 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -8,7 +8,7 @@ use core::{ }; use alloc::{boxed::Box, sync::Arc}; -use kernel_util::{ +use libk::{ block, signal_process_group, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, util::ring::{LossyRingQueue, RingBuffer}, diff --git a/lib/vfs/src/shared_memory.rs b/lib/vfs/src/shared_memory.rs index 86e0f4f5..161e1709 100644 --- a/lib/vfs/src/shared_memory.rs +++ b/lib/vfs/src/shared_memory.rs @@ -1,7 +1,7 @@ use core::mem::MaybeUninit; use alloc::vec::Vec; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, PageProvider, }; diff --git a/lib/vfs/src/timer.rs b/lib/vfs/src/timer.rs index d1370376..e6a34049 100644 --- a/lib/vfs/src/timer.rs +++ b/lib/vfs/src/timer.rs @@ -7,7 +7,7 @@ use core::{ use alloc::boxed::Box; use futures_util::FutureExt; -use kernel_util::{ +use libk::{ runtime::{self, SleepFuture}, sync::{mutex::Mutex, LockMethod}, }; diff --git a/lib/kernel-util/Cargo.toml b/libk/Cargo.toml similarity index 85% rename from lib/kernel-util/Cargo.toml rename to libk/Cargo.toml index 217d5daa..76bba981 100644 --- a/lib/kernel-util/Cargo.toml +++ b/libk/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "kernel-util" +name = "libk" version = "0.1.0" edition = "2021" authors = ["Mark Poliakov "] @@ -8,7 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -device-api = { path = "../device-api", features = ["derive"] } +device-api = { path = "../lib/device-api", features = ["derive"] } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/lib/kernel-util/src/api.rs b/libk/src/api.rs similarity index 100% rename from lib/kernel-util/src/api.rs rename to libk/src/api.rs diff --git a/lib/kernel-util/src/lib.rs b/libk/src/lib.rs similarity index 100% rename from lib/kernel-util/src/lib.rs rename to libk/src/lib.rs diff --git a/lib/kernel-util/src/mem/address.rs b/libk/src/mem/address.rs similarity index 100% rename from lib/kernel-util/src/mem/address.rs rename to libk/src/mem/address.rs diff --git a/lib/kernel-util/src/mem/device.rs b/libk/src/mem/device.rs similarity index 100% rename from lib/kernel-util/src/mem/device.rs rename to libk/src/mem/device.rs diff --git a/lib/kernel-util/src/mem/mod.rs b/libk/src/mem/mod.rs similarity index 100% rename from lib/kernel-util/src/mem/mod.rs rename to libk/src/mem/mod.rs diff --git a/lib/kernel-util/src/mem/phys.rs b/libk/src/mem/phys.rs similarity index 100% rename from lib/kernel-util/src/mem/phys.rs rename to libk/src/mem/phys.rs diff --git a/lib/kernel-util/src/mem/pointer.rs b/libk/src/mem/pointer.rs similarity index 100% rename from lib/kernel-util/src/mem/pointer.rs rename to libk/src/mem/pointer.rs diff --git a/lib/kernel-util/src/mem/table.rs b/libk/src/mem/table.rs similarity index 100% rename from lib/kernel-util/src/mem/table.rs rename to libk/src/mem/table.rs diff --git a/lib/kernel-util/src/runtime/executor.rs b/libk/src/runtime/executor.rs similarity index 100% rename from lib/kernel-util/src/runtime/executor.rs rename to libk/src/runtime/executor.rs diff --git a/lib/kernel-util/src/runtime/macros.rs b/libk/src/runtime/macros.rs similarity index 100% rename from lib/kernel-util/src/runtime/macros.rs rename to libk/src/runtime/macros.rs diff --git a/lib/kernel-util/src/runtime/mod.rs b/libk/src/runtime/mod.rs similarity index 100% rename from lib/kernel-util/src/runtime/mod.rs rename to libk/src/runtime/mod.rs diff --git a/lib/kernel-util/src/runtime/task.rs b/libk/src/runtime/task.rs similarity index 100% rename from lib/kernel-util/src/runtime/task.rs rename to libk/src/runtime/task.rs diff --git a/lib/kernel-util/src/runtime/task_queue.rs b/libk/src/runtime/task_queue.rs similarity index 100% rename from lib/kernel-util/src/runtime/task_queue.rs rename to libk/src/runtime/task_queue.rs diff --git a/lib/kernel-util/src/runtime/timer.rs b/libk/src/runtime/timer.rs similarity index 100% rename from lib/kernel-util/src/runtime/timer.rs rename to libk/src/runtime/timer.rs diff --git a/lib/kernel-util/src/runtime/waker.rs b/libk/src/runtime/waker.rs similarity index 100% rename from lib/kernel-util/src/runtime/waker.rs rename to libk/src/runtime/waker.rs diff --git a/lib/kernel-util/src/sync/fence.rs b/libk/src/sync/fence.rs similarity index 100% rename from lib/kernel-util/src/sync/fence.rs rename to libk/src/sync/fence.rs diff --git a/lib/kernel-util/src/sync/guard.rs b/libk/src/sync/guard.rs similarity index 100% rename from lib/kernel-util/src/sync/guard.rs rename to libk/src/sync/guard.rs diff --git a/lib/kernel-util/src/sync/mod.rs b/libk/src/sync/mod.rs similarity index 100% rename from lib/kernel-util/src/sync/mod.rs rename to libk/src/sync/mod.rs diff --git a/lib/kernel-util/src/sync/mutex.rs b/libk/src/sync/mutex.rs similarity index 100% rename from lib/kernel-util/src/sync/mutex.rs rename to libk/src/sync/mutex.rs diff --git a/lib/kernel-util/src/sync/spin_rwlock.rs b/libk/src/sync/spin_rwlock.rs similarity index 100% rename from lib/kernel-util/src/sync/spin_rwlock.rs rename to libk/src/sync/spin_rwlock.rs diff --git a/lib/kernel-util/src/sync/spinlock.rs b/libk/src/sync/spinlock.rs similarity index 100% rename from lib/kernel-util/src/sync/spinlock.rs rename to libk/src/sync/spinlock.rs diff --git a/lib/kernel-util/src/thread.rs b/libk/src/thread.rs similarity index 100% rename from lib/kernel-util/src/thread.rs rename to libk/src/thread.rs diff --git a/lib/kernel-util/src/util/event.rs b/libk/src/util/event.rs similarity index 100% rename from lib/kernel-util/src/util/event.rs rename to libk/src/util/event.rs diff --git a/lib/kernel-util/src/util/mod.rs b/libk/src/util/mod.rs similarity index 100% rename from lib/kernel-util/src/util/mod.rs rename to libk/src/util/mod.rs diff --git a/lib/kernel-util/src/util/queue.rs b/libk/src/util/queue.rs similarity index 100% rename from lib/kernel-util/src/util/queue.rs rename to libk/src/util/queue.rs diff --git a/lib/kernel-util/src/util/ring.rs b/libk/src/util/ring.rs similarity index 100% rename from lib/kernel-util/src/util/ring.rs rename to libk/src/util/ring.rs diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 734e41b8..8c422fc5 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -6,7 +6,7 @@ use aarch64_cpu::{ registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; use kernel_fs::devfs; -use kernel_util::{ +use libk::{ mem::{ address::{IntoRaw, PhysicalAddress}, table::EntryLevel, diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index b0a8e637..e9245e18 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -3,7 +3,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use kernel_util::mem::address::PhysicalAddress; +use libk::mem::address::PhysicalAddress; use crate::{ mem::phys, diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 3e8f97ab..3214ccee 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -6,7 +6,7 @@ use core::{ use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::{ +use libk::{ sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 50152751..35739dce 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -278,7 +278,7 @@ extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) { let iss = esr_el1 & 0x1FFFFFF; unsafe { - kernel_util::sync::hack_locks(); + libk::sync::hack_locks(); } dump_irrecoverable_exception(frame, ec, iss); diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs index 745c236c..a2dded00 100644 --- a/src/arch/aarch64/gic/gicc.rs +++ b/src/arch/aarch64/gic/gicc.rs @@ -1,5 +1,5 @@ //! ARM GICv2 CPU registers -use kernel_util::mem::device::DeviceMemoryIo; +use libk::mem::device::DeviceMemoryIo; use tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, register_structs, diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index ff69a78c..c757540a 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -1,6 +1,6 @@ //! ARM GICv2 Distributor registers use device_api::interrupt::{IpiDeliveryTarget, IrqLevel, IrqOptions, IrqTrigger}; -use kernel_util::mem::device::DeviceMemoryIo; +use libk::mem::device::DeviceMemoryIo; use spinning_top::Spinlock; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 4c524dfe..b21bd3dd 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -14,7 +14,7 @@ use device_api::{ Device, }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, PhysicalAddress}, device::{DeviceMemoryIo, RawDeviceMemoryMapping}, diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index 650a6075..59a4f49f 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -7,7 +7,7 @@ use core::{ use abi::error::Error; use cfg_if::cfg_if; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index f7372ca4..b053d917 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::PhysicalRefMut, table::{EntryLevel, EntryLevelExt}, diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs index 3f44c383..92acb031 100644 --- a/src/arch/aarch64/mem/table.rs +++ b/src/arch/aarch64/mem/table.rs @@ -5,7 +5,7 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevel, diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 5ef05811..eb9016ca 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -16,7 +16,7 @@ use device_api::{ }; use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; use git_version::git_version; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 734f012f..b20087d2 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -7,7 +7,7 @@ use abi::error::Error; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; use device_tree::device_tree_driver; -use kernel_util::runtime; +use libk::runtime; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, ARCHITECTURE}; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 2fc43bbc..980c5da4 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -30,7 +30,7 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use kernel_util::{ +use libk::{ mem::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index b5a268ab..0685ef01 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -15,7 +15,7 @@ use device_api::{ interrupt::{InterruptHandler, IpiDeliveryTarget}, Device, }; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, PhysicalAddress}, pointer::PhysicalRef, diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 585adcac..2649bf0a 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -8,7 +8,7 @@ use device_api::{ }, Device, }; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIo, diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 4f81958e..88e5f0ec 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -10,7 +10,7 @@ use device_api::{ }, Device, }; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 4667b1ae..a19ce8f7 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -2,7 +2,7 @@ use core::{arch::global_asm, sync::atomic::Ordering}; use kernel_fs::devfs; -use kernel_util::runtime; +use libk::runtime; use tock_registers::interfaces::Writeable; use yboot_proto::{ v1::{FramebufferOption, MemoryMap}, diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 31a3b15e..44c3a66e 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -3,7 +3,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use kernel_util::mem::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; +use libk::mem::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; use crate::{ arch::x86_64::mem::KERNEL_TABLES, diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 13bbecb5..d80b8ba1 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -6,7 +6,7 @@ use core::{ }; use alloc::{boxed::Box, vec::Vec}; -use kernel_util::{ +use libk::{ sync::{IrqGuard, IrqSafeSpinlock}, util::OneTimeInit, }; diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 71d93742..536ea72c 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -1,7 +1,7 @@ //! x86-64 CPUID interface use bitflags::bitflags; -use kernel_util::util::OneTimeInit; +use libk::util::OneTimeInit; use tock_registers::interfaces::ReadWriteable; use super::registers::{CR4, XCR0}; diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 0b4c39c8..3a52780d 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -6,7 +6,7 @@ use core::{ }; use abi::error::Error; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index c3a09527..9e689dc5 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,5 +1,5 @@ //! x86-64-specific process address space management functions -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, table::EntryLevelExt, diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index 8032f066..7d1f14e5 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -6,7 +6,7 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, }; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 1a61f38c..eccc5e6c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -11,7 +11,7 @@ use device_api::{ }; use git_version::git_version; use kernel_fs::devfs; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index 62530857..1964084b 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -2,7 +2,7 @@ use core::time::Duration; use abi::error::Error; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; -use kernel_util::{runtime, sync::IrqSafeSpinlock}; +use libk::{runtime, sync::IrqSafeSpinlock}; use crate::arch::{ x86_64::{ diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index ca2b46b9..f8d836de 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -4,7 +4,7 @@ use abi::{ io::{KeyboardKey, KeyboardKeyEvent}, }; use device_api::{interrupt::InterruptHandler, Device}; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use crate::{ arch::{ diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index ae263164..e4dcd91b 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -1,7 +1,7 @@ //! Driver for x86 COM ports use abi::error::Error; use device_api::{serial::SerialDevice, Device}; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use crate::{ arch::x86_64::{ diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index f8a08ae1..e4d0d1b5 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; -use kernel_util::mem::{ +use libk::mem::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, }; diff --git a/src/debug.rs b/src/debug.rs index 94a5fd79..8df4e481 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,7 +2,7 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use kernel_util::{ +use libk::{ sync::IrqSafeSpinlock, util::{ring::RingBuffer, StaticVector}, }; diff --git a/src/device/bus/dt_pci.rs b/src/device/bus/dt_pci.rs index 9f042721..ce514527 100644 --- a/src/device/bus/dt_pci.rs +++ b/src/device/bus/dt_pci.rs @@ -6,7 +6,7 @@ use device_tree::{ device_tree_driver, dt::{self, DevTreeIndexNodeExt, DevTreeIndexPropExt, DevTreeNodeInfo}, }; -use kernel_util::mem::address::{FromRaw, PhysicalAddress}; +use libk::mem::address::{FromRaw, PhysicalAddress}; use ygg_driver_pci::{ device::{PciInterrupt, PciInterruptPin, PciInterruptRoute}, PciAddress, PciAddressRange, PciBusManager, PciRangeType, diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 0823aadb..67e354db 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -5,7 +5,7 @@ use core::time::Duration; use abi::{error::Error, primitive_enum}; use alloc::{vec, vec::Vec}; use bitflags::bitflags; -use kernel_util::{runtime, sync::IrqSafeSpinlock, util::StaticVector}; +use libk::{runtime, sync::IrqSafeSpinlock, util::StaticVector}; use crate::debug::DebugSink; diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 33d5e19d..9f91085f 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -1,7 +1,7 @@ //! Framebuffer console driver use abi::error::Error; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use crate::debug::DebugSink; diff --git a/src/device/display/font.rs b/src/device/display/font.rs index 76450009..f45824a3 100644 --- a/src/device/display/font.rs +++ b/src/device/display/font.rs @@ -4,7 +4,7 @@ use core::mem::size_of; use abi::error::Error; use bytemuck::{Pod, Zeroable}; -use kernel_util::AlignedTo; +use libk::AlignedTo; static CONSOLE_FONT: &AlignedTo = &AlignedTo { align: [], diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 1b6c67a7..899deba6 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -7,7 +7,7 @@ use core::{ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; -use kernel_util::{ +use libk::{ mem::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, diff --git a/src/device/input.rs b/src/device/input.rs index 1b7bdf13..84e6c31e 100644 --- a/src/device/input.rs +++ b/src/device/input.rs @@ -7,7 +7,7 @@ use abi::{ error::Error, io::{DeviceRequest, KeyboardKeyEvent}, }; -use kernel_util::{block, util::ring::LossyRingQueue}; +use libk::{block, util::ring::LossyRingQueue}; use vfs::{CharDevice, FileReadiness}; pub struct KeyboardDevice; diff --git a/src/device/mod.rs b/src/device/mod.rs index 9e04c40b..0734c65f 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,7 +1,7 @@ //! Device management and interfaces use device_api::{manager::DeviceManager, Device, DeviceId}; -use kernel_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; +use libk::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; pub mod bus; diff --git a/src/device/power/sunxi_rwdog.rs b/src/device/power/sunxi_rwdog.rs index 917adf60..450b5145 100644 --- a/src/device/power/sunxi_rwdog.rs +++ b/src/device/power/sunxi_rwdog.rs @@ -3,7 +3,7 @@ use abi::error::Error; use alloc::boxed::Box; use device_api::{Device, ResetDevice}; -use kernel_util::util::OneTimeInit; +use libk::util::OneTimeInit; use tock_registers::{ interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, }; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index b74b088c..d015bc32 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -5,7 +5,7 @@ use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use futures_util::task::{Context, Poll}; use kernel_fs::devfs::{self, CharDeviceType}; -use kernel_util::{ +use libk::{ block, mem::{ address::{FromRaw, PhysicalAddress}, diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs index 8cf502be..e340fd69 100644 --- a/src/device/serial/sunxi_uart.rs +++ b/src/device/serial/sunxi_uart.rs @@ -3,7 +3,7 @@ use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; -use kernel_util::util::OneTimeInit; +use libk::util::OneTimeInit; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, diff --git a/src/device/tty.rs b/src/device/tty.rs index 8cb96ef4..3afc787b 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -12,7 +12,7 @@ use abi::{ }; use device_api::serial::SerialDevice; use futures_util::Future; -use kernel_util::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBuffer}; +use libk::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBuffer}; use crate::task::process::{Process, ProcessId}; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 22811030..5a4cc370 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -3,7 +3,7 @@ use core::ptr::NonNull; use kernel_fs::devfs; -use kernel_util::{mem::address::PhysicalAddress, util::OneTimeInit}; +use libk::{mem::address::PhysicalAddress, util::OneTimeInit}; use memfs::block::{self, BlockAllocator}; use vfs::{impls::read_fn_node, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 4dd02c3a..9f792249 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -2,7 +2,7 @@ use abi::error::Error; use git_version::git_version; -use kernel_util::util::OneTimeInit; +use libk::util::OneTimeInit; use vfs::{ impls::{const_value_node, mdir, read_fn_node, ReadOnlyFnValueNode}, NodeRef, diff --git a/src/init.rs b/src/init.rs index 50c78db2..4423cd9c 100644 --- a/src/init.rs +++ b/src/init.rs @@ -3,7 +3,7 @@ use abi::error::Error; use alloc::borrow::ToOwned; use kernel_fs::devfs; -use kernel_util::runtime; +use libk::runtime; use memfs::MemoryFilesystem; use vfs::{IoContext, NodeRef}; diff --git a/src/main.rs b/src/main.rs index 1870b640..c952bff3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,7 +39,7 @@ #![no_main] use arch::Architecture; -use kernel_util::sync::SpinFence; +use libk::sync::SpinFence; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, diff --git a/src/mem/address.rs b/src/mem/address.rs index dfbf9f90..a1a660c2 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -1,6 +1,6 @@ //! Address manipulation interfaces and utilities -use kernel_util::mem::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; +use libk::mem::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; use super::KERNEL_VIRT_OFFSET; use core::ops::{Deref, DerefMut}; diff --git a/src/mem/heap.rs b/src/mem/heap.rs index 5a7a146e..b9aab1e9 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -5,7 +5,7 @@ use core::{ ptr::{null_mut, NonNull}, }; -use kernel_util::sync::IrqSafeSpinlock; +use libk::sync::IrqSafeSpinlock; use linked_list_allocator::Heap; /// Kernel heap manager diff --git a/src/mem/mod.rs b/src/mem/mod.rs index aeb48ead..43614e3d 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -7,7 +7,7 @@ use core::{ }; use abi::error::Error; -use kernel_util::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; +use libk::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; use crate::arch::{Architecture, ArchitectureImpl}; diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 76d1d231..1c6b3f27 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use abi::{error::Error, system::SystemMemoryStats}; -use kernel_util::mem::{ +use libk::mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, table::EntryLevel, diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index ef6d62a5..8e63101e 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,7 +1,7 @@ //! Physical memory management utilities use abi::{error::Error, system::SystemMemoryStats}; -use kernel_util::{ +use libk::{ mem::{ address::{FromRaw, IntoRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 58e58a29..11451749 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -1,6 +1,6 @@ //! Utilities for handling reserved memory regions -use kernel_util::util::StaticVector; +use libk::util::StaticVector; use crate::mem::PhysicalAddress; diff --git a/src/mem/process.rs b/src/mem/process.rs index 474d580a..a8e22a9b 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -4,7 +4,7 @@ use core::ops::Range; use abi::error::Error; use cfg_if::cfg_if; -use kernel_util::{ +use libk::{ mem::{ pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevelExt, diff --git a/src/mem/table.rs b/src/mem/table.rs index 89ff3312..d175acab 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -3,7 +3,7 @@ use core::ops::{Deref, DerefMut, Range}; use abi::error::Error; use bitflags::bitflags; -pub use kernel_util::mem::table::EntryLevel; +pub use libk::mem::table::EntryLevel; // TODO EXECUTABLE bitflags! { diff --git a/src/panic.rs b/src/panic.rs index 9661fdf3..8f5cfffb 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use device_api::interrupt::IpiDeliveryTarget; -use kernel_util::sync::{hack_locks, SpinFence}; +use libk::sync::{hack_locks, SpinFence}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess, CpuMessage, ARCHITECTURE}, diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 902c5b59..f324c07c 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -7,7 +7,7 @@ use elf::{ segment::ProgramHeader, ElfStream, ParseError, }; -use kernel_util::mem::{ +use libk::mem::{ pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevelExt, }; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 3d017919..2dda93f8 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -14,7 +14,7 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; -use kernel_util::mem::pointer::PhysicalRefMut; +use libk::mem::pointer::PhysicalRefMut; use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ diff --git a/src/proc/random.rs b/src/proc/random.rs index b80c896e..659ff67d 100644 --- a/src/proc/random.rs +++ b/src/proc/random.rs @@ -1,6 +1,6 @@ //! Random generation utilities -use kernel_util::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; use crate::arch::{Architecture, ARCHITECTURE}; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 3a7fdd3f..b24d17bd 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -18,7 +18,7 @@ use abi::{ system::SystemInfo, }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; -use kernel_util::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; +use libk::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; diff --git a/src/task/context.rs b/src/task/context.rs index b7ad8906..d78d486f 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -3,7 +3,7 @@ use abi::{arch::SavedFrame, error::Error}; use alloc::{boxed::Box, sync::Arc}; use cfg_if::cfg_if; -use kernel_util::thread::Termination; +use libk::thread::Termination; use crate::task::thread::Thread; diff --git a/src/task/mod.rs b/src/task/mod.rs index a81006a9..83675104 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -4,7 +4,7 @@ use abi::error::Error; use alloc::{string::String, vec::Vec}; -use kernel_util::{runtime, sync::SpinFence, thread::Termination}; +use libk::{runtime, sync::SpinFence, thread::Termination}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess}, diff --git a/src/task/process.rs b/src/task/process.rs index ba7ff411..5e03b06c 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -16,7 +16,7 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; -use kernel_util::{ +use libk::{ sync::{ spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, IrqSafeSpinlock, diff --git a/src/task/sched.rs b/src/task/sched.rs index fbe17b92..170c2c55 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -5,7 +5,7 @@ use core::cell::Cell; use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; -use kernel_util::{sync::IrqGuard, util::OneTimeInit}; +use libk::{sync::IrqGuard, util::OneTimeInit}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess}, diff --git a/src/task/sync.rs b/src/task/sync.rs index 3776b154..76207a77 100644 --- a/src/task/sync.rs +++ b/src/task/sync.rs @@ -8,7 +8,7 @@ use core::{ use alloc::sync::Arc; use futures_util::Future; -use kernel_util::runtime::QueueWaker; +use libk::runtime::QueueWaker; /// User-space mutex (like BSD/Linux's futex) data structure pub struct UserspaceMutex { diff --git a/src/task/thread.rs b/src/task/thread.rs index de014f4b..933c17d5 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -15,7 +15,7 @@ use abi::{ use alloc::{collections::BTreeMap, string::String, sync::Arc}; use atomic_enum::atomic_enum; use crossbeam_queue::SegQueue; -use kernel_util::{ +use libk::{ block, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, util::event::BoolEvent, From 579994739a6282788d3b276bc689b2a157d5f9dd Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 5 Feb 2024 11:55:50 +0200 Subject: [PATCH 177/211] libk: move memory management to libk-mm crate --- Cargo.toml | 1 + driver/block/ahci/Cargo.toml | 1 + driver/block/ahci/src/command.rs | 2 +- driver/block/ahci/src/data.rs | 6 ++---- driver/block/ahci/src/lib.rs | 8 ++------ driver/block/ahci/src/port.rs | 8 ++------ driver/block/ahci/src/regs.rs | 2 +- driver/block/core/Cargo.toml | 1 + driver/block/core/src/device.rs | 6 ++---- driver/block/core/src/lib.rs | 2 +- driver/block/core/src/request.rs | 2 +- driver/bus/pci/Cargo.toml | 1 + driver/bus/pci/src/capability.rs | 2 +- driver/bus/pci/src/lib.rs | 7 ++----- driver/bus/pci/src/space/ecam.rs | 2 +- driver/net/core/Cargo.toml | 1 + driver/net/core/src/ethernet.rs | 2 +- driver/net/core/src/interface.rs | 2 +- driver/net/core/src/l3/mod.rs | 6 ++---- driver/net/core/src/lib.rs | 3 ++- driver/net/core/src/socket.rs | 5 ++--- driver/net/loopback/Cargo.toml | 1 + driver/net/loopback/src/lib.rs | 3 ++- driver/virtio/core/Cargo.toml | 2 +- driver/virtio/core/src/queue.rs | 2 +- driver/virtio/core/src/transport/mod.rs | 2 +- driver/virtio/core/src/transport/pci.rs | 2 +- driver/virtio/net/Cargo.toml | 1 + driver/virtio/net/src/lib.rs | 2 +- lib/device-tree/Cargo.toml | 1 + lib/device-tree/src/dt.rs | 2 +- lib/vfs/Cargo.toml | 1 + lib/vfs/src/file/mod.rs | 6 ++---- lib/vfs/src/shared_memory.rs | 2 +- libk-mm/Cargo.toml | 11 +++++++++++ {libk/src/mem => libk-mm/src}/address.rs | 0 libk-mm/src/api.rs | 23 +++++++++++++++++++++++ {libk/src/mem => libk-mm/src}/device.rs | 0 libk/src/mem/mod.rs => libk-mm/src/lib.rs | 15 ++++++++++++++- libk-mm/src/mod.rs | 0 {libk/src/mem => libk-mm/src}/phys.rs | 0 {libk/src/mem => libk-mm/src}/pointer.rs | 0 {libk/src/mem => libk-mm/src}/table.rs | 0 libk/Cargo.toml | 2 ++ libk/src/api.rs | 16 ++-------------- libk/src/lib.rs | 1 - src/arch/aarch64/boot/mod.rs | 10 ++++------ src/arch/aarch64/context.rs | 2 +- src/arch/aarch64/gic/gicc.rs | 2 +- src/arch/aarch64/gic/gicd.rs | 2 +- src/arch/aarch64/gic/mod.rs | 11 ++++------- src/arch/aarch64/mem/mod.rs | 12 +++++------- src/arch/aarch64/mem/process.rs | 2 +- src/arch/aarch64/mem/table.rs | 2 +- src/arch/aarch64/mod.rs | 16 +++++++--------- src/arch/mod.rs | 14 ++++++-------- src/device/bus/dt_pci.rs | 2 +- src/device/display/linear_fb.rs | 14 ++++++-------- src/device/serial/pl011.rs | 12 ++++-------- src/fs/mod.rs | 3 ++- src/mem/address.rs | 2 +- src/mem/mod.rs | 2 +- src/mem/phys/manager.rs | 2 +- src/mem/phys/mod.rs | 11 ++++------- src/mem/phys/reserved.rs | 5 +---- src/mem/process.rs | 15 +++++++-------- src/mem/table.rs | 2 +- src/proc/elf.rs | 2 +- src/proc/exec.rs | 2 +- src/syscall/mod.rs | 3 ++- 70 files changed, 160 insertions(+), 155 deletions(-) create mode 100644 libk-mm/Cargo.toml rename {libk/src/mem => libk-mm/src}/address.rs (100%) create mode 100644 libk-mm/src/api.rs rename {libk/src/mem => libk-mm/src}/device.rs (100%) rename libk/src/mem/mod.rs => libk-mm/src/lib.rs (96%) create mode 100644 libk-mm/src/mod.rs rename {libk/src/mem => libk-mm/src}/phys.rs (100%) rename {libk/src/mem => libk-mm/src}/pointer.rs (100%) rename {libk/src/mem => libk-mm/src}/table.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 97729396..ce67dbc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } device-api = { path = "lib/device-api", features = ["derive"] } libk = { path = "libk" } +libk-mm = { path = "libk-mm" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } diff --git a/driver/block/ahci/Cargo.toml b/driver/block/ahci/Cargo.toml index e31db109..ae8b6f54 100644 --- a/driver/block/ahci/Cargo.toml +++ b/driver/block/ahci/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/ahci/src/command.rs b/driver/block/ahci/src/command.rs index bca757f4..d2c57561 100644 --- a/driver/block/ahci/src/command.rs +++ b/driver/block/ahci/src/command.rs @@ -1,6 +1,6 @@ use core::mem::{size_of, MaybeUninit}; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; diff --git a/driver/block/ahci/src/data.rs b/driver/block/ahci/src/data.rs index 06cdd1c2..5287e176 100644 --- a/driver/block/ahci/src/data.rs +++ b/driver/block/ahci/src/data.rs @@ -2,10 +2,8 @@ use core::mem::size_of; use alloc::string::String; use bytemuck::{Pod, Zeroable}; -use libk::{ - mem::address::{IntoRaw, PhysicalAddress}, - util::{ConstAssert, IsTrue}, -}; +use libk::util::{ConstAssert, IsTrue}; +use libk_mm::address::{IntoRaw, PhysicalAddress}; use static_assertions::const_assert_eq; use crate::{ diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index c748fab6..1d1b24e8 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -13,12 +13,8 @@ use device_api::{ }; use error::AhciError; use kernel_fs::devfs; -use libk::{ - mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, - runtime, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use libk::{runtime, sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}; use port::AhciPort; use regs::{PortRegs, Regs}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index 02835c57..3bed2294 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -7,12 +7,8 @@ use core::{ use alloc::{boxed::Box, string::String}; use bytemuck::Zeroable; use futures_util::{task::AtomicWaker, Future}; -use libk::{ - mem::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}, - runtime::QueueWaker, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use libk::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}; use tock_registers::interfaces::{Readable, Writeable}; use ygg_driver_block::{IoOperation, IoRequest, IoSubmissionId, NgBlockDevice}; use yggdrasil_abi::error::Error; diff --git a/driver/block/ahci/src/regs.rs b/driver/block/ahci/src/regs.rs index a2b674b3..5a2a29be 100644 --- a/driver/block/ahci/src/regs.rs +++ b/driver/block/ahci/src/regs.rs @@ -1,4 +1,4 @@ -use libk::mem::address::{IntoRaw, PhysicalAddress}; +use libk_mm::address::{IntoRaw, PhysicalAddress}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, diff --git a/driver/block/core/Cargo.toml b/driver/block/core/Cargo.toml index 60988ab0..e5c05db4 100644 --- a/driver/block/core/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index d7ef4d68..2d23aa72 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -8,10 +8,8 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use libk::{ - mem::{address::PhysicalAddress, PageBox, PageProvider}, - runtime::QueueWaker, -}; +use libk::runtime::QueueWaker; +use libk_mm::{address::PhysicalAddress, PageBox, PageProvider}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::{ diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index dfa6335a..80d7f1e0 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::task::{Context, Poll}; -use libk::mem::PageProvider; +use libk_mm::PageProvider; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; diff --git a/driver/block/core/src/request.rs b/driver/block/core/src/request.rs index 8d47c655..d379ff69 100644 --- a/driver/block/core/src/request.rs +++ b/driver/block/core/src/request.rs @@ -1,6 +1,6 @@ use core::mem::MaybeUninit; -use libk::mem::PageBox; +use libk_mm::PageBox; pub enum IoOperation { Read { lba: u64, count: usize }, diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index 6673d7ef..a54e7fe0 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -10,6 +10,7 @@ authors = ["Mark Poliakov "] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../../../lib/device-api", features = ["derive"] } libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } log = "0.4.20" bitflags = "2.3.3" diff --git a/driver/bus/pci/src/capability.rs b/driver/bus/pci/src/capability.rs index 6b9a0efc..ffd33379 100644 --- a/driver/bus/pci/src/capability.rs +++ b/driver/bus/pci/src/capability.rs @@ -4,7 +4,7 @@ use alloc::{vec, vec::Vec}; use device_api::interrupt::{ InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo, }; -use libk::mem::{address::PhysicalAddress, device::DeviceMemoryIoMut}; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIoMut}; use tock_registers::{ interfaces::{Readable, Writeable}, registers::{ReadWrite, WriteOnly}, diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index 57c0b562..6b89df6e 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -11,11 +11,8 @@ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use bitflags::bitflags; use device::{PciBusDevice, PciDeviceInfo, PciDriver, PciInterrupt, PciInterruptRoute, PciMatch}; use device_api::Device; -use libk::{ - mem::address::{FromRaw, PhysicalAddress}, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; +use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_mm::address::{FromRaw, PhysicalAddress}; use yggdrasil_abi::error::Error; pub mod capability; diff --git a/driver/bus/pci/src/space/ecam.rs b/driver/bus/pci/src/space/ecam.rs index 86c3c7f6..15ab2744 100644 --- a/driver/bus/pci/src/space/ecam.rs +++ b/driver/bus/pci/src/space/ecam.rs @@ -1,5 +1,5 @@ //! PCI Express ECAM interface -use libk::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; use yggdrasil_abi::error::Error; use super::{PciAddress, PciConfigurationSpace}; diff --git a/driver/net/core/Cargo.toml b/driver/net/core/Cargo.toml index d6f383d5..356ce8f5 100644 --- a/driver/net/core/Cargo.toml +++ b/driver/net/core/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["serde_kernel", "bytemuck"] } libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } vfs = { path = "../../../lib/vfs" } kernel-fs = { path = "../../fs/kernel-fs" } diff --git a/driver/net/core/src/ethernet.rs b/driver/net/core/src/ethernet.rs index 492453e3..9ac95c7c 100644 --- a/driver/net/core/src/ethernet.rs +++ b/driver/net/core/src/ethernet.rs @@ -2,7 +2,7 @@ use core::mem::size_of; use alloc::sync::Arc; use bytemuck::Pod; -use libk::mem::PageBox; +use libk_mm::PageBox; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs index 7a1fbe7a..dae07449 100644 --- a/driver/net/core/src/interface.rs +++ b/driver/net/core/src/interface.rs @@ -6,10 +6,10 @@ use core::{ use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; // TODO: link state management? use libk::{ - mem::PageBox, sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, util::OneTimeInit, }; +use libk_mm::PageBox; use yggdrasil_abi::{ error::Error, net::{protocols::EthernetFrame, IpAddr, MacAddress}, diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index a124d3ba..76e06c81 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -2,10 +2,8 @@ use core::{fmt, mem::size_of}; use alloc::{sync::Arc, vec::Vec}; use bytemuck::{Pod, Zeroable}; -use libk::{ - mem::PageBox, - sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard}, -}; +use libk::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard}; +use libk_mm::PageBox; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index b4ae4b13..5bae4451 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -9,8 +9,9 @@ use core::mem::size_of; use alloc::sync::Arc; use bytemuck::Pod; use ethernet::L2Packet; -use libk::{mem::PageBox, runtime, util::queue::UnboundedMpmcQueue}; use l3::L3Packet; +use libk::{runtime, util::queue::UnboundedMpmcQueue}; +use libk_mm::PageBox; use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; pub mod ethernet; diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 8f720a65..c486fcc9 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -8,9 +8,7 @@ use core::{ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use libk::{ - block, - mem::PageBox, - monotonic_timestamp, + block, monotonic_timestamp, runtime::{run_with_timeout, FutureTimeout, QueueWaker}, sync::{ spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, @@ -18,6 +16,7 @@ use libk::{ }, util::queue::BoundedMpmcQueue, }; +use libk_mm::PageBox; use vfs::{ConnectionSocket, FileReadiness, ListenerSocket, PacketSocket, Socket}; use yggdrasil_abi::{ error::Error, diff --git a/driver/net/loopback/Cargo.toml b/driver/net/loopback/Cargo.toml index 19c39d16..f1ceaf59 100644 --- a/driver/net/loopback/Cargo.toml +++ b/driver/net/loopback/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } ygg_driver_net_core = { path = "../../net/core" } diff --git a/driver/net/loopback/src/lib.rs b/driver/net/loopback/src/lib.rs index 1ae87eff..8e962471 100644 --- a/driver/net/loopback/src/lib.rs +++ b/driver/net/loopback/src/lib.rs @@ -1,6 +1,7 @@ #![no_std] -use libk::{mem::PageBox, util::OneTimeInit}; +use libk::util::OneTimeInit; +use libk_mm::PageBox; use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, Packet, diff --git a/driver/virtio/core/Cargo.toml b/driver/virtio/core/Cargo.toml index 3dd59960..b61400da 100644 --- a/driver/virtio/core/Cargo.toml +++ b/driver/virtio/core/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } ygg_driver_pci = { path = "../../bus/pci", optional = true } diff --git a/driver/virtio/core/src/queue.rs b/driver/virtio/core/src/queue.rs index c96ca2a2..885f1a64 100644 --- a/driver/virtio/core/src/queue.rs +++ b/driver/virtio/core/src/queue.rs @@ -8,7 +8,7 @@ use core::{ sync::atomic::{fence, Ordering}, }; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, IntoRaw}, PageBox, }; diff --git a/driver/virtio/core/src/transport/mod.rs b/driver/virtio/core/src/transport/mod.rs index 0ea91b5e..0c9df6f1 100644 --- a/driver/virtio/core/src/transport/mod.rs +++ b/driver/virtio/core/src/transport/mod.rs @@ -1,6 +1,6 @@ use core::mem::size_of; -use libk::mem::{ +use libk_mm::{ address::{IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, }; diff --git a/driver/virtio/core/src/transport/pci.rs b/driver/virtio/core/src/transport/pci.rs index b1456949..20a838d8 100644 --- a/driver/virtio/core/src/transport/pci.rs +++ b/driver/virtio/core/src/transport/pci.rs @@ -1,4 +1,4 @@ -use libk::mem::device::DeviceMemoryIo; +use libk_mm::device::DeviceMemoryIo; use tock_registers::{ interfaces::Readable, registers::{ReadOnly, WriteOnly}, diff --git a/driver/virtio/net/Cargo.toml b/driver/virtio/net/Cargo.toml index 9a0c4ea2..ccedf75e 100644 --- a/driver/virtio/net/Cargo.toml +++ b/driver/virtio/net/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-mm = { path = "../../../libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } ygg_driver_virtio_core = { path = "../core" } diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs index 717241df..88e9f1fa 100644 --- a/driver/virtio/net/src/lib.rs +++ b/driver/virtio/net/src/lib.rs @@ -13,10 +13,10 @@ use device_api::{ Device, }; use libk::{ - mem::PageBox, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, util::OneTimeInit, }; +use libk_mm::PageBox; use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, Packet, diff --git a/lib/device-tree/Cargo.toml b/lib/device-tree/Cargo.toml index d25c13dc..3363fbb9 100644 --- a/lib/device-tree/Cargo.toml +++ b/lib/device-tree/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../device-api", features = ["derive"] } libk = { path = "../../libk" } +libk-mm = { path = "../../libk-mm" } fdt-rs = { version = "0.4.3", default-features = false } log = "0.4.20" diff --git a/lib/device-tree/src/dt.rs b/lib/device-tree/src/dt.rs index b4405619..1304d55f 100644 --- a/lib/device-tree/src/dt.rs +++ b/lib/device-tree/src/dt.rs @@ -5,7 +5,7 @@ use fdt_rs::{ index::{iters::DevTreeIndexNodeSiblingIter, DevTreeIndex, DevTreeIndexNode, DevTreeIndexProp}, prelude::PropReader, }; -use libk::mem::{ +use libk_mm::{ address::{FromRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, }; diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 9e561d0c..808c521c 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } libk = { path = "../../libk" } +libk-mm = { path = "../../libk-mm" } ygg_driver_block = { path = "../../driver/block/core" } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index aaf9eaf6..c8287ada 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -9,10 +9,8 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use libk::{ - mem::{address::PhysicalAddress, PageProvider}, - sync::IrqSafeSpinlock, -}; +use libk::sync::IrqSafeSpinlock; +use libk_mm::{address::PhysicalAddress, PageProvider}; use yggdrasil_abi::{ error::Error, io::{ diff --git a/lib/vfs/src/shared_memory.rs b/lib/vfs/src/shared_memory.rs index 161e1709..85aac31f 100644 --- a/lib/vfs/src/shared_memory.rs +++ b/lib/vfs/src/shared_memory.rs @@ -1,7 +1,7 @@ use core::mem::MaybeUninit; use alloc::vec::Vec; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, PageProvider, }; diff --git a/libk-mm/Cargo.toml b/libk-mm/Cargo.toml new file mode 100644 index 00000000..6f19b1ba --- /dev/null +++ b/libk-mm/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "libk-mm" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } + +log = "0.4.20" diff --git a/libk/src/mem/address.rs b/libk-mm/src/address.rs similarity index 100% rename from libk/src/mem/address.rs rename to libk-mm/src/address.rs diff --git a/libk-mm/src/api.rs b/libk-mm/src/api.rs new file mode 100644 index 00000000..b319d8f0 --- /dev/null +++ b/libk-mm/src/api.rs @@ -0,0 +1,23 @@ +use yggdrasil_abi::error::Error; + +use crate::{ + address::PhysicalAddress, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, +}; + +extern "Rust" { + pub fn __allocate_2m_page() -> Result; + pub fn __allocate_page() -> Result; + pub fn __allocate_contiguous_pages(count: usize) -> Result; + pub fn __free_page(page: PhysicalAddress); + + pub fn __virtualize(phys: u64) -> usize; + pub fn __physicalize(virt: usize) -> u64; + + pub fn __map_device_pages( + base: PhysicalAddress, + count: usize, + attrs: DeviceMemoryAttributes, + ) -> Result; + pub fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping); +} diff --git a/libk/src/mem/device.rs b/libk-mm/src/device.rs similarity index 100% rename from libk/src/mem/device.rs rename to libk-mm/src/device.rs diff --git a/libk/src/mem/mod.rs b/libk-mm/src/lib.rs similarity index 96% rename from libk/src/mem/mod.rs rename to libk-mm/src/lib.rs index 3f439981..347b8326 100644 --- a/libk/src/mem/mod.rs +++ b/libk-mm/src/lib.rs @@ -1,3 +1,15 @@ +#![feature( + strict_provenance, + maybe_uninit_slice, + slice_ptr_get, + step_trait, + const_trait_impl, + effects +)] +#![no_std] + +extern crate alloc; + use core::{ alloc::Layout, fmt, @@ -7,11 +19,12 @@ use core::{ use yggdrasil_abi::error::Error; -use crate::api::{self, __allocate_contiguous_pages, __free_page, __physicalize}; +use crate::api::{__allocate_contiguous_pages, __free_page, __physicalize}; use self::address::{AsPhysicalAddress, PhysicalAddress}; pub mod address; +pub mod api; pub mod device; pub mod phys; pub mod pointer; diff --git a/libk-mm/src/mod.rs b/libk-mm/src/mod.rs new file mode 100644 index 00000000..e69de29b diff --git a/libk/src/mem/phys.rs b/libk-mm/src/phys.rs similarity index 100% rename from libk/src/mem/phys.rs rename to libk-mm/src/phys.rs diff --git a/libk/src/mem/pointer.rs b/libk-mm/src/pointer.rs similarity index 100% rename from libk/src/mem/pointer.rs rename to libk-mm/src/pointer.rs diff --git a/libk/src/mem/table.rs b/libk-mm/src/table.rs similarity index 100% rename from libk/src/mem/table.rs rename to libk-mm/src/table.rs diff --git a/libk/Cargo.toml b/libk/Cargo.toml index 76bba981..040bfb8d 100644 --- a/libk/Cargo.toml +++ b/libk/Cargo.toml @@ -7,6 +7,8 @@ authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +libk-mm = { path = "../libk-mm" } + yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../lib/device-api", features = ["derive"] } diff --git a/libk/src/api.rs b/libk/src/api.rs index d3beedf2..47343acd 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -2,18 +2,13 @@ use core::time::Duration; use alloc::{string::String, sync::Arc}; use device_api::interrupt::{InterruptHandler, IrqOptions, MessageInterruptController}; +use libk_mm::address::PhysicalAddress; use yggdrasil_abi::{ error::Error, process::{ExitCode, Signal}, }; -use crate::{ - mem::{ - address::PhysicalAddress, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - }, - thread::{CurrentThread, Thread}, -}; +use crate::thread::{CurrentThread, Thread}; extern "Rust" { pub fn __acquire_irq_guard() -> bool; @@ -30,13 +25,6 @@ extern "Rust" { pub fn __virtualize(phys: u64) -> usize; pub fn __physicalize(virt: usize) -> u64; - pub fn __map_device_pages( - base: PhysicalAddress, - count: usize, - attrs: DeviceMemoryAttributes, - ) -> Result; - pub fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping); - // SAFETY: Both the kernel-side and api-side Threads are sized and Arc<___> has the same value // for them pub fn __create_kthread( diff --git a/libk/src/lib.rs b/libk/src/lib.rs index 87a53add..8fa22c36 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -24,7 +24,6 @@ extern crate alloc; pub(crate) mod api; -pub mod mem; pub mod runtime; pub mod sync; pub mod thread; diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 8c422fc5..f17d608d 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -6,12 +6,10 @@ use aarch64_cpu::{ registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; use kernel_fs::devfs; -use libk::{ - mem::{ - address::{IntoRaw, PhysicalAddress}, - table::EntryLevel, - }, - runtime, +use libk::runtime; +use libk_mm::{ + address::{IntoRaw, PhysicalAddress}, + table::EntryLevel, }; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index e9245e18..ef5b114a 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -3,7 +3,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use libk::mem::address::PhysicalAddress; +use libk_mm::address::PhysicalAddress; use crate::{ mem::phys, diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs index a2dded00..32518521 100644 --- a/src/arch/aarch64/gic/gicc.rs +++ b/src/arch/aarch64/gic/gicc.rs @@ -1,5 +1,5 @@ //! ARM GICv2 CPU registers -use libk::mem::device::DeviceMemoryIo; +use libk_mm::device::DeviceMemoryIo; use tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, register_structs, diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index c757540a..5b48ec21 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -1,6 +1,6 @@ //! ARM GICv2 Distributor registers use device_api::interrupt::{IpiDeliveryTarget, IrqLevel, IrqOptions, IrqTrigger}; -use libk::mem::device::DeviceMemoryIo; +use libk_mm::device::DeviceMemoryIo; use spinning_top::Spinlock; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index b21bd3dd..d90c242b 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -14,13 +14,10 @@ use device_api::{ Device, }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; -use libk::{ - mem::{ - address::{FromRaw, PhysicalAddress}, - device::{DeviceMemoryIo, RawDeviceMemoryMapping}, - }, - sync::IrqSafeSpinlock, - util::OneTimeInit, +use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_mm::{ + address::{FromRaw, PhysicalAddress}, + device::{DeviceMemoryIo, RawDeviceMemoryMapping}, }; use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, CpuMessage}; diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index 59a4f49f..451955ec 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -7,13 +7,11 @@ use core::{ use abi::error::Error; use cfg_if::cfg_if; -use libk::{ - mem::{ - address::{FromRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - table::{EntryLevel, EntryLevelExt}, - }, - util::OneTimeInit, +use libk::util::OneTimeInit; +use libk_mm::{ + address::{FromRaw, PhysicalAddress}, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + table::{EntryLevel, EntryLevelExt}, }; use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index b053d917..f71a72fd 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::PhysicalRefMut, table::{EntryLevel, EntryLevelExt}, diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs index 92acb031..f87c59e9 100644 --- a/src/arch/aarch64/mem/table.rs +++ b/src/arch/aarch64/mem/table.rs @@ -5,7 +5,7 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevel, diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index eb9016ca..2c2933e2 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -16,15 +16,13 @@ use device_api::{ }; use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; use git_version::git_version; -use libk::{ - mem::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - phys::PhysicalMemoryRegion, - pointer::PhysicalRef, - table::{EntryLevel, EntryLevelExt}, - }, - util::OneTimeInit, +use libk::util::OneTimeInit; +use libk_mm::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + phys::PhysicalMemoryRegion, + pointer::PhysicalRef, + table::{EntryLevel, EntryLevelExt}, }; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use ygg_driver_pci::PciBusManager; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 980c5da4..cce0b280 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -30,14 +30,12 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use libk::{ - mem::{ - address::PhysicalAddress, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - phys::PhysicalMemoryRegion, - table::EntryLevel, - }, - sync::IrqGuard, +use libk::sync::IrqGuard; +use libk_mm::{ + address::PhysicalAddress, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + phys::PhysicalMemoryRegion, + table::EntryLevel, }; use crate::task::{sched::CpuQueue, thread::ThreadId, Cpu}; diff --git a/src/device/bus/dt_pci.rs b/src/device/bus/dt_pci.rs index ce514527..5cd98231 100644 --- a/src/device/bus/dt_pci.rs +++ b/src/device/bus/dt_pci.rs @@ -6,7 +6,7 @@ use device_tree::{ device_tree_driver, dt::{self, DevTreeIndexNodeExt, DevTreeIndexPropExt, DevTreeNodeInfo}, }; -use libk::mem::address::{FromRaw, PhysicalAddress}; +use libk_mm::address::{FromRaw, PhysicalAddress}; use ygg_driver_pci::{ device::{PciInterrupt, PciInterruptPin, PciInterruptRoute}, PciAddress, PciAddressRange, PciBusManager, PciRangeType, diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 899deba6..f077a120 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -7,14 +7,12 @@ use core::{ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; -use libk::{ - mem::{ - address::PhysicalAddress, - device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, - table::EntryLevel, - PageProvider, - }, - sync::IrqSafeSpinlock, +use libk::sync::IrqSafeSpinlock; +use libk_mm::{ + address::PhysicalAddress, + device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, + table::EntryLevel, + PageProvider, }; use ygg_driver_block::BlockDevice; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index d015bc32..5a88f5f9 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -5,14 +5,10 @@ use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use futures_util::task::{Context, Poll}; use kernel_fs::devfs::{self, CharDeviceType}; -use libk::{ - block, - mem::{ - address::{FromRaw, PhysicalAddress}, - device::DeviceMemoryIo, - }, - sync::IrqSafeSpinlock, - util::OneTimeInit, +use libk::{block, sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_mm::{ + address::{FromRaw, PhysicalAddress}, + device::DeviceMemoryIo, }; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 5a4cc370..1d007865 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -3,7 +3,8 @@ use core::ptr::NonNull; use kernel_fs::devfs; -use libk::{mem::address::PhysicalAddress, util::OneTimeInit}; +use libk::util::OneTimeInit; +use libk_mm::address::PhysicalAddress; use memfs::block::{self, BlockAllocator}; use vfs::{impls::read_fn_node, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; diff --git a/src/mem/address.rs b/src/mem/address.rs index a1a660c2..60eb04b9 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -1,6 +1,6 @@ //! Address manipulation interfaces and utilities -use libk::mem::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; +use libk_mm::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; use super::KERNEL_VIRT_OFFSET; use core::ops::{Deref, DerefMut}; diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 43614e3d..8d63b161 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -7,7 +7,7 @@ use core::{ }; use abi::error::Error; -use libk::mem::{address::PhysicalAddress, device::DeviceMemoryMapping}; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; use crate::arch::{Architecture, ArchitectureImpl}; diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 1c6b3f27..cecff52a 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use abi::{error::Error, system::SystemMemoryStats}; -use libk::mem::{ +use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, table::EntryLevel, diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 8e63101e..b94275c9 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,13 +1,10 @@ //! Physical memory management utilities use abi::{error::Error, system::SystemMemoryStats}; -use libk::{ - mem::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, - phys::PhysicalMemoryRegion, - }, - sync::IrqSafeSpinlock, - util::OneTimeInit, +use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_mm::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + phys::PhysicalMemoryRegion, }; use crate::{ diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 11451749..314203a4 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -1,10 +1,7 @@ //! Utilities for handling reserved memory regions use libk::util::StaticVector; - -use crate::mem::PhysicalAddress; - -use super::PhysicalMemoryRegion; +use libk_mm::{address::PhysicalAddress, phys::PhysicalMemoryRegion}; static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); diff --git a/src/mem/process.rs b/src/mem/process.rs index a8e22a9b..2b41325e 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -4,20 +4,19 @@ use core::ops::Range; use abi::error::Error; use cfg_if::cfg_if; -use libk::{ - mem::{ - pointer::{PhysicalRef, PhysicalRefMut}, - table::EntryLevelExt, - PageProvider, - }, - sync::IrqSafeSpinlock, +use libk::sync::IrqSafeSpinlock; +use libk_mm::{ + address::PhysicalAddress, + pointer::{PhysicalRef, PhysicalRefMut}, + table::EntryLevelExt, + PageProvider, }; use vfs::FileRef; use vmalloc::{RangeData, VirtualMemoryAllocator}; use crate::{arch::L3, mem::phys}; -use super::{table::MapAttributes, PhysicalAddress}; +use super::table::MapAttributes; cfg_if! { if #[cfg(target_arch = "aarch64")] { diff --git a/src/mem/table.rs b/src/mem/table.rs index d175acab..ecd1dd34 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -3,7 +3,7 @@ use core::ops::{Deref, DerefMut, Range}; use abi::error::Error; use bitflags::bitflags; -pub use libk::mem::table::EntryLevel; +pub use libk_mm::table::EntryLevel; // TODO EXECUTABLE bitflags! { diff --git a/src/proc/elf.rs b/src/proc/elf.rs index f324c07c..05b061b6 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -7,7 +7,7 @@ use elf::{ segment::ProgramHeader, ElfStream, ParseError, }; -use libk::mem::{ +use libk_mm::{ pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevelExt, }; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 2dda93f8..37c0f88f 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -14,7 +14,7 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; -use libk::mem::pointer::PhysicalRefMut; +use libk_mm::pointer::PhysicalRefMut; use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index b24d17bd..8baab4c9 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -18,7 +18,8 @@ use abi::{ system::SystemInfo, }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; -use libk::{block, mem::table::EntryLevelExt, runtime, sync::IrqSafeSpinlockGuard}; +use libk::{block, runtime, sync::IrqSafeSpinlockGuard}; +use libk_mm::table::EntryLevelExt; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; From 0576849e233be8c8770e84dfb347a1f678fd6d39 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 5 Feb 2024 12:35:09 +0200 Subject: [PATCH 178/211] libk: move util to libk-util --- Cargo.toml | 1 + driver/block/ahci/Cargo.toml | 1 + driver/block/ahci/src/data.rs | 2 +- driver/block/ahci/src/lib.rs | 3 +- driver/block/ahci/src/port.rs | 2 +- driver/block/core/Cargo.toml | 1 + driver/block/core/src/device.rs | 2 +- driver/bus/pci/Cargo.toml | 1 + driver/bus/pci/src/device.rs | 6 +- driver/bus/pci/src/lib.rs | 2 +- driver/fs/kernel-fs/Cargo.toml | 2 +- driver/fs/kernel-fs/src/devfs.rs | 2 +- driver/fs/memfs/Cargo.toml | 1 + driver/fs/memfs/src/file.rs | 2 +- driver/net/core/Cargo.toml | 1 + driver/net/core/src/interface.rs | 8 +-- driver/net/core/src/l3/arp.rs | 6 +- driver/net/core/src/l3/mod.rs | 4 +- driver/net/core/src/l4/tcp.rs | 2 +- driver/net/core/src/lib.rs | 3 +- driver/net/core/src/socket.rs | 9 ++- driver/net/loopback/Cargo.toml | 2 +- driver/net/loopback/src/lib.rs | 2 +- driver/virtio/net/Cargo.toml | 2 +- driver/virtio/net/src/lib.rs | 8 +-- lib/vfs/Cargo.toml | 1 + lib/vfs/src/channel.rs | 6 +- lib/vfs/src/file/device.rs | 2 +- lib/vfs/src/file/directory.rs | 8 ++- lib/vfs/src/file/mod.rs | 2 +- lib/vfs/src/file/pipe.rs | 3 +- lib/vfs/src/file/regular.rs | 2 +- lib/vfs/src/node/impls.rs | 2 +- lib/vfs/src/node/mod.rs | 2 +- lib/vfs/src/poll.rs | 6 +- lib/vfs/src/pty.rs | 6 +- lib/vfs/src/timer.rs | 3 +- libk-mm/src/lib.rs | 3 +- libk-util/Cargo.toml | 12 ++++ libk-util/src/api.rs | 4 ++ {libk/src/util => libk-util/src}/event.rs | 2 +- libk/src/util/mod.rs => libk-util/src/lib.rs | 10 ++- {libk/src/util => libk-util/src}/queue.rs | 2 +- {libk/src/util => libk-util/src}/ring.rs | 2 +- {libk => libk-util}/src/sync/fence.rs | 0 {libk => libk-util}/src/sync/guard.rs | 0 libk-util/src/sync/mod.rs | 37 ++++++++++ {libk => libk-util}/src/sync/spin_rwlock.rs | 0 {libk => libk-util}/src/sync/spinlock.rs | 0 libk-util/src/waker.rs | 72 +++++++++++++++++++ libk/Cargo.toml | 1 + libk/src/api.rs | 3 - libk/src/lib.rs | 1 - libk/src/runtime/mod.rs | 1 - libk/src/runtime/task.rs | 3 +- libk/src/runtime/task_queue.rs | 3 +- libk/src/runtime/timer.rs | 3 +- libk/src/runtime/waker.rs | 73 +------------------- libk/src/sync/mod.rs | 37 ---------- libk/src/sync/mutex.rs | 3 +- libk/src/thread.rs | 3 +- src/arch/aarch64/cpu.rs | 4 +- src/arch/aarch64/exception.rs | 2 +- src/arch/aarch64/gic/mod.rs | 2 +- src/arch/aarch64/mem/mod.rs | 2 +- src/arch/aarch64/mod.rs | 2 +- src/arch/mod.rs | 2 +- src/debug.rs | 5 +- src/device/display/console.rs | 3 +- src/device/display/fb_console.rs | 2 +- src/device/display/linear_fb.rs | 2 +- src/device/input.rs | 3 +- src/device/mod.rs | 2 +- src/device/serial/pl011.rs | 3 +- src/device/tty.rs | 2 +- src/fs/mod.rs | 2 +- src/fs/sysfs.rs | 2 +- src/main.rs | 2 +- src/mem/heap.rs | 2 +- src/mem/phys/mod.rs | 2 +- src/mem/phys/reserved.rs | 2 +- src/mem/process.rs | 2 +- src/panic.rs | 2 +- src/proc/random.rs | 2 +- src/syscall/mod.rs | 3 +- src/task/mod.rs | 3 +- src/task/process.rs | 4 +- src/task/sched.rs | 2 +- src/task/sync.rs | 2 +- src/task/thread.rs | 6 +- 90 files changed, 253 insertions(+), 214 deletions(-) create mode 100644 libk-util/Cargo.toml create mode 100644 libk-util/src/api.rs rename {libk/src/util => libk-util/src}/event.rs (98%) rename libk/src/util/mod.rs => libk-util/src/lib.rs (97%) rename {libk/src/util => libk-util/src}/queue.rs (98%) rename {libk/src/util => libk-util/src}/ring.rs (99%) rename {libk => libk-util}/src/sync/fence.rs (100%) rename {libk => libk-util}/src/sync/guard.rs (100%) create mode 100644 libk-util/src/sync/mod.rs rename {libk => libk-util}/src/sync/spin_rwlock.rs (100%) rename {libk => libk-util}/src/sync/spinlock.rs (100%) create mode 100644 libk-util/src/waker.rs diff --git a/Cargo.toml b/Cargo.toml index ce67dbc8..47c6e1fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } device-api = { path = "lib/device-api", features = ["derive"] } libk = { path = "libk" } +libk-util = { path = "libk-util" } libk-mm = { path = "libk-mm" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } diff --git a/driver/block/ahci/Cargo.toml b/driver/block/ahci/Cargo.toml index ae8b6f54..b962c397 100644 --- a/driver/block/ahci/Cargo.toml +++ b/driver/block/ahci/Cargo.toml @@ -10,6 +10,7 @@ authors = ["Mark Poliakov "] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk-util" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/ahci/src/data.rs b/driver/block/ahci/src/data.rs index 5287e176..2c5ff64f 100644 --- a/driver/block/ahci/src/data.rs +++ b/driver/block/ahci/src/data.rs @@ -2,8 +2,8 @@ use core::mem::size_of; use alloc::string::String; use bytemuck::{Pod, Zeroable}; -use libk::util::{ConstAssert, IsTrue}; use libk_mm::address::{IntoRaw, PhysicalAddress}; +use libk_util::{ConstAssert, IsTrue}; use static_assertions::const_assert_eq; use crate::{ diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 1d1b24e8..e06dae56 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -13,8 +13,9 @@ use device_api::{ }; use error::AhciError; use kernel_fs::devfs; -use libk::{runtime, sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk::runtime; use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use port::AhciPort; use regs::{PortRegs, Regs}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; diff --git a/driver/block/ahci/src/port.rs b/driver/block/ahci/src/port.rs index 3bed2294..14afc1d9 100644 --- a/driver/block/ahci/src/port.rs +++ b/driver/block/ahci/src/port.rs @@ -7,8 +7,8 @@ use core::{ use alloc::{boxed::Box, string::String}; use bytemuck::Zeroable; use futures_util::{task::AtomicWaker, Future}; -use libk::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::OneTimeInit}; use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}; +use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker, OneTimeInit}; use tock_registers::interfaces::{Readable, Writeable}; use ygg_driver_block::{IoOperation, IoRequest, IoSubmissionId, NgBlockDevice}; use yggdrasil_abi::error::Error; diff --git a/driver/block/core/Cargo.toml b/driver/block/core/Cargo.toml index e5c05db4..e9c521d3 100644 --- a/driver/block/core/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-util = { path = "../../../libk-util" } libk-mm = { path = "../../../libk-mm" } log = "0.4.20" diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index 2d23aa72..99a693f8 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -8,8 +8,8 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use libk::runtime::QueueWaker; use libk_mm::{address::PhysicalAddress, PageBox, PageProvider}; +use libk_util::waker::QueueWaker; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::{ diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index a54e7fe0..0f3231b3 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -11,6 +11,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../../../lib/device-api", features = ["derive"] } libk = { path = "../../../libk" } libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk-util" } log = "0.4.20" bitflags = "2.3.3" diff --git a/driver/bus/pci/src/device.rs b/driver/bus/pci/src/device.rs index 50a4cda6..c2ddcda0 100644 --- a/driver/bus/pci/src/device.rs +++ b/driver/bus/pci/src/device.rs @@ -5,10 +5,8 @@ use device_api::{ interrupt::{InterruptAffinity, InterruptHandler, IrqOptions, MsiInfo}, Device, }; -use libk::{ - message_interrupt_controller, register_global_interrupt, sync::spin_rwlock::IrqSafeRwLock, - util::OneTimeInit, -}; +use libk::{message_interrupt_controller, register_global_interrupt}; +use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; use yggdrasil_abi::error::Error; use crate::{ diff --git a/driver/bus/pci/src/lib.rs b/driver/bus/pci/src/lib.rs index 6b89df6e..aa768167 100644 --- a/driver/bus/pci/src/lib.rs +++ b/driver/bus/pci/src/lib.rs @@ -11,8 +11,8 @@ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use bitflags::bitflags; use device::{PciBusDevice, PciDeviceInfo, PciDriver, PciInterrupt, PciInterruptRoute, PciMatch}; use device_api::Device; -use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; use libk_mm::address::{FromRaw, PhysicalAddress}; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use yggdrasil_abi::error::Error; pub mod capability; diff --git a/driver/fs/kernel-fs/Cargo.toml b/driver/fs/kernel-fs/Cargo.toml index 0156d07a..3428724d 100644 --- a/driver/fs/kernel-fs/Cargo.toml +++ b/driver/fs/kernel-fs/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "../../../lib/vfs" } -libk = { path = "../../../libk" } +libk-util = { path = "../../../libk-util" } ygg_driver_block = { path = "../../block/core" } diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 37285e83..63c8e814 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::{format, string::String}; -use libk::util::OneTimeInit; +use libk_util::OneTimeInit; use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; diff --git a/driver/fs/memfs/Cargo.toml b/driver/fs/memfs/Cargo.toml index f8a6d023..4dd636cc 100644 --- a/driver/fs/memfs/Cargo.toml +++ b/driver/fs/memfs/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-util = { path = "../../../libk-util" } vfs = { path = "../../../lib/vfs" } static_assertions = "1.1.0" diff --git a/driver/fs/memfs/src/file.rs b/driver/fs/memfs/src/file.rs index afbe6f58..e89dd4d7 100644 --- a/driver/fs/memfs/src/file.rs +++ b/driver/fs/memfs/src/file.rs @@ -1,6 +1,6 @@ use core::any::Any; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use vfs::{CommonImpl, InstanceData, Node, NodeFlags, NodeRef, RegularImpl}; use yggdrasil_abi::{error::Error, io::OpenOptions}; diff --git a/driver/net/core/Cargo.toml b/driver/net/core/Cargo.toml index 356ce8f5..eb7ab6c0 100644 --- a/driver/net/core/Cargo.toml +++ b/driver/net/core/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["serde_kernel", "bytemuck"] } libk = { path = "../../../libk" } libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk-util" } vfs = { path = "../../../lib/vfs" } kernel-fs = { path = "../../fs/kernel-fs" } diff --git a/driver/net/core/src/interface.rs b/driver/net/core/src/interface.rs index dae07449..90c3b46c 100644 --- a/driver/net/core/src/interface.rs +++ b/driver/net/core/src/interface.rs @@ -5,11 +5,11 @@ use core::{ use alloc::{boxed::Box, collections::BTreeMap, format, sync::Arc}; // TODO: link state management? -use libk::{ - sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, - util::OneTimeInit, -}; use libk_mm::PageBox; +use libk_util::{ + sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}, + OneTimeInit, +}; use yggdrasil_abi::{ error::Error, net::{protocols::EthernetFrame, IpAddr, MacAddress}, diff --git a/driver/net/core/src/l3/arp.rs b/driver/net/core/src/l3/arp.rs index fe15b9c3..18cdc3b7 100644 --- a/driver/net/core/src/l3/arp.rs +++ b/driver/net/core/src/l3/arp.rs @@ -7,10 +7,8 @@ use core::{ }; use alloc::{boxed::Box, collections::BTreeMap}; -use libk::{ - runtime::{self, QueueWaker}, - sync::spin_rwlock::IrqSafeRwLock, -}; +use libk::runtime; +use libk_util::{sync::spin_rwlock::IrqSafeRwLock, waker::QueueWaker}; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/l3/mod.rs b/driver/net/core/src/l3/mod.rs index 76e06c81..7d42fd5f 100644 --- a/driver/net/core/src/l3/mod.rs +++ b/driver/net/core/src/l3/mod.rs @@ -2,8 +2,10 @@ use core::{fmt, mem::size_of}; use alloc::{sync::Arc, vec::Vec}; use bytemuck::{Pod, Zeroable}; -use libk::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard}; use libk_mm::PageBox; +use libk_util::sync::spin_rwlock::{ + IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard, +}; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/l4/tcp.rs b/driver/net/core/src/l4/tcp.rs index d0bb4b07..00694bc1 100644 --- a/driver/net/core/src/l4/tcp.rs +++ b/driver/net/core/src/l4/tcp.rs @@ -5,7 +5,7 @@ use core::{ use alloc::{vec, vec::Vec}; use bytemuck::Zeroable; -use libk::runtime::QueueWaker; +use libk_util::waker::QueueWaker; use yggdrasil_abi::{ error::Error, net::{ diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index 5bae4451..91fd89e4 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -10,8 +10,9 @@ use alloc::sync::Arc; use bytemuck::Pod; use ethernet::L2Packet; use l3::L3Packet; -use libk::{runtime, util::queue::UnboundedMpmcQueue}; +use libk::runtime; use libk_mm::PageBox; +use libk_util::queue::UnboundedMpmcQueue; use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; pub mod ethernet; diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index c486fcc9..08ed82bf 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -9,14 +9,17 @@ use core::{ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use libk::{ block, monotonic_timestamp, - runtime::{run_with_timeout, FutureTimeout, QueueWaker}, + runtime::{run_with_timeout, FutureTimeout}, +}; +use libk_mm::PageBox; +use libk_util::{ + queue::BoundedMpmcQueue, sync::{ spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, IrqSafeSpinlock, IrqSafeSpinlockGuard, }, - util::queue::BoundedMpmcQueue, + waker::QueueWaker, }; -use libk_mm::PageBox; use vfs::{ConnectionSocket, FileReadiness, ListenerSocket, PacketSocket, Socket}; use yggdrasil_abi::{ error::Error, diff --git a/driver/net/loopback/Cargo.toml b/driver/net/loopback/Cargo.toml index f1ceaf59..4dce1a70 100644 --- a/driver/net/loopback/Cargo.toml +++ b/driver/net/loopback/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } +libk-util = { path = "../../../libk-util" } libk-mm = { path = "../../../libk-mm" } ygg_driver_net_core = { path = "../../net/core" } diff --git a/driver/net/loopback/src/lib.rs b/driver/net/loopback/src/lib.rs index 8e962471..a422da34 100644 --- a/driver/net/loopback/src/lib.rs +++ b/driver/net/loopback/src/lib.rs @@ -1,7 +1,7 @@ #![no_std] -use libk::util::OneTimeInit; use libk_mm::PageBox; +use libk_util::OneTimeInit; use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, Packet, diff --git a/driver/virtio/net/Cargo.toml b/driver/virtio/net/Cargo.toml index ccedf75e..0ae88914 100644 --- a/driver/virtio/net/Cargo.toml +++ b/driver/virtio/net/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } +libk-util = { path = "../../../libk-util" } libk-mm = { path = "../../../libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } diff --git a/driver/virtio/net/src/lib.rs b/driver/virtio/net/src/lib.rs index 88e9f1fa..79ae7f60 100644 --- a/driver/virtio/net/src/lib.rs +++ b/driver/virtio/net/src/lib.rs @@ -12,11 +12,11 @@ use device_api::{ interrupt::{InterruptAffinity, InterruptHandler}, Device, }; -use libk::{ - sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, - util::OneTimeInit, -}; use libk_mm::PageBox; +use libk_util::{ + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}, + OneTimeInit, +}; use ygg_driver_net_core::{ interface::{NetworkDevice, NetworkInterfaceType}, Packet, diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 808c521c..b7e91861 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -10,6 +10,7 @@ authors = ["Mark Poliakov "] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } libk = { path = "../../libk" } libk-mm = { path = "../../libk-mm" } +libk-util = { path = "../../libk-util" } ygg_driver_block = { path = "../../driver/block/core" } diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index 6743beb3..71ac68dc 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -11,10 +11,8 @@ use alloc::{ sync::Arc, }; use futures_util::{task::AtomicWaker, Future}; -use libk::{ - block, - sync::{mutex::Mutex, IrqSafeSpinlock, LockMethod}, -}; +use libk::{block, sync::mutex::Mutex}; +use libk_util::sync::{IrqSafeSpinlock, LockMethod}; use yggdrasil_abi::{error::Error, io::MessageDestination}; use crate::{FileReadiness, FileRef}; diff --git a/lib/vfs/src/file/device.rs b/lib/vfs/src/file/device.rs index 471e0882..4abf8f3d 100644 --- a/lib/vfs/src/file/device.rs +++ b/lib/vfs/src/file/device.rs @@ -1,4 +1,4 @@ -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ diff --git a/lib/vfs/src/file/directory.rs b/lib/vfs/src/file/directory.rs index 10e180dd..a90f6e93 100644 --- a/lib/vfs/src/file/directory.rs +++ b/lib/vfs/src/file/directory.rs @@ -1,6 +1,6 @@ use core::{mem::MaybeUninit, str::FromStr}; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::DirectoryEntry, util::FixedString}; use crate::node::NodeRef; @@ -46,11 +46,13 @@ impl DirectoryFile { pos = DirectoryCachePosition::Index(0); Some((FixedString::from_str("..").unwrap(), node.parent())) } - DirectoryCachePosition::Index(index) if let Some((name, node)) = children.get(index) => { + DirectoryCachePosition::Index(index) + if let Some((name, node)) = children.get(index) => + { pos = DirectoryCachePosition::Index(index + 1); Some((FixedString::from_str(name)?, node.clone())) } - DirectoryCachePosition::Index(_) => None + DirectoryCachePosition::Index(_) => None, }; let Some((name, node)) = entry else { diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index c8287ada..061f3ec3 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -9,8 +9,8 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use libk::sync::IrqSafeSpinlock; use libk_mm::{address::PhysicalAddress, PageProvider}; +use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{ error::Error, io::{ diff --git a/lib/vfs/src/file/pipe.rs b/lib/vfs/src/file/pipe.rs index aec3febe..1c4da47f 100644 --- a/lib/vfs/src/file/pipe.rs +++ b/lib/vfs/src/file/pipe.rs @@ -6,7 +6,8 @@ use core::{ use alloc::{sync::Arc, vec, vec::Vec}; use futures_util::{task::AtomicWaker, Future}; -use libk::{block, sync::IrqSafeSpinlock}; +use libk::block; +use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::error::Error; struct PipeInner { diff --git a/lib/vfs/src/file/regular.rs b/lib/vfs/src/file/regular.rs index 4600d17e..c36a413d 100644 --- a/lib/vfs/src/file/regular.rs +++ b/lib/vfs/src/file/regular.rs @@ -1,4 +1,4 @@ -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; use super::InstanceData; diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index d279a4de..a9e6b52b 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -7,7 +7,7 @@ use alloc::{ vec::Vec, }; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::OpenOptions}; use crate::{DirectoryOpenPosition, InstanceData}; diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index 65e9dcbc..ea4a8093 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -1,7 +1,7 @@ use core::{any::Any, fmt}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use ygg_driver_block::BlockDevice; use yggdrasil_abi::{ bitflags, diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index 55ca1c82..30b22846 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -7,10 +7,8 @@ use core::{ use alloc::collections::BTreeMap; use futures_util::{future::BoxFuture, FutureExt}; -use libk::{ - runtime, - sync::{mutex::Mutex, LockMethod}, -}; +use libk::{runtime, sync::mutex::Mutex}; +use libk_util::sync::LockMethod; use yggdrasil_abi::{error::Error, io::RawFd}; use crate::{FileReadiness, FileRef}; diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 91c387a2..550b2e10 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -8,10 +8,10 @@ use core::{ }; use alloc::{boxed::Box, sync::Arc}; -use libk::{ - block, signal_process_group, +use libk::{block, signal_process_group}; +use libk_util::{ + ring::{LossyRingQueue, RingBuffer}, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, - util::ring::{LossyRingQueue, RingBuffer}, }; use yggdrasil_abi::{ error::Error, diff --git a/lib/vfs/src/timer.rs b/lib/vfs/src/timer.rs index e6a34049..b44ae92d 100644 --- a/lib/vfs/src/timer.rs +++ b/lib/vfs/src/timer.rs @@ -9,8 +9,9 @@ use alloc::boxed::Box; use futures_util::FutureExt; use libk::{ runtime::{self, SleepFuture}, - sync::{mutex::Mutex, LockMethod}, + sync::mutex::Mutex, }; +use libk_util::sync::LockMethod; use yggdrasil_abi::error::Error; use crate::{FileReadiness, Write}; diff --git a/libk-mm/src/lib.rs b/libk-mm/src/lib.rs index 347b8326..bed02a2b 100644 --- a/libk-mm/src/lib.rs +++ b/libk-mm/src/lib.rs @@ -23,8 +23,9 @@ use crate::api::{__allocate_contiguous_pages, __free_page, __physicalize}; use self::address::{AsPhysicalAddress, PhysicalAddress}; +pub(crate) mod api; + pub mod address; -pub mod api; pub mod device; pub mod phys; pub mod pointer; diff --git a/libk-util/Cargo.toml b/libk-util/Cargo.toml new file mode 100644 index 00000000..df98c82c --- /dev/null +++ b/libk-util/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "libk-util" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } + +log = "0.4.20" +crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } diff --git a/libk-util/src/api.rs b/libk-util/src/api.rs new file mode 100644 index 00000000..85f8d31f --- /dev/null +++ b/libk-util/src/api.rs @@ -0,0 +1,4 @@ +extern "Rust" { + pub fn __acquire_irq_guard() -> bool; + pub fn __release_irq_guard(mask: bool); +} diff --git a/libk/src/util/event.rs b/libk-util/src/event.rs similarity index 98% rename from libk/src/util/event.rs rename to libk-util/src/event.rs index 09455a0c..6d532dcd 100644 --- a/libk/src/util/event.rs +++ b/libk-util/src/event.rs @@ -4,7 +4,7 @@ use core::{ task::Poll, }; -use crate::{runtime::QueueWaker, sync::spin_rwlock::IrqSafeRwLock}; +use crate::{sync::spin_rwlock::IrqSafeRwLock, waker::QueueWaker}; pub struct OneTimeEvent { // TODO lockless like OneTimeInit? diff --git a/libk/src/util/mod.rs b/libk-util/src/lib.rs similarity index 97% rename from libk/src/util/mod.rs rename to libk-util/src/lib.rs index 50a1cf9e..3f6bd0f5 100644 --- a/libk/src/util/mod.rs +++ b/libk-util/src/lib.rs @@ -1,4 +1,8 @@ -//! Synchronization utilities +#![no_std] +#![feature(maybe_uninit_slice, new_uninit, allocator_api, let_chains)] + +extern crate alloc; + use core::{ cell::UnsafeCell, mem::MaybeUninit, @@ -7,9 +11,13 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; +pub(crate) mod api; + pub mod event; pub mod queue; pub mod ring; +pub mod sync; +pub mod waker; pub enum ConstAssert {} pub trait IsTrue {} diff --git a/libk/src/util/queue.rs b/libk-util/src/queue.rs similarity index 98% rename from libk/src/util/queue.rs rename to libk-util/src/queue.rs index c42dc39f..620048b1 100644 --- a/libk/src/util/queue.rs +++ b/libk-util/src/queue.rs @@ -5,7 +5,7 @@ use core::{ use crossbeam_queue::{ArrayQueue, SegQueue}; -use crate::runtime::QueueWaker; +use crate::waker::QueueWaker; pub type BoundedQueue = ArrayQueue; pub type UnboundedQueue = SegQueue; diff --git a/libk/src/util/ring.rs b/libk-util/src/ring.rs similarity index 99% rename from libk/src/util/ring.rs rename to libk-util/src/ring.rs index cbacddd8..c711043f 100644 --- a/libk/src/util/ring.rs +++ b/libk-util/src/ring.rs @@ -10,8 +10,8 @@ use alloc::boxed::Box; use yggdrasil_abi::error::Error; use crate::{ - runtime::QueueWaker, sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}, + waker::QueueWaker, }; // TODO use lockless ring data structure diff --git a/libk/src/sync/fence.rs b/libk-util/src/sync/fence.rs similarity index 100% rename from libk/src/sync/fence.rs rename to libk-util/src/sync/fence.rs diff --git a/libk/src/sync/guard.rs b/libk-util/src/sync/guard.rs similarity index 100% rename from libk/src/sync/guard.rs rename to libk-util/src/sync/guard.rs diff --git a/libk-util/src/sync/mod.rs b/libk-util/src/sync/mod.rs new file mode 100644 index 00000000..13966f17 --- /dev/null +++ b/libk-util/src/sync/mod.rs @@ -0,0 +1,37 @@ +use core::sync::atomic::{AtomicBool, Ordering}; + +use yggdrasil_abi::error::Error; + +pub mod fence; +pub mod guard; +pub mod spin_rwlock; +pub mod spinlock; + +pub use fence::SpinFence; +pub use guard::IrqGuard; +pub use spinlock::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; + +static LOCK_HACK: AtomicBool = AtomicBool::new(false); + +pub trait LockMethod<'q>: Sync { + type Guard<'a> + where + 'a: 'q, + Self: 'a; + + fn lock(&'q self) -> Result, Error>; + + /// # Safety + /// + /// Only meant to be called from Guard's [Drop] impl. + unsafe fn release(&self); +} + +/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. +/// +/// # Safety +/// +/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. +pub unsafe fn hack_locks() { + LOCK_HACK.store(true, Ordering::Release); +} diff --git a/libk/src/sync/spin_rwlock.rs b/libk-util/src/sync/spin_rwlock.rs similarity index 100% rename from libk/src/sync/spin_rwlock.rs rename to libk-util/src/sync/spin_rwlock.rs diff --git a/libk/src/sync/spinlock.rs b/libk-util/src/sync/spinlock.rs similarity index 100% rename from libk/src/sync/spinlock.rs rename to libk-util/src/sync/spinlock.rs diff --git a/libk-util/src/waker.rs b/libk-util/src/waker.rs new file mode 100644 index 00000000..24b3df55 --- /dev/null +++ b/libk-util/src/waker.rs @@ -0,0 +1,72 @@ +use core::task::Waker; + +use alloc::collections::VecDeque; + +use crate::sync::IrqSafeSpinlock; + +/// Async/await primitive to suspend and wake up tasks waiting on some shared resource +pub struct QueueWaker { + queue: IrqSafeSpinlock>, +} + +impl QueueWaker { + /// Constructs an empty [QueueWaker] + pub const fn new() -> Self { + Self { + queue: IrqSafeSpinlock::new(VecDeque::new()), + } + } + + /// Registers a [Waker] reference to be waken up by this [QueueWaker] + pub fn register(&self, waker: &Waker) { + let mut queue = self.queue.lock(); + + if queue.iter().any(|other| other.will_wake(waker)) { + return; + } + + queue.push_back(waker.clone()); + } + + /// Removes a [Waker] reference from this [QueueWaker] + pub fn remove(&self, waker: &Waker) -> bool { + let mut queue = self.queue.lock(); + let mut index = 0; + let mut removed = false; + + while index < queue.len() { + if queue[index].will_wake(waker) { + removed = true; + queue.remove(index); + } + index += 1; + } + + removed + } + + /// Wakes up up to `limit` tasks waiting on this queue + pub fn wake_some(&self, limit: usize) -> usize { + let mut queue = self.queue.lock(); + let mut count = 0; + + while count < limit + && let Some(item) = queue.pop_front() + { + item.wake(); + count += 1; + } + + count + } + + /// Wakes up a single task waiting on this queue + pub fn wake_one(&self) -> bool { + self.wake_some(1) != 0 + } + + /// Wakes up all tasks waiting on this queue + pub fn wake_all(&self) -> usize { + self.wake_some(usize::MAX) + } +} diff --git a/libk/Cargo.toml b/libk/Cargo.toml index 040bfb8d..a4466c62 100644 --- a/libk/Cargo.toml +++ b/libk/Cargo.toml @@ -8,6 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] libk-mm = { path = "../libk-mm" } +libk-util = { path = "../libk-util" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../lib/device-api", features = ["derive"] } diff --git a/libk/src/api.rs b/libk/src/api.rs index 47343acd..05abfe75 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -11,9 +11,6 @@ use yggdrasil_abi::{ use crate::thread::{CurrentThread, Thread}; extern "Rust" { - pub fn __acquire_irq_guard() -> bool; - pub fn __release_irq_guard(mask: bool); - pub fn __cpu_index() -> usize; pub fn __cpu_count() -> usize; diff --git a/libk/src/lib.rs b/libk/src/lib.rs index 8fa22c36..b69ce580 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -27,7 +27,6 @@ pub(crate) mod api; pub mod runtime; pub mod sync; pub mod thread; -pub mod util; #[inline] pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController { diff --git a/libk/src/runtime/mod.rs b/libk/src/runtime/mod.rs index 8b537a04..6a6659b0 100644 --- a/libk/src/runtime/mod.rs +++ b/libk/src/runtime/mod.rs @@ -10,4 +10,3 @@ mod waker; pub use executor::{run_to_completion, spawn, spawn_async_worker}; pub use task_queue::init_task_queue; pub use timer::{run_with_timeout, sleep, tick, FutureTimeout, SleepFuture}; -pub use waker::QueueWaker; diff --git a/libk/src/runtime/task.rs b/libk/src/runtime/task.rs index 1d5602a1..61125442 100644 --- a/libk/src/runtime/task.rs +++ b/libk/src/runtime/task.rs @@ -2,8 +2,7 @@ use core::fmt; use alloc::sync::Arc; use futures_util::{future::BoxFuture, task::ArcWake, Future, FutureExt}; - -use crate::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use super::executor; diff --git a/libk/src/runtime/task_queue.rs b/libk/src/runtime/task_queue.rs index 18e036f2..c7d01496 100644 --- a/libk/src/runtime/task_queue.rs +++ b/libk/src/runtime/task_queue.rs @@ -1,8 +1,9 @@ use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; +use libk_util::{sync::IrqGuard, OneTimeInit}; use yggdrasil_abi::error::Error; -use crate::{sync::IrqGuard, thread::Thread, util::OneTimeInit}; +use crate::thread::Thread; use super::task::Task; diff --git a/libk/src/runtime/timer.rs b/libk/src/runtime/timer.rs index 88769d11..f573296a 100644 --- a/libk/src/runtime/timer.rs +++ b/libk/src/runtime/timer.rs @@ -6,8 +6,9 @@ use core::{ use alloc::vec::Vec; use futures_util::{future::BoxFuture, Future, FutureExt}; +use libk_util::sync::IrqSafeSpinlock; -use crate::{api, sync::IrqSafeSpinlock}; +use crate::api; // 1..32ms, tick every 1ms static SHORT_TERM_SLEEPS: IrqSafeSpinlock> = diff --git a/libk/src/runtime/waker.rs b/libk/src/runtime/waker.rs index 8ca256b9..14272c5e 100644 --- a/libk/src/runtime/waker.rs +++ b/libk/src/runtime/waker.rs @@ -1,75 +1,8 @@ -use core::task::{RawWaker, RawWakerVTable, Waker}; +use core::task::{RawWaker, RawWakerVTable}; -use alloc::{collections::VecDeque, sync::Weak}; +use alloc::sync::Weak; -use crate::{sync::IrqSafeSpinlock, thread::Thread}; - -/// Async/await primitive to suspend and wake up tasks waiting on some shared resource -pub struct QueueWaker { - queue: IrqSafeSpinlock>, -} - -impl QueueWaker { - /// Constructs an empty [QueueWaker] - pub const fn new() -> Self { - Self { - queue: IrqSafeSpinlock::new(VecDeque::new()), - } - } - - /// Registers a [Waker] reference to be waken up by this [QueueWaker] - pub fn register(&self, waker: &Waker) { - let mut queue = self.queue.lock(); - - if queue.iter().any(|other| other.will_wake(waker)) { - return; - } - - queue.push_back(waker.clone()); - } - - /// Removes a [Waker] reference from this [QueueWaker] - pub fn remove(&self, waker: &Waker) -> bool { - let mut queue = self.queue.lock(); - let mut index = 0; - let mut removed = false; - - while index < queue.len() { - if queue[index].will_wake(waker) { - removed = true; - queue.remove(index); - } - index += 1; - } - - removed - } - - /// Wakes up up to `limit` tasks waiting on this queue - pub fn wake_some(&self, limit: usize) -> usize { - let mut queue = self.queue.lock(); - let mut count = 0; - - while count < limit - && let Some(item) = queue.pop_front() - { - item.wake(); - count += 1; - } - - count - } - - /// Wakes up a single task waiting on this queue - pub fn wake_one(&self) -> bool { - self.wake_some(1) != 0 - } - - /// Wakes up all tasks waiting on this queue - pub fn wake_all(&self) -> usize { - self.wake_some(usize::MAX) - } -} +use crate::thread::Thread; unsafe fn drop_weak_waker(ptr: *const ()) { let weak = Weak::from_raw(ptr as *const Thread); diff --git a/libk/src/sync/mod.rs b/libk/src/sync/mod.rs index 3b8870eb..5e260883 100644 --- a/libk/src/sync/mod.rs +++ b/libk/src/sync/mod.rs @@ -1,39 +1,2 @@ //! Synchronization primitives -use core::sync::atomic::{AtomicBool, Ordering}; - -use yggdrasil_abi::error::Error; - -pub mod fence; -pub mod guard; pub mod mutex; -pub mod spin_rwlock; -pub mod spinlock; - -pub use fence::SpinFence; -pub use guard::IrqGuard; -pub use spinlock::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; - -static LOCK_HACK: AtomicBool = AtomicBool::new(false); - -pub trait LockMethod<'q>: Sync { - type Guard<'a> - where - 'a: 'q, - Self: 'a; - - fn lock(&'q self) -> Result, Error>; - - /// # Safety - /// - /// Only meant to be called from Guard's [Drop] impl. - unsafe fn release(&self); -} - -/// "Hacks" all the locks in the kernel to make them function as "NULL"-locks instead of spinlocks. -/// -/// # Safety -/// -/// Only meant to be called from panic handler when the caller is sure other CPUs are halted. -pub unsafe fn hack_locks() { - LOCK_HACK.store(true, Ordering::Release); -} diff --git a/libk/src/sync/mutex.rs b/libk/src/sync/mutex.rs index 2a97a404..ac4e3308 100644 --- a/libk/src/sync/mutex.rs +++ b/libk/src/sync/mutex.rs @@ -6,12 +6,11 @@ use core::{ use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; +use libk_util::sync::LockMethod; use yggdrasil_abi::error::Error; use crate::{api, thread::Thread}; -use super::LockMethod; - struct ThreadedMutexInner { queue: ArrayQueue>, lock: AtomicU32, diff --git a/libk/src/thread.rs b/libk/src/thread.rs index 6ddb99ce..92742d86 100644 --- a/libk/src/thread.rs +++ b/libk/src/thread.rs @@ -6,9 +6,10 @@ use alloc::{ sync::{Arc, Weak}, }; use futures_util::task::ArcWake; +use libk_util::sync::IrqGuard; use yggdrasil_abi::{error::Error, process::ExitCode}; -use crate::{api, sync::IrqGuard}; +use crate::api; #[repr(C)] pub(crate) struct Thread(!); diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 3214ccee..c20eed41 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -6,9 +6,9 @@ use core::{ use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; -use libk::{ +use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, - util::OneTimeInit, + OneTimeInit, }; use tock_registers::interfaces::{Readable, Writeable}; diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 35739dce..5962b9a1 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -278,7 +278,7 @@ extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) { let iss = esr_el1 & 0x1FFFFFF; unsafe { - libk::sync::hack_locks(); + libk_util::sync::hack_locks(); } dump_irrecoverable_exception(frame, ec, iss); diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index d90c242b..03730947 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -14,11 +14,11 @@ use device_api::{ Device, }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; -use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; use libk_mm::{ address::{FromRaw, PhysicalAddress}, device::{DeviceMemoryIo, RawDeviceMemoryMapping}, }; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, CpuMessage}; diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index 451955ec..b77693e6 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -7,12 +7,12 @@ use core::{ use abi::error::Error; use cfg_if::cfg_if; -use libk::util::OneTimeInit; use libk_mm::{ address::{FromRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, table::{EntryLevel, EntryLevelExt}, }; +use libk_util::OneTimeInit; use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1}; diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 2c2933e2..04f6dab3 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -16,7 +16,6 @@ use device_api::{ }; use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; use git_version::git_version; -use libk::util::OneTimeInit; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, @@ -24,6 +23,7 @@ use libk_mm::{ pointer::PhysicalRef, table::{EntryLevel, EntryLevelExt}, }; +use libk_util::OneTimeInit; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use ygg_driver_pci::PciBusManager; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index cce0b280..d133c3d8 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -30,13 +30,13 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use libk::sync::IrqGuard; use libk_mm::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, phys::PhysicalMemoryRegion, table::EntryLevel, }; +use libk_util::sync::IrqGuard; use crate::task::{sched::CpuQueue, thread::ThreadId, Cpu}; diff --git a/src/debug.rs b/src/debug.rs index 8df4e481..dd97a7e9 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,10 +2,7 @@ use core::fmt::{self, Arguments}; use abi::error::Error; -use libk::{ - sync::IrqSafeSpinlock, - util::{ring::RingBuffer, StaticVector}, -}; +use libk_util::{ring::RingBuffer, sync::IrqSafeSpinlock, StaticVector}; use crate::task::process::Process; diff --git a/src/device/display/console.rs b/src/device/display/console.rs index 67e354db..e5f5fec5 100644 --- a/src/device/display/console.rs +++ b/src/device/display/console.rs @@ -5,7 +5,8 @@ use core::time::Duration; use abi::{error::Error, primitive_enum}; use alloc::{vec, vec::Vec}; use bitflags::bitflags; -use libk::{runtime, sync::IrqSafeSpinlock, util::StaticVector}; +use libk::runtime; +use libk_util::{sync::IrqSafeSpinlock, StaticVector}; use crate::debug::DebugSink; diff --git a/src/device/display/fb_console.rs b/src/device/display/fb_console.rs index 9f91085f..50ad2fab 100644 --- a/src/device/display/fb_console.rs +++ b/src/device/display/fb_console.rs @@ -1,7 +1,7 @@ //! Framebuffer console driver use abi::error::Error; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use crate::debug::DebugSink; diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index f077a120..387647ce 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -7,13 +7,13 @@ use core::{ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; -use libk::sync::IrqSafeSpinlock; use libk_mm::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, table::EntryLevel, PageProvider, }; +use libk_util::sync::IrqSafeSpinlock; use ygg_driver_block::BlockDevice; use crate::{ diff --git a/src/device/input.rs b/src/device/input.rs index 84e6c31e..a2493b5b 100644 --- a/src/device/input.rs +++ b/src/device/input.rs @@ -7,7 +7,8 @@ use abi::{ error::Error, io::{DeviceRequest, KeyboardKeyEvent}, }; -use libk::{block, util::ring::LossyRingQueue}; +use libk::block; +use libk_util::ring::LossyRingQueue; use vfs::{CharDevice, FileReadiness}; pub struct KeyboardDevice; diff --git a/src/device/mod.rs b/src/device/mod.rs index 0734c65f..7046c1c3 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -1,7 +1,7 @@ //! Device management and interfaces use device_api::{manager::DeviceManager, Device, DeviceId}; -use libk::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; +use libk_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; pub mod bus; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 5a88f5f9..cb57042c 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -5,11 +5,12 @@ use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use futures_util::task::{Context, Poll}; use kernel_fs::devfs::{self, CharDeviceType}; -use libk::{block, sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk::block; use libk_mm::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIo, }; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, register_bitfields, register_structs, diff --git a/src/device/tty.rs b/src/device/tty.rs index 3afc787b..2317a348 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -12,7 +12,7 @@ use abi::{ }; use device_api::serial::SerialDevice; use futures_util::Future; -use libk::{runtime::QueueWaker, sync::IrqSafeSpinlock, util::ring::RingBuffer}; +use libk_util::{ring::RingBuffer, sync::IrqSafeSpinlock, waker::QueueWaker}; use crate::task::process::{Process, ProcessId}; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 1d007865..20d1a218 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -3,8 +3,8 @@ use core::ptr::NonNull; use kernel_fs::devfs; -use libk::util::OneTimeInit; use libk_mm::address::PhysicalAddress; +use libk_util::OneTimeInit; use memfs::block::{self, BlockAllocator}; use vfs::{impls::read_fn_node, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 9f792249..2fccba57 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -2,7 +2,7 @@ use abi::error::Error; use git_version::git_version; -use libk::util::OneTimeInit; +use libk_util::OneTimeInit; use vfs::{ impls::{const_value_node, mdir, read_fn_node, ReadOnlyFnValueNode}, NodeRef, diff --git a/src/main.rs b/src/main.rs index c952bff3..385ffed2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,7 +39,7 @@ #![no_main] use arch::Architecture; -use libk::sync::SpinFence; +use libk_util::sync::SpinFence; use crate::{ arch::{ArchitectureImpl, ARCHITECTURE}, diff --git a/src/mem/heap.rs b/src/mem/heap.rs index b9aab1e9..59c58ace 100644 --- a/src/mem/heap.rs +++ b/src/mem/heap.rs @@ -5,7 +5,7 @@ use core::{ ptr::{null_mut, NonNull}, }; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use linked_list_allocator::Heap; /// Kernel heap manager diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index b94275c9..98eb1506 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,11 +1,11 @@ //! Physical memory management utilities use abi::{error::Error, system::SystemMemoryStats}; -use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, }; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use crate::{ arch::{Architecture, ARCHITECTURE}, diff --git a/src/mem/phys/reserved.rs b/src/mem/phys/reserved.rs index 314203a4..5515a936 100644 --- a/src/mem/phys/reserved.rs +++ b/src/mem/phys/reserved.rs @@ -1,7 +1,7 @@ //! Utilities for handling reserved memory regions -use libk::util::StaticVector; use libk_mm::{address::PhysicalAddress, phys::PhysicalMemoryRegion}; +use libk_util::StaticVector; static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); diff --git a/src/mem/process.rs b/src/mem/process.rs index 2b41325e..a7cfe5a1 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -4,13 +4,13 @@ use core::ops::Range; use abi::error::Error; use cfg_if::cfg_if; -use libk::sync::IrqSafeSpinlock; use libk_mm::{ address::PhysicalAddress, pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevelExt, PageProvider, }; +use libk_util::sync::IrqSafeSpinlock; use vfs::FileRef; use vmalloc::{RangeData, VirtualMemoryAllocator}; diff --git a/src/panic.rs b/src/panic.rs index 8f5cfffb..e1c880cd 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use device_api::interrupt::IpiDeliveryTarget; -use libk::sync::{hack_locks, SpinFence}; +use libk_util::sync::{hack_locks, SpinFence}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess, CpuMessage, ARCHITECTURE}, diff --git a/src/proc/random.rs b/src/proc/random.rs index 659ff67d..2f8c3ddb 100644 --- a/src/proc/random.rs +++ b/src/proc/random.rs @@ -1,6 +1,6 @@ //! Random generation utilities -use libk::{sync::IrqSafeSpinlock, util::OneTimeInit}; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use crate::arch::{Architecture, ARCHITECTURE}; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 8baab4c9..edc3d940 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -18,8 +18,9 @@ use abi::{ system::SystemInfo, }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; -use libk::{block, runtime, sync::IrqSafeSpinlockGuard}; +use libk::{block, runtime}; use libk_mm::table::EntryLevelExt; +use libk_util::sync::IrqSafeSpinlockGuard; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; diff --git a/src/task/mod.rs b/src/task/mod.rs index 83675104..e40dc062 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -4,7 +4,8 @@ use abi::error::Error; use alloc::{string::String, vec::Vec}; -use libk::{runtime, sync::SpinFence, thread::Termination}; +use libk::{runtime, thread::Termination}; +use libk_util::sync::SpinFence; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess}, diff --git a/src/task/process.rs b/src/task/process.rs index 5e03b06c..f4b06c3b 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -16,12 +16,12 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; -use libk::{ +use libk_util::{ + event::OneTimeEvent, sync::{ spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, IrqSafeSpinlock, }, - util::event::OneTimeEvent, }; use vfs::{IoContext, NodeRef}; diff --git a/src/task/sched.rs b/src/task/sched.rs index 170c2c55..2e1be619 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -5,7 +5,7 @@ use core::cell::Cell; use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; -use libk::{sync::IrqGuard, util::OneTimeInit}; +use libk_util::{sync::IrqGuard, OneTimeInit}; use crate::{ arch::{Architecture, ArchitectureImpl, CpuAccess}, diff --git a/src/task/sync.rs b/src/task/sync.rs index 76207a77..262261ad 100644 --- a/src/task/sync.rs +++ b/src/task/sync.rs @@ -8,7 +8,7 @@ use core::{ use alloc::sync::Arc; use futures_util::Future; -use libk::runtime::QueueWaker; +use libk_util::waker::QueueWaker; /// User-space mutex (like BSD/Linux's futex) data structure pub struct UserspaceMutex { diff --git a/src/task/thread.rs b/src/task/thread.rs index 933c17d5..c55563cd 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -15,10 +15,10 @@ use abi::{ use alloc::{collections::BTreeMap, string::String, sync::Arc}; use atomic_enum::atomic_enum; use crossbeam_queue::SegQueue; -use libk::{ - block, +use libk::block; +use libk_util::{ + event::BoolEvent, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, - util::event::BoolEvent, }; use crate::{ From 72b8967339bf8a6c768aa9606a448d2c4a6de61f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 5 Feb 2024 12:59:23 +0200 Subject: [PATCH 179/211] libk: move waker to libk-util --- driver/block/nvme/Cargo.toml | 2 + driver/block/nvme/src/command.rs | 2 +- driver/block/nvme/src/drive.rs | 4 +- driver/block/nvme/src/lib.rs | 15 +++--- driver/block/nvme/src/queue.rs | 11 ++--- libk-util/Cargo.toml | 1 + libk-util/src/waker.rs | 66 +++++++++++++++++++++++++- libk/src/runtime/executor.rs | 5 +- libk/src/runtime/mod.rs | 1 - libk/src/runtime/waker.rs | 57 ---------------------- src/arch/x86_64/acpi.rs | 11 ++--- src/arch/x86_64/apic/ioapic.rs | 10 ++-- src/arch/x86_64/apic/local.rs | 14 +++--- src/arch/x86_64/context.rs | 2 +- src/arch/x86_64/cpu.rs | 4 +- src/arch/x86_64/cpuid.rs | 2 +- src/arch/x86_64/mem/mod.rs | 12 ++--- src/arch/x86_64/mem/process.rs | 2 +- src/arch/x86_64/mem/table.rs | 2 +- src/arch/x86_64/mod.rs | 16 +++---- src/arch/x86_64/peripherals/i8253.rs | 3 +- src/arch/x86_64/peripherals/ps2/mod.rs | 2 +- src/arch/x86_64/peripherals/serial.rs | 2 +- src/arch/x86_64/smp.rs | 2 +- 24 files changed, 123 insertions(+), 125 deletions(-) delete mode 100644 libk/src/runtime/waker.rs diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml index 8034c6ec..e3eb4638 100644 --- a/driver/block/nvme/Cargo.toml +++ b/driver/block/nvme/Cargo.toml @@ -9,6 +9,8 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk = { path = "../../../libk" } +libk-util = { path = "../../../libk-util" } +libk-mm = { path = "../../../libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/nvme/src/command.rs b/driver/block/nvme/src/command.rs index 20473935..3eb4dbcc 100644 --- a/driver/block/nvme/src/command.rs +++ b/driver/block/nvme/src/command.rs @@ -2,7 +2,7 @@ use core::fmt::{self, Write}; -use libk::mem::address::PhysicalAddress; +use libk_mm::address::PhysicalAddress; use tock_registers::{interfaces::Readable, register_structs, registers::ReadOnly, UIntLike}; use crate::queue::PhysicalRegionPage; diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index 7a0c5209..68e30ff6 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -2,7 +2,9 @@ use core::task::Poll; use alloc::{boxed::Box, format}; use kernel_fs::devfs; -use libk::{cpu_index, mem::address::AsPhysicalAddress, runtime::QueueWaker}; +use libk::cpu_index; +use libk_mm::address::AsPhysicalAddress; +use libk_util::waker::QueueWaker; use ygg_driver_block::{ probe_partitions, IoOperation, IoRequest, IoSubmissionId, NgBlockDevice, NgBlockDeviceWrapper, }; diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index acada38c..cc70209a 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -17,15 +17,14 @@ use device_api::{ Device, }; use drive::NvmeDrive; -use libk::{ - cpu_count, cpu_index, - mem::{ - address::{IntoRaw, PhysicalAddress}, - device::DeviceMemoryIo, - }, - runtime, +use libk::{cpu_count, cpu_index, runtime}; +use libk_mm::{ + address::{IntoRaw, PhysicalAddress}, + device::DeviceMemoryIo, +}; +use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, - util::OneTimeInit, + OneTimeInit, }; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, diff --git a/driver/block/nvme/src/queue.rs b/driver/block/nvme/src/queue.rs index 38beb468..f99afc9b 100644 --- a/driver/block/nvme/src/queue.rs +++ b/driver/block/nvme/src/queue.rs @@ -11,14 +11,11 @@ use alloc::{ }; use bytemuck::{Pod, Zeroable}; use futures_util::Future; -use libk::{ - mem::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, - PageBox, - }, - runtime::QueueWaker, - sync::IrqSafeSpinlock, +use libk_mm::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + PageBox, }; +use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker}; use static_assertions::const_assert; use yggdrasil_abi::error::Error; diff --git a/libk-util/Cargo.toml b/libk-util/Cargo.toml index df98c82c..e019bed6 100644 --- a/libk-util/Cargo.toml +++ b/libk-util/Cargo.toml @@ -10,3 +10,4 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } log = "0.4.20" crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/libk-util/src/waker.rs b/libk-util/src/waker.rs index 24b3df55..3345f682 100644 --- a/libk-util/src/waker.rs +++ b/libk-util/src/waker.rs @@ -1,6 +1,10 @@ -use core::task::Waker; +use core::{ + marker::PhantomData, + task::{RawWaker, RawWakerVTable, Waker}, +}; -use alloc::collections::VecDeque; +use alloc::{collections::VecDeque, sync::Weak}; +use futures_util::task::ArcWake; use crate::sync::IrqSafeSpinlock; @@ -9,6 +13,64 @@ pub struct QueueWaker { queue: IrqSafeSpinlock>, } +pub struct WakeWeak { + _pd: PhantomData, +} + +impl WakeWeak { + pub const VTABLE: RawWakerVTable = RawWakerVTable::new( + Self::clone_waker, + Self::wake, + Self::wake_by_ref, + Self::drop_waker, + ); + + unsafe fn drop_waker(ptr: *const ()) { + let weak = Weak::from_raw(ptr as *const T); + + // Explicitly drop the thing + drop(weak); + } + + unsafe fn wake(ptr: *const ()) { + let weak = Weak::from_raw(ptr as *const T); + + if let Some(strong) = weak.upgrade() { + strong.wake(); + } + + // Weak gets dropped + } + + unsafe fn wake_by_ref(ptr: *const ()) { + let weak = Weak::from_raw(ptr as *const T); + + if let Some(strong) = weak.upgrade() { + strong.wake(); + } + + // Weak doesn't get dropped + core::mem::forget(weak); + } + + unsafe fn clone_waker(ptr: *const ()) -> RawWaker { + let weak = Weak::from_raw(ptr as *const T); + + let waker = Self::weak_waker(weak.clone()); + + // Prevent decrement of the refcount + core::mem::forget(weak); + + waker + } + + pub fn weak_waker(weak: Weak) -> RawWaker { + let raw = Weak::into_raw(weak); + + RawWaker::new(raw as *const (), &Self::VTABLE) + } +} + impl QueueWaker { /// Constructs an empty [QueueWaker] pub const fn new() -> Self { diff --git a/libk/src/runtime/executor.rs b/libk/src/runtime/executor.rs index a6aadfb0..486e02ec 100644 --- a/libk/src/runtime/executor.rs +++ b/libk/src/runtime/executor.rs @@ -2,13 +2,14 @@ use core::task::{Context, Poll, Waker}; use alloc::{boxed::Box, format, sync::Arc}; use futures_util::{task::waker_ref, Future}; +use libk_util::waker::WakeWeak; use yggdrasil_abi::error::Error; use crate::thread::Thread; use super::{ task::{Task, Termination}, - task_queue, waker, + task_queue, }; /// Pushes a task into the executor's queue @@ -49,7 +50,7 @@ pub fn run_to_completion<'a, T, F: Future + Send + 'a>(future: F) -> // Make a weak ref for the waker let weak = Thread::current().downgrade(); - let waker = waker::weak_thread_waker(weak.clone()); + let waker = WakeWeak::weak_waker(weak.clone()); let waker = unsafe { Waker::from_raw(waker) }; loop { diff --git a/libk/src/runtime/mod.rs b/libk/src/runtime/mod.rs index 6a6659b0..25248abf 100644 --- a/libk/src/runtime/mod.rs +++ b/libk/src/runtime/mod.rs @@ -5,7 +5,6 @@ mod executor; mod task; mod task_queue; mod timer; -mod waker; pub use executor::{run_to_completion, spawn, spawn_async_worker}; pub use task_queue::init_task_queue; diff --git a/libk/src/runtime/waker.rs b/libk/src/runtime/waker.rs deleted file mode 100644 index 14272c5e..00000000 --- a/libk/src/runtime/waker.rs +++ /dev/null @@ -1,57 +0,0 @@ -use core::task::{RawWaker, RawWakerVTable}; - -use alloc::sync::Weak; - -use crate::thread::Thread; - -unsafe fn drop_weak_waker(ptr: *const ()) { - let weak = Weak::from_raw(ptr as *const Thread); - - // Explicitly drop the thing - drop(weak); -} - -unsafe fn wake_weak_waker(ptr: *const ()) { - let weak = Weak::from_raw(ptr as *const Thread); - - if let Some(strong) = weak.upgrade() { - strong.enqueue(); - } - - // Weak gets dropped -} - -unsafe fn wake_by_ref_weak_waker(ptr: *const ()) { - let weak = Weak::from_raw(ptr as *const Thread); - - if let Some(strong) = weak.upgrade() { - strong.enqueue(); - } - - // Weak doesn't get dropped - core::mem::forget(weak); -} - -unsafe fn clone_weak_waker(ptr: *const ()) -> RawWaker { - let weak = Weak::from_raw(ptr as *const Thread); - - let waker = weak_thread_waker(weak.clone()); - - // Prevent decrement of the refcount - core::mem::forget(weak); - - waker -} - -pub fn weak_thread_waker(weak: Weak) -> RawWaker { - const VTABLE: RawWakerVTable = RawWakerVTable::new( - clone_weak_waker, - wake_weak_waker, - wake_by_ref_weak_waker, - drop_weak_waker, - ); - - let raw = Weak::into_raw(weak); - - RawWaker::new(raw as *const (), &VTABLE) -} diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 0685ef01..b679e643 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -15,14 +15,11 @@ use device_api::{ interrupt::{InterruptHandler, IpiDeliveryTarget}, Device, }; -use libk::{ - mem::{ - address::{FromRaw, PhysicalAddress}, - pointer::PhysicalRef, - }, - sync::IrqSafeSpinlock, - util::OneTimeInit, +use libk_mm::{ + address::{FromRaw, PhysicalAddress}, + pointer::PhysicalRef, }; +use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use yggdrasil_abi::error::Error; use crate::{ diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 2649bf0a..f68ff4b8 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -8,13 +8,11 @@ use device_api::{ }, Device, }; -use libk::{ - mem::{ - address::{FromRaw, PhysicalAddress}, - device::DeviceMemoryIo, - }, - sync::IrqSafeSpinlock, +use libk_mm::{ + address::{FromRaw, PhysicalAddress}, + device::DeviceMemoryIo, }; +use libk_util::sync::IrqSafeSpinlock; use tock_registers::{ interfaces::{Readable, Writeable}, register_structs, diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 88e5f0ec..a987ef2e 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -10,14 +10,14 @@ use device_api::{ }, Device, }; -use libk::{ - mem::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, - device::DeviceMemoryIo, - table::EntryLevelExt, - }, +use libk_mm::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::DeviceMemoryIo, + table::EntryLevelExt, +}; +use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, - util::OneTimeInit, + OneTimeInit, }; use tock_registers::{ interfaces::{ReadWriteable, Readable, Writeable}, diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 44c3a66e..5ab83730 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -3,7 +3,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use libk::mem::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; +use libk_mm::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; use crate::{ arch::x86_64::mem::KERNEL_TABLES, diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index d80b8ba1..0ce96b0b 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -6,9 +6,9 @@ use core::{ }; use alloc::{boxed::Box, vec::Vec}; -use libk::{ +use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, - util::OneTimeInit, + OneTimeInit, }; use tock_registers::interfaces::Writeable; diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index 536ea72c..d59532e2 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -1,7 +1,7 @@ //! x86-64 CPUID interface use bitflags::bitflags; -use libk::util::OneTimeInit; +use libk_util::OneTimeInit; use tock_registers::interfaces::ReadWriteable; use super::registers::{CR4, XCR0}; diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index 3a52780d..a496e97b 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -6,14 +6,12 @@ use core::{ }; use abi::error::Error; -use libk::{ - mem::{ - address::{FromRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - table::EntryLevelExt, - }, - util::OneTimeInit, +use libk_mm::{ + address::{FromRaw, PhysicalAddress}, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + table::EntryLevelExt, }; +use libk_util::OneTimeInit; use memtables::FixedTables; use static_assertions::{const_assert_eq, const_assert_ne}; diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index 9e689dc5..a6bac2ea 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,5 +1,5 @@ //! x86-64-specific process address space management functions -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, table::EntryLevelExt, diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index 7d1f14e5..b74329ff 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -6,7 +6,7 @@ use core::{ use abi::error::Error; use bitflags::bitflags; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, }; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index eccc5e6c..7506261c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -11,15 +11,13 @@ use device_api::{ }; use git_version::git_version; use kernel_fs::devfs; -use libk::{ - mem::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - table::EntryLevelExt, - }, - sync::SpinFence, - util::OneTimeInit, +use libk_mm::{ + address::{FromRaw, IntoRaw, PhysicalAddress}, + device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + phys::PhysicalMemoryRegion, + table::EntryLevelExt, }; +use libk_util::{sync::SpinFence, OneTimeInit}; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; use ygg_driver_pci::PciBusManager; @@ -55,7 +53,7 @@ use crate::{ fs::{Initrd, INITRD_DATA}, mem::{ heap, - phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, + phys::{self, reserved::reserve_region}, table::EntryLevel, }, }; diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index 1964084b..3ee5357d 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -2,7 +2,8 @@ use core::time::Duration; use abi::error::Error; use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; -use libk::{runtime, sync::IrqSafeSpinlock}; +use libk::runtime; +use libk_util::sync::IrqSafeSpinlock; use crate::arch::{ x86_64::{ diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index f8d836de..77964744 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -4,7 +4,7 @@ use abi::{ io::{KeyboardKey, KeyboardKeyEvent}, }; use device_api::{interrupt::InterruptHandler, Device}; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use crate::{ arch::{ diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index e4dcd91b..8d6de2f3 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -1,7 +1,7 @@ //! Driver for x86 COM ports use abi::error::Error; use device_api::{serial::SerialDevice, Device}; -use libk::sync::IrqSafeSpinlock; +use libk_util::sync::IrqSafeSpinlock; use crate::{ arch::x86_64::{ diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index e4d0d1b5..d1066e66 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -2,7 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; -use libk::mem::{ +use libk_mm::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, }; From b653ec3c4a51fbeb2e154efbb2fd171b522371d0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 5 Feb 2024 13:44:21 +0200 Subject: [PATCH 180/211] libk: move trivial arch stuff to kernel-arch --- Cargo.toml | 2 + arch/Cargo.toml | 17 ++++++ arch/aarch64/Cargo.toml | 12 ++++ arch/aarch64/src/lib.rs | 27 +++++++++ arch/interface/Cargo.toml | 8 +++ arch/interface/src/lib.rs | 9 +++ arch/src/lib.rs | 34 ++++++++++++ arch/x86_64/Cargo.toml | 9 +++ arch/x86_64/src/lib.rs | 33 +++++++++++ libk-util/Cargo.toml | 1 + libk-util/src/api.rs | 4 -- libk-util/src/lib.rs | 2 - libk-util/src/sync/guard.rs | 9 ++- libk/src/runtime/task_queue.rs | 2 +- src/arch/aarch64/boot/mod.rs | 18 +++--- src/arch/aarch64/exception.rs | 7 ++- src/arch/aarch64/gic/mod.rs | 10 ++-- src/arch/aarch64/mod.rs | 22 +------- src/arch/aarch64/smp.rs | 4 +- src/arch/aarch64/timer.rs | 6 +- src/arch/mod.rs | 76 +++++--------------------- src/arch/x86_64/acpi.rs | 6 +- src/arch/x86_64/apic/mod.rs | 8 +-- src/arch/x86_64/boot/mod.rs | 10 ++-- src/arch/x86_64/exception.rs | 4 +- src/arch/x86_64/mod.rs | 29 +--------- src/arch/x86_64/peripherals/i8253.rs | 4 +- src/arch/x86_64/peripherals/ps2/mod.rs | 4 +- src/arch/x86_64/smp.rs | 3 +- src/device/power/arm_psci.rs | 5 +- src/device/power/sunxi_rwdog.rs | 4 +- src/device/serial/pl011.rs | 4 +- src/device/serial/sunxi_uart.rs | 4 +- src/main.rs | 10 ++-- src/mem/mod.rs | 4 +- src/mem/phys/mod.rs | 5 +- src/panic.rs | 9 +-- src/proc/random.rs | 4 +- src/task/mod.rs | 4 +- src/task/sched.rs | 6 +- 40 files changed, 254 insertions(+), 185 deletions(-) create mode 100644 arch/Cargo.toml create mode 100644 arch/aarch64/Cargo.toml create mode 100644 arch/aarch64/src/lib.rs create mode 100644 arch/interface/Cargo.toml create mode 100644 arch/interface/src/lib.rs create mode 100644 arch/src/lib.rs create mode 100644 arch/x86_64/Cargo.toml create mode 100644 arch/x86_64/src/lib.rs delete mode 100644 libk-util/src/api.rs diff --git a/Cargo.toml b/Cargo.toml index 47c6e1fc..084d3dd8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } +kernel-arch = { path = "arch" } + # Drivers ygg_driver_pci = { path = "driver/bus/pci" } ygg_driver_block = { path = "driver/block/core" } diff --git a/arch/Cargo.toml b/arch/Cargo.toml new file mode 100644 index 00000000..0ac2a342 --- /dev/null +++ b/arch/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "kernel-arch" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[target.'cfg(target_arch = "x86_64")'.dependencies] +kernel-arch-x86_64 = { path = "x86_64" } + +[target.'cfg(target_arch = "aarch64")'.dependencies] +kernel-arch-aarch64 = { path = "aarch64" } + +[dependencies] +kernel-arch-interface = { path = "interface" } + +cfg-if = "1.0.0" diff --git a/arch/aarch64/Cargo.toml b/arch/aarch64/Cargo.toml new file mode 100644 index 00000000..50af2b8a --- /dev/null +++ b/arch/aarch64/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kernel-arch-aarch64" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +kernel-arch-interface = { path = "../interface" } + +aarch64-cpu = "9.3.1" +tock-registers = "0.8.1" diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs new file mode 100644 index 00000000..7f85495b --- /dev/null +++ b/arch/aarch64/src/lib.rs @@ -0,0 +1,27 @@ +#![no_std] + +use aarch64_cpu::registers::DAIF; +use kernel_arch_interface::Architecture; +use tock_registers::interfaces::{ReadWriteable, Readable}; + +pub struct ArchitectureImpl; + +impl Architecture for ArchitectureImpl { + fn interrupt_mask() -> bool { + DAIF.read(DAIF::I) != 0 + } + + unsafe fn set_interrupt_mask(mask: bool) -> bool { + let old = Self::interrupt_mask(); + if mask { + DAIF.modify(DAIF::I::SET); + } else { + DAIF.modify(DAIF::I::CLEAR); + } + old + } + + fn wait_for_interrupt() { + aarch64_cpu::asm::wfi(); + } +} diff --git a/arch/interface/Cargo.toml b/arch/interface/Cargo.toml new file mode 100644 index 00000000..8714e738 --- /dev/null +++ b/arch/interface/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "kernel-arch-interface" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs new file mode 100644 index 00000000..f1aed0c8 --- /dev/null +++ b/arch/interface/src/lib.rs @@ -0,0 +1,9 @@ +#![no_std] + +pub trait Architecture { + fn interrupt_mask() -> bool; + + unsafe fn set_interrupt_mask(mask: bool) -> bool; + + fn wait_for_interrupt(); +} diff --git a/arch/src/lib.rs b/arch/src/lib.rs new file mode 100644 index 00000000..1a53ef3e --- /dev/null +++ b/arch/src/lib.rs @@ -0,0 +1,34 @@ +#![no_std] + +use cfg_if::cfg_if; + +pub use kernel_arch_interface::Architecture; + +/// Returns an absolute address to the given symbol +#[macro_export] +macro_rules! absolute_address { + ($sym:expr) => {{ + let mut _x: usize; + #[cfg(target_arch = "aarch64")] + unsafe { + core::arch::asm!("ldr {0}, ={1}", out(reg) _x, sym $sym); + } + #[cfg(target_arch = "x86_64")] + unsafe { + core::arch::asm!("movabsq ${1}, {0}", out(reg) _x, sym $sym, options(att_syntax)); + } + _x + }}; +} + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + extern crate kernel_arch_aarch64 as imp; + } else if #[cfg(target_arch = "x86_64")] { + extern crate kernel_arch_x86_64 as imp; + } else { + compile_error!("Unsupported architecture"); + } +} + +pub use imp::ArchitectureImpl; diff --git a/arch/x86_64/Cargo.toml b/arch/x86_64/Cargo.toml new file mode 100644 index 00000000..9501db4a --- /dev/null +++ b/arch/x86_64/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "kernel-arch-x86_64" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +kernel-arch-interface = { path = "../interface" } diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs new file mode 100644 index 00000000..bff7e580 --- /dev/null +++ b/arch/x86_64/src/lib.rs @@ -0,0 +1,33 @@ +#![no_std] + +use kernel_arch_interface::Architecture; + +pub struct ArchitectureImpl; + +impl Architecture for ArchitectureImpl { + fn interrupt_mask() -> bool { + let mut flags: u64; + unsafe { + core::arch::asm!("pushfq; pop {0}", out(reg) flags, options(att_syntax)); + } + // If IF is zero, interrupts are disabled (masked) + flags & (1 << 9) == 0 + } + + unsafe fn set_interrupt_mask(mask: bool) -> bool { + let old = Self::interrupt_mask(); + if mask { + core::arch::asm!("cli"); + } else { + core::arch::asm!("sti"); + } + old + } + + #[inline] + fn wait_for_interrupt() { + unsafe { + core::arch::asm!("hlt"); + } + } +} diff --git a/libk-util/Cargo.toml b/libk-util/Cargo.toml index e019bed6..fe3e89b2 100644 --- a/libk-util/Cargo.toml +++ b/libk-util/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-arch = { path = "../arch" } log = "0.4.20" crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } diff --git a/libk-util/src/api.rs b/libk-util/src/api.rs deleted file mode 100644 index 85f8d31f..00000000 --- a/libk-util/src/api.rs +++ /dev/null @@ -1,4 +0,0 @@ -extern "Rust" { - pub fn __acquire_irq_guard() -> bool; - pub fn __release_irq_guard(mask: bool); -} diff --git a/libk-util/src/lib.rs b/libk-util/src/lib.rs index 3f6bd0f5..1e7aec4b 100644 --- a/libk-util/src/lib.rs +++ b/libk-util/src/lib.rs @@ -11,8 +11,6 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; -pub(crate) mod api; - pub mod event; pub mod queue; pub mod ring; diff --git a/libk-util/src/sync/guard.rs b/libk-util/src/sync/guard.rs index d2db9036..8b4625a2 100644 --- a/libk-util/src/sync/guard.rs +++ b/libk-util/src/sync/guard.rs @@ -1,4 +1,4 @@ -use crate::api; +use kernel_arch::{Architecture, ArchitectureImpl}; /// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation /// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. @@ -8,12 +8,15 @@ pub struct IrqGuard(bool); impl IrqGuard { /// Saves the current IRQ state and masks them pub fn acquire() -> Self { - Self(unsafe { api::__acquire_irq_guard() }) + let mask = unsafe { ArchitectureImpl::set_interrupt_mask(true) }; + Self(mask) } } impl Drop for IrqGuard { fn drop(&mut self) { - unsafe { api::__release_irq_guard(self.0) } + unsafe { + ArchitectureImpl::set_interrupt_mask(self.0); + } } } diff --git a/libk/src/runtime/task_queue.rs b/libk/src/runtime/task_queue.rs index c7d01496..fa5cf6fe 100644 --- a/libk/src/runtime/task_queue.rs +++ b/libk/src/runtime/task_queue.rs @@ -41,7 +41,7 @@ impl TaskQueue { pub fn dequeue(&self) -> Result, Error> { let thread = Thread::current(); - // assert!(ArchitectureImpl::interrupt_mask()); + // assert!(PlatformImpl::interrupt_mask()); loop { if let Some(task) = self.task_queue.pop() { return Ok(task); diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index f17d608d..131ea423 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -5,6 +5,8 @@ use aarch64_cpu::{ asm::barrier, registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; +use kernel_arch::absolute_address; +use kernel_arch::{Architecture, ArchitectureImpl}; use kernel_fs::devfs; use libk::runtime; use libk_mm::{ @@ -14,11 +16,11 @@ use libk_mm::{ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use super::{ - exception, mem::load_fixed_tables, smp::CPU_COUNT, AArch64, BootStack, ARCHITECTURE, - BOOT_STACK_SIZE, + exception, mem::load_fixed_tables, smp::CPU_COUNT, AArch64, BootStack, BOOT_STACK_SIZE, + PLATFORM, }; use crate::{ - arch::{aarch64::mem::table::L3, Architecture}, + arch::{aarch64::mem::table::L3, Platform}, kernel_main, kernel_secondary_main, mem::{phys, KERNEL_VIRT_OFFSET}, }; @@ -75,7 +77,7 @@ unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! { // NOTE executes in "lower-half" address space, MMU not yet enabled unsafe extern "C" fn __aarch64_el1_bsp_lower_entry(dtb: PhysicalAddress) -> ! { - AArch64::set_interrupt_mask(true); + ArchitectureImpl::set_interrupt_mask(true); // Don't trap FP operations CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); @@ -98,7 +100,7 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { TTBR0_EL1.set(0); // Setup the "runtime" part of the kernel tables - ARCHITECTURE + PLATFORM .init_memory_management(dtb) .expect("Could not initialize memory management"); barrier::isb(barrier::SY); @@ -110,7 +112,7 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { runtime::init_task_queue(); // Initialize the BSP CPU + the devices - ARCHITECTURE + PLATFORM .init_platform(true) .expect("Could not initialize the platform"); @@ -119,7 +121,7 @@ unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { unsafe extern "C" fn __aarch64_el1_ap_lower_entry() -> ! { const AP_STACK_PAGES: usize = 8; - AArch64::set_interrupt_mask(true); + ArchitectureImpl::set_interrupt_mask(true); // Unmask FP operations CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); @@ -149,7 +151,7 @@ extern "C" fn __aarch64_ap_upper_entry() -> ! { exception::init_exceptions(); unsafe { - ARCHITECTURE + PLATFORM .init_platform(false) .expect("Could not initialize the AP"); } diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 5962b9a1..b784f86f 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -13,16 +13,17 @@ use abi::{ process::{Signal, SignalEntryData}, syscall::SyscallFunction, }; +use kernel_arch::{Architecture, ArchitectureImpl}; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{Architecture, ArchitectureImpl}, + arch::{Platform, PlatformImpl}, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, thread::Thread}, }; -use super::ARCHITECTURE; +use super::PLATFORM; /// Struct for register values saved when taking an exception #[repr(C)] @@ -351,7 +352,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { } fn irq_common() { - ARCHITECTURE + PLATFORM .external_interrupt_controller() .handle_pending_irqs(); } diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 03730947..46c0c0d1 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -20,11 +20,11 @@ use libk_mm::{ }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, CpuMessage}; +use crate::arch::{aarch64::IrqNumber, CpuAccess, CpuMessage, Platform}; use self::{gicc::Gicc, gicd::Gicd}; -use super::{cpu::Cpu, smp::CPU_COUNT, ARCHITECTURE}; +use super::{cpu::Cpu, smp::CPU_COUNT, PLATFORM}; const MAX_IRQ: usize = 300; const IPI_VECTOR: u64 = 1; @@ -68,9 +68,9 @@ impl Device for Gic { self.gicd.init(gicd); self.gicc.init(gicc); - ARCHITECTURE.register_external_interrupt_controller(self)?; - ARCHITECTURE.register_local_interrupt_controller(self)?; - ARCHITECTURE.register_message_interrupt_controller(self)?; + PLATFORM.register_external_interrupt_controller(self)?; + PLATFORM.register_local_interrupt_controller(self)?; + PLATFORM.register_message_interrupt_controller(self)?; Ok(()) } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 04f6dab3..910a0fed 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -43,7 +43,7 @@ use crate::{ use self::mem::{table::L1, EarlyMapping}; -use super::{Architecture, CpuMessage}; +use super::{CpuMessage, Platform}; pub mod boot; pub mod context; @@ -78,7 +78,7 @@ pub struct AArch64 { initrd: OneTimeInit>, } -impl Architecture for AArch64 { +impl Platform for AArch64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; type IrqNumber = IrqNumber; @@ -96,22 +96,6 @@ impl Architecture for AArch64 { smp::CPU_COUNT.load(Ordering::Acquire) } - unsafe fn set_interrupt_mask(mask: bool) { - if mask { - DAIF.modify(DAIF::I::SET); - } else { - DAIF.modify(DAIF::I::CLEAR); - } - } - - fn interrupt_mask() -> bool { - DAIF.read(DAIF::I) != 0 - } - - fn wait_for_interrupt() { - aarch64_cpu::asm::wfi(); - } - unsafe fn map_device_memory( &self, base: PhysicalAddress, @@ -455,7 +439,7 @@ impl AArch64 { } /// AArch64 implementation value -pub static ARCHITECTURE: AArch64 = AArch64 { +pub static PLATFORM: AArch64 = AArch64 { dt: OneTimeInit::new(), initrd: OneTimeInit::new(), diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index f3144442..829793e4 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -5,7 +5,7 @@ use abi::error::Error; use device_api::CpuBringupDevice; use device_tree::dt::{DevTreeIndexNodePropGet, DeviceTree}; -use crate::arch::ARCHITECTURE; +use crate::arch::PLATFORM; use crate::mem::KERNEL_VIRT_OFFSET; use super::{BootStack, BOOT_STACK_SIZE}; @@ -50,7 +50,7 @@ impl CpuEnableMethod { unsafe fn start_cpu(&self, id: usize, ip: usize, sp: usize) -> Result<(), Error> { match self { Self::Psci => { - let psci = ARCHITECTURE.psci.try_get().ok_or_else(|| { + let psci = PLATFORM.psci.try_get().ok_or_else(|| { warnln!( "cpu{} has to be enabled through PSCI, but no PSCI found", id diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index b20087d2..d06113d4 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -10,7 +10,7 @@ use device_tree::device_tree_driver; use libk::runtime; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use crate::arch::{aarch64::IrqNumber, Architecture, CpuAccess, ARCHITECTURE}; +use crate::arch::{aarch64::IrqNumber, CpuAccess, Platform, PLATFORM}; use super::cpu::Cpu; @@ -53,12 +53,12 @@ impl Device for ArmTimer { unsafe fn init(&'static self) -> Result<(), Error> { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); - ARCHITECTURE.register_monotonic_timer(self)?; + PLATFORM.register_monotonic_timer(self)?; Ok(()) } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); intc.register_irq(self.irq, Default::default(), self)?; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index d133c3d8..51532302 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -4,23 +4,6 @@ use core::{ops::DerefMut, time::Duration}; use abi::error::Error; -/// Returns an absolute address to the given symbol -#[macro_export] -macro_rules! absolute_address { - ($sym:expr) => {{ - let mut _x: usize; - #[cfg(target_arch = "aarch64")] - unsafe { - core::arch::asm!("ldr {0}, ={1}", out(reg) _x, sym $sym); - } - #[cfg(target_arch = "x86_64")] - unsafe { - core::arch::asm!("movabsq ${1}, {0}", out(reg) _x, sym $sym, options(att_syntax)); - } - _x - }}; -} - use cfg_if::cfg_if; use device_api::{ interrupt::{ @@ -30,6 +13,7 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk_mm::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, @@ -44,18 +28,18 @@ cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod aarch64; - pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; + pub use aarch64::{AArch64 as PlatformImpl, PLATFORM}; } else if #[cfg(target_arch = "x86_64")] { pub mod x86_64; - pub use x86_64::{X86_64 as ArchitectureImpl, ARCHITECTURE}; + pub use x86_64::{X86_64 as PlatformImpl, PLATFORM}; } else { compile_error!("Architecture is not supported"); } } /// Architecture-specific lowest level of page mapping -pub type L3 = ::L3; +pub type L3 = ::L3; // Architecture interfaces @@ -72,7 +56,7 @@ pub enum CpuMessage { /// Interface for an architecture-specific facilities #[allow(unused)] -pub trait Architecture { +pub trait Platform { /// Address, to which "zero" address is mapped in the virtual address space const KERNEL_VIRT_OFFSET: usize; @@ -125,20 +109,6 @@ pub trait Architecture { // Architecture intrinsics - /// Suspends CPU until an interrupt is received - fn wait_for_interrupt(); - - /// Sets the local CPU's interrupt mask. - /// - /// # Safety - /// - /// Enabling interrupts may lead to unexpected behavior unless the context explicitly expects - /// them. - unsafe fn set_interrupt_mask(mask: bool); - - /// Returns the local CPU's interrupt mask - fn interrupt_mask() -> bool; - /// Returns the count of present CPUs, including the BSP fn cpu_count() -> usize; @@ -225,9 +195,9 @@ pub trait Architecture { /// The caller must ensure it is actually safe to reset, i.e. no critical processes will be /// aborted and no data will be lost. unsafe fn reset(&self) -> ! { - Self::set_interrupt_mask(true); + ArchitectureImpl::set_interrupt_mask(true); loop { - Self::wait_for_interrupt(); + ArchitectureImpl::wait_for_interrupt(); } } } @@ -300,22 +270,6 @@ pub trait CpuAccess: Sized { // External API for architecture specifics -#[no_mangle] -fn __acquire_irq_guard() -> bool { - let mask = ArchitectureImpl::interrupt_mask(); - unsafe { - ArchitectureImpl::set_interrupt_mask(true); - } - mask -} - -#[no_mangle] -fn __release_irq_guard(mask: bool) { - unsafe { - ArchitectureImpl::set_interrupt_mask(mask); - } -} - #[no_mangle] fn __cpu_index() -> usize { Cpu::local().id() as _ @@ -323,17 +277,17 @@ fn __cpu_index() -> usize { #[no_mangle] fn __cpu_count() -> usize { - ArchitectureImpl::cpu_count() + PlatformImpl::cpu_count() } #[no_mangle] fn __virtualize(addr: u64) -> usize { - ArchitectureImpl::virtualize(addr) + PlatformImpl::virtualize(addr) } #[no_mangle] fn __physicalize(addr: usize) -> u64 { - ArchitectureImpl::physicalize(addr) + PlatformImpl::physicalize(addr) } #[no_mangle] @@ -342,21 +296,21 @@ fn __map_device_pages( count: usize, attrs: DeviceMemoryAttributes, ) -> Result { - unsafe { ARCHITECTURE.map_device_memory(base, count, attrs) } + unsafe { PLATFORM.map_device_memory(base, count, attrs) } } #[no_mangle] fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping) { - unsafe { ARCHITECTURE.unmap_device_memory(mapping) } + unsafe { PLATFORM.unmap_device_memory(mapping) } } #[no_mangle] fn __monotonic_timestamp() -> Result { - ARCHITECTURE.monotonic_timer().monotonic_timestamp() + PLATFORM.monotonic_timer().monotonic_timestamp() } #[no_mangle] fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { - ARCHITECTURE.message_interrupt_controller() + PLATFORM.message_interrupt_controller() } #[no_mangle] @@ -365,7 +319,7 @@ fn __register_global_interrupt( options: IrqOptions, handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); let irq = { #[cfg(target_arch = "aarch64")] diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index b679e643..323f7d70 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -25,7 +25,7 @@ use yggdrasil_abi::error::Error; use crate::{ arch::{ x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, - Architecture, CpuMessage, ARCHITECTURE, + CpuMessage, Platform, PLATFORM, }, mem::{heap::GLOBAL_HEAP, read_memory, write_memory}, }; @@ -169,7 +169,7 @@ impl acpi_system::Handler for AcpiHandlerImpl { fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> { infoln!("Installing ACPI SCI handler at IRQ #{}", irq); - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); let handler = Box::leak(Box::new(SciHandler)); let irq = IrqNumber::Isa(irq as _); @@ -360,7 +360,7 @@ pub fn init_acpi(tables: &'static AcpiTables) -> Result<(), Err // 7. Actually enter the S5 state unsafe { - ARCHITECTURE + PLATFORM .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Shutdown) .unwrap(); } diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index d816d388..d4a3132c 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -5,13 +5,13 @@ use core::arch::global_asm; use static_assertions::{const_assert, const_assert_eq}; use crate::{ - arch::{x86_64::cpu::Cpu, Architecture, CpuAccess}, + arch::{x86_64::cpu::Cpu, CpuAccess, Platform}, task::thread::Thread, }; use super::{ exception::{self, IrqFrame}, - ARCHITECTURE, + PLATFORM, }; pub mod ioapic; @@ -67,7 +67,7 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { let cpu = Cpu::local(); let frame = &mut *frame; - ARCHITECTURE + PLATFORM .external_interrupt_controller() .handle_specific_irq(vector); cpu.local_apic().clear_interrupt(); @@ -85,7 +85,7 @@ unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) { let cpu = Cpu::local(); let frame = &mut *frame; - ARCHITECTURE + PLATFORM .message_interrupt_controller() .handle_msi(vector); cpu.local_apic().clear_interrupt(); diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index a19ce8f7..8d6c03e8 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -15,7 +15,7 @@ use crate::{ mem::KERNEL_VIRT_OFFSET, }; -use super::{cpuid::init_cpuid, exception, ARCHITECTURE}; +use super::{cpuid::init_cpuid, exception, PLATFORM}; pub enum BootData { YBoot(&'static LoadProtocolV1), @@ -79,14 +79,14 @@ extern "C" fn __x86_64_upper_entry() -> ! { init_dummy_cpu(); } - ARCHITECTURE.set_boot_data(BootData::YBoot(&YBOOT_DATA)); + PLATFORM.set_boot_data(BootData::YBoot(&YBOOT_DATA)); // Gather available CPU features init_cpuid(); // Setup memory management: kernel virtual memory tables, physical page manager and heap unsafe { - ARCHITECTURE + PLATFORM .init_memory_management() .expect("Could not initialize memory management"); } @@ -102,7 +102,7 @@ extern "C" fn __x86_64_upper_entry() -> ! { // Initializes: local CPU, platform devices (timers/serials/etc), debug output unsafe { - ARCHITECTURE + PLATFORM .init_platform(0) .expect("Could not initialize the platform"); } @@ -125,7 +125,7 @@ pub extern "C" fn __x86_64_ap_entry() -> ! { // syscall::init_syscall(); exception::init_exceptions(cpu_id); - ARCHITECTURE + PLATFORM .init_platform(cpu_id) .expect("Could not initialize the platform (AP)"); } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 3ac46a63..93ad15c4 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -13,7 +13,7 @@ use crate::{ task::{context::TaskFrame, thread::Thread, Cpu}, }; -use super::ARCHITECTURE; +use super::PLATFORM; primitive_enum! { enum ExceptionKind: u64 { @@ -404,7 +404,7 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { if let Some(msg) = cpu.get_ipi() { unsafe { - ARCHITECTURE.handle_ipi(msg); + PLATFORM.handle_ipi(msg); } } } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 7506261c..9f9283b8 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -73,7 +73,7 @@ use self::{ smp::CPU_COUNT, }; -use super::{Architecture, CpuAccess, CpuMessage}; +use super::{CpuAccess, CpuMessage, Platform}; /// x86-64-specific interrupt number #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] @@ -100,7 +100,7 @@ pub struct X86_64 { static SHUTDOWN_FENCE: SpinFence = SpinFence::new(); /// Global x86-64 architecture value -pub static ARCHITECTURE: X86_64 = X86_64 { +pub static PLATFORM: X86_64 = X86_64 { boot_data: OneTimeInit::new(), acpi: OneTimeInit::new(), @@ -111,7 +111,7 @@ pub static ARCHITECTURE: X86_64 = X86_64 { timer: OneTimeInit::new(), }; -impl Architecture for X86_64 { +impl Platform for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; type IrqNumber = IrqNumber; type L3 = mem::table::L3; @@ -134,29 +134,6 @@ impl Architecture for X86_64 { CPU_COUNT.load(Ordering::Acquire) } - unsafe fn set_interrupt_mask(mask: bool) { - if mask { - core::arch::asm!("cli"); - } else { - core::arch::asm!("sti"); - } - } - - fn interrupt_mask() -> bool { - let mut flags: u64; - unsafe { - core::arch::asm!("pushfq; pop {0}", out(reg) flags, options(att_syntax)); - } - // If IF is zero, interrupts are disabled (masked) - flags & (1 << 9) == 0 - } - - fn wait_for_interrupt() { - unsafe { - core::arch::asm!("hlt"); - } - } - #[inline] unsafe fn map_device_memory( &self, diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index 3ee5357d..dbe63084 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -10,7 +10,7 @@ use crate::arch::{ intrinsics::{IoPort, IoPortAccess}, IrqNumber, }, - Architecture, ARCHITECTURE, + Platform, PLATFORM, }; const FREQUENCY: u32 = 1193180; @@ -62,7 +62,7 @@ impl Device for I8253 { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); let inner = self.inner.lock(); let div: u16 = (FREQUENCY / 1000).try_into().unwrap(); diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index 77964744..c78be67f 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -13,7 +13,7 @@ use crate::{ peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, IrqNumber, }, - Architecture, ARCHITECTURE, + Platform, PLATFORM, }, device::input, }; @@ -121,7 +121,7 @@ impl Device for PS2Controller { unsafe fn init_irq(&'static self) -> Result<(), Error> { let mut inner = self.inner.lock(); // let intc = PLATFORM.interrupt_controller(); - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); intc.register_irq(self.primary_irq, Default::default(), self)?; diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index d1066e66..c594cf89 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -2,6 +2,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk_mm::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, @@ -17,7 +18,7 @@ use crate::{ KERNEL_TABLES, }, }, - Architecture, ArchitectureImpl, CpuAccess, + CpuAccess, }, mem::phys, task::Cpu, diff --git a/src/device/power/arm_psci.rs b/src/device/power/arm_psci.rs index ca578752..140b44c0 100644 --- a/src/device/power/arm_psci.rs +++ b/src/device/power/arm_psci.rs @@ -4,8 +4,9 @@ use abi::error::Error; use alloc::boxed::Box; use device_api::{CpuBringupDevice, Device, ResetDevice}; use device_tree::{device_tree_driver, dt::DevTreeIndexNodePropGet}; +use kernel_arch::{Architecture, ArchitectureImpl}; -use crate::arch::{Architecture, ArchitectureImpl, ARCHITECTURE}; +use crate::arch::{Platform, PlatformImpl, PLATFORM}; enum CallMethod { Hvc, @@ -28,7 +29,7 @@ impl Device for Psci { } unsafe fn init(&'static self) -> Result<(), Error> { - ARCHITECTURE.psci.init(self); + PLATFORM.psci.init(self); Ok(()) } } diff --git a/src/device/power/sunxi_rwdog.rs b/src/device/power/sunxi_rwdog.rs index 450b5145..a8868427 100644 --- a/src/device/power/sunxi_rwdog.rs +++ b/src/device/power/sunxi_rwdog.rs @@ -9,7 +9,7 @@ use tock_registers::{ }; use crate::{ - arch::{Architecture, ARCHITECTURE}, + arch::{Architecture, PLATFORM}, device::devtree::{self, DevTreeIndexNodePropGet, DevTreeIndexPropExt}, device_tree_driver, mem::device::DeviceMemoryIo, @@ -81,7 +81,7 @@ impl Device for RWdog { self.inner.init(IrqSafeSpinlock::new(Inner { regs })); - ARCHITECTURE.register_reset_device(self)?; + PLATFORM.register_reset_device(self)?; Ok(()) } diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index cb57042c..fb6e0133 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -19,7 +19,7 @@ use tock_registers::{ use vfs::{CharDevice, FileReadiness}; use crate::{ - arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, + arch::{aarch64::IrqNumber, Platform, PLATFORM}, debug::{self, DebugSink, LogLevel}, device::tty::{TtyContext, TtyDevice}, task::process::ProcessId, @@ -221,7 +221,7 @@ impl Device for Pl011 { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); intc.register_irq(self.irq, Default::default(), self)?; self.inner.get().lock().regs.IMSC.modify(IMSC::RXIM::SET); diff --git a/src/device/serial/sunxi_uart.rs b/src/device/serial/sunxi_uart.rs index e340fd69..79fac776 100644 --- a/src/device/serial/sunxi_uart.rs +++ b/src/device/serial/sunxi_uart.rs @@ -12,7 +12,7 @@ use tock_registers::{ use vfs::CharDevice; use crate::{ - arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, + arch::{aarch64::IrqNumber, Architecture, PLATFORM}, debug::{self, DebugSink, LogLevel}, device::{ devtree::{self, DevTreeIndexPropExt}, @@ -173,7 +173,7 @@ impl Device for SunxiUart { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = ARCHITECTURE.external_interrupt_controller(); + let intc = PLATFORM.external_interrupt_controller(); intc.register_irq(self.irq, Default::default(), self)?; intc.enable_irq(self.irq)?; diff --git a/src/main.rs b/src/main.rs index 385ffed2..0631189f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,11 +38,11 @@ #![no_std] #![no_main] -use arch::Architecture; +use arch::Platform; use libk_util::sync::SpinFence; use crate::{ - arch::{ArchitectureImpl, ARCHITECTURE}, + arch::{PlatformImpl, PLATFORM}, fs::sysfs, mem::heap, task::{spawn_kernel_closure, Cpu}, @@ -74,7 +74,7 @@ static CPU_INIT_FENCE: SpinFence = SpinFence::new(); pub fn kernel_secondary_main() -> ! { // Synchronize the CPUs to this point CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(ArchitectureImpl::cpu_count()); + CPU_INIT_FENCE.wait_all(PlatformImpl::cpu_count()); unsafe { task::enter(); @@ -100,14 +100,14 @@ pub fn kernel_main() -> ! { fs::add_pseudo_devices().unwrap(); unsafe { - ARCHITECTURE.start_application_processors(); + PLATFORM.start_application_processors(); } Cpu::init_ipi_queues(); // Wait until all APs initialize CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(ArchitectureImpl::cpu_count()); + CPU_INIT_FENCE.wait_all(PlatformImpl::cpu_count()); task::init().expect("Failed to initialize the scheduler"); diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 8d63b161..7ee410ae 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -9,7 +9,7 @@ use core::{ use abi::error::Error; use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; -use crate::arch::{Architecture, ArchitectureImpl}; +use crate::arch::{PlatformImpl, Platform}; pub mod address; pub mod heap; @@ -20,7 +20,7 @@ pub mod table; use self::process::ProcessAddressSpace; /// Offset applied to the physical kernel image when translating it into the virtual address space -pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET; +pub const KERNEL_VIRT_OFFSET: usize = PlatformImpl::KERNEL_VIRT_OFFSET; /// Reads a value from an arbitrary physical address. /// diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index 98eb1506..00dbae0b 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -1,6 +1,7 @@ //! Physical memory management utilities use abi::{error::Error, system::SystemMemoryStats}; +use kernel_arch::absolute_address; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, @@ -8,7 +9,7 @@ use libk_mm::{ use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use crate::{ - arch::{Architecture, ARCHITECTURE}, + arch::{Platform, PLATFORM}, mem::phys::reserved::is_reserved, }; @@ -122,7 +123,7 @@ pub unsafe fn init_from_iter + Clone>( reserve_region("kernel", kernel_physical_memory_region()); - ARCHITECTURE.map_physical_memory(it.clone(), phys_start, phys_end)?; + PLATFORM.map_physical_memory(it.clone(), phys_start, phys_end)?; let total_count = (phys_end - phys_start) / 0x1000; let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / (BITMAP_WORD_SIZE / 8); diff --git a/src/panic.rs b/src/panic.rs index e1c880cd..35a63aa2 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -2,10 +2,11 @@ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use device_api::interrupt::IpiDeliveryTarget; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk_util::sync::{hack_locks, SpinFence}; use crate::{ - arch::{Architecture, ArchitectureImpl, CpuAccess, CpuMessage, ARCHITECTURE}, + arch::{CpuAccess, CpuMessage, Platform, PlatformImpl, PLATFORM}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, task::Cpu, @@ -53,12 +54,12 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { { // Let other CPUs know we're screwed unsafe { - ARCHITECTURE + PLATFORM .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) .ok(); } - let ap_count = ArchitectureImpl::cpu_count() - 1; + let ap_count = PlatformImpl::cpu_count() - 1; PANIC_HANDLED_FENCE.wait_all(ap_count); unsafe { @@ -101,7 +102,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { PANIC_SEQUENCE.fetch_add(1, Ordering::Release); unsafe { - ARCHITECTURE.reset(); + PLATFORM.reset(); } } diff --git a/src/proc/random.rs b/src/proc/random.rs index 2f8c3ddb..c9340947 100644 --- a/src/proc/random.rs +++ b/src/proc/random.rs @@ -2,7 +2,7 @@ use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::arch::{Architecture, ARCHITECTURE}; +use crate::arch::{Platform, PLATFORM}; const BUFFER_SIZE: usize = 1024; @@ -67,7 +67,7 @@ pub fn read(buf: &mut [u8]) { /// Initializes the random generator state pub fn init() { - let now = ARCHITECTURE + let now = PLATFORM .monotonic_timer() .monotonic_timestamp() .unwrap(); diff --git a/src/task/mod.rs b/src/task/mod.rs index e40dc062..2df94a87 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -8,7 +8,7 @@ use libk::{runtime, thread::Termination}; use libk_util::sync::SpinFence; use crate::{ - arch::{Architecture, ArchitectureImpl, CpuAccess}, + arch::{PlatformImpl, CpuAccess, Platform}, task::{sched::CpuQueue, thread::Thread}, }; @@ -35,7 +35,7 @@ pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send /// Sets up CPU queues and gives them some processes to run pub fn init() -> Result<(), Error> { - let cpu_count = ArchitectureImpl::cpu_count(); + let cpu_count = PlatformImpl::cpu_count(); // Create a queue for each CPU sched::init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); diff --git a/src/task/sched.rs b/src/task/sched.rs index 2e1be619..7ba464e1 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -5,12 +5,10 @@ use core::cell::Cell; use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk_util::{sync::IrqGuard, OneTimeInit}; -use crate::{ - arch::{Architecture, ArchitectureImpl, CpuAccess}, - task::thread::ThreadState, -}; +use crate::{arch::CpuAccess, task::thread::ThreadState}; use super::{ context::TaskContextImpl, From 996876ba46c6463e53e6cf85fbe132bc06d9e4b8 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 5 Feb 2024 14:45:17 +0200 Subject: [PATCH 181/211] libk: move physical memory management to libk-mm --- libk-mm/Cargo.toml | 2 + libk-mm/src/api.rs | 5 - libk-mm/src/lib.rs | 34 ++-- libk-mm/src/mod.rs | 0 libk-mm/src/phys.rs | 36 ----- {src/mem => libk-mm/src}/phys/manager.rs | 21 ++- {src/mem => libk-mm/src}/phys/mod.rs | 80 +++++++--- {src/mem => libk-mm/src}/phys/reserved.rs | 3 +- libk/src/api.rs | 9 -- src/arch/aarch64/boot/mod.rs | 8 +- src/arch/aarch64/context.rs | 7 +- src/arch/aarch64/exception.rs | 2 +- src/arch/aarch64/mem/process.rs | 2 +- src/arch/aarch64/mem/table.rs | 6 +- src/arch/aarch64/mod.rs | 57 ++++--- src/arch/mod.rs | 10 -- src/arch/x86_64/context.rs | 6 +- src/arch/x86_64/mem/process.rs | 2 +- src/arch/x86_64/mem/table.rs | 6 +- src/arch/x86_64/mod.rs | 184 +++++++++++----------- src/arch/x86_64/smp.rs | 2 +- src/device/power/arm_psci.rs | 2 +- src/fs/mod.rs | 4 +- src/fs/sysfs.rs | 3 +- src/mem/mod.rs | 3 +- src/mem/process.rs | 3 +- src/syscall/mod.rs | 4 +- 27 files changed, 230 insertions(+), 271 deletions(-) delete mode 100644 libk-mm/src/mod.rs delete mode 100644 libk-mm/src/phys.rs rename {src/mem => libk-mm/src}/phys/manager.rs (91%) rename {src/mem => libk-mm/src}/phys/mod.rs (71%) rename {src/mem => libk-mm/src}/phys/reserved.rs (91%) diff --git a/libk-mm/Cargo.toml b/libk-mm/Cargo.toml index 6f19b1ba..7971cc2e 100644 --- a/libk-mm/Cargo.toml +++ b/libk-mm/Cargo.toml @@ -7,5 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +kernel-arch = { path = "../arch" } +libk-util = { path = "../libk-util" } log = "0.4.20" diff --git a/libk-mm/src/api.rs b/libk-mm/src/api.rs index b319d8f0..0aaf8a32 100644 --- a/libk-mm/src/api.rs +++ b/libk-mm/src/api.rs @@ -6,11 +6,6 @@ use crate::{ }; extern "Rust" { - pub fn __allocate_2m_page() -> Result; - pub fn __allocate_page() -> Result; - pub fn __allocate_contiguous_pages(count: usize) -> Result; - pub fn __free_page(page: PhysicalAddress); - pub fn __virtualize(phys: u64) -> usize; pub fn __physicalize(virt: usize) -> u64; diff --git a/libk-mm/src/lib.rs b/libk-mm/src/lib.rs index bed02a2b..5c96f56b 100644 --- a/libk-mm/src/lib.rs +++ b/libk-mm/src/lib.rs @@ -19,8 +19,6 @@ use core::{ use yggdrasil_abi::error::Error; -use crate::api::{__allocate_contiguous_pages, __free_page, __physicalize}; - use self::address::{AsPhysicalAddress, PhysicalAddress}; pub(crate) mod api; @@ -31,6 +29,10 @@ pub mod phys; pub mod pointer; pub mod table; +// TODO find a way to integrate this nicely with Architecture? +pub const L3_PAGE_SIZE: usize = 1 << 12; +pub const L2_PAGE_SIZE: usize = 1 << 21; + pub trait PageProvider { fn get_page(&self, offset: u64) -> Result; fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>; @@ -46,20 +48,14 @@ impl PageBox { fn alloc_slice(count: usize) -> Result<(PhysicalAddress, usize), Error> { // TODO hardcoded page sizes let layout = Layout::array::(count).unwrap(); - let page_count = (layout.size() + 0xFFF) / 0x1000; - Ok(( - unsafe { __allocate_contiguous_pages(page_count) }?, - page_count, - )) + let page_count = (layout.size() + L3_PAGE_SIZE - 1) / L3_PAGE_SIZE; + Ok((phys::alloc_pages_contiguous(page_count)?, page_count)) } #[inline] fn alloc() -> Result<(PhysicalAddress, usize), Error> { - let page_count = (size_of::() + 0xFFF) / 0x1000; - Ok(( - unsafe { __allocate_contiguous_pages(page_count) }?, - page_count, - )) + let page_count = (size_of::() + L3_PAGE_SIZE - 1) / L3_PAGE_SIZE; + Ok((phys::alloc_pages_contiguous(page_count)?, page_count)) } pub fn new(init: T) -> Result, Error> { @@ -209,7 +205,7 @@ impl PageBox<[MaybeUninit]> { impl AsPhysicalAddress for PageBox { #[inline] unsafe fn as_physical_address(&self) -> PhysicalAddress { - PhysicalAddress(__physicalize(self.value.addr())) + PhysicalAddress(api::__physicalize(self.value.addr())) } } @@ -236,11 +232,11 @@ impl Drop for PageBox { core::ptr::drop_in_place(self.value); } // SAFETY: Safe, pointer obtained through "virtualize" - let base = PhysicalAddress(unsafe { __physicalize(self.value.addr()) }); + let base = PhysicalAddress(unsafe { api::__physicalize(self.value.addr()) }); for i in 0..self.page_count { // SAFETY: Safe, page allocated only by this PageBox unsafe { - __free_page(base.add(0x1000 * i)); + phys::free_page(base.add(L3_PAGE_SIZE * i)); } } } @@ -269,11 +265,3 @@ impl fmt::Display for PageBox { unsafe impl Send for PageBox {} unsafe impl Sync for PageBox {} - -pub fn allocate_page() -> Result { - unsafe { api::__allocate_page() } -} - -pub fn allocate_contiguous_pages(count: usize) -> Result { - unsafe { api::__allocate_contiguous_pages(count) } -} diff --git a/libk-mm/src/mod.rs b/libk-mm/src/mod.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/libk-mm/src/phys.rs b/libk-mm/src/phys.rs deleted file mode 100644 index 8177323a..00000000 --- a/libk-mm/src/phys.rs +++ /dev/null @@ -1,36 +0,0 @@ -use core::ops::Range; - -use super::address::PhysicalAddress; - -/// Defines an usable memory region -#[derive(Clone, Copy, Debug)] -pub struct PhysicalMemoryRegion { - /// Start of the region - pub base: PhysicalAddress, - /// Length of the region - pub size: usize, -} - -impl PhysicalMemoryRegion { - /// Returns the end address of the region - pub const fn end(&self) -> PhysicalAddress { - self.base.add(self.size) - } - - /// Returns an address range covered by the region - pub fn range(&self) -> Range { - self.base..self.end() - } - - /// Constrains the [PhysicalMemoryRegion] to global memory limits set in the kernel - pub fn clamp(self, limit: PhysicalAddress) -> Option<(PhysicalAddress, PhysicalAddress)> { - let start = self.base.min(limit); - let end = self.end().min(limit); - - if start < end { - Some((start, end)) - } else { - None - } - } -} diff --git a/src/mem/phys/manager.rs b/libk-mm/src/phys/manager.rs similarity index 91% rename from src/mem/phys/manager.rs rename to libk-mm/src/phys/manager.rs index cecff52a..b7842e14 100644 --- a/src/mem/phys/manager.rs +++ b/libk-mm/src/phys/manager.rs @@ -1,15 +1,14 @@ //! Physical memory manager implementation use core::sync::atomic::{AtomicUsize, Ordering}; -use abi::{error::Error, system::SystemMemoryStats}; -use libk_mm::{ +use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; + +use crate::{ address::{FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, - table::EntryLevel, + L3_PAGE_SIZE, }; -use crate::arch::L3; - pub type BitmapWord = u64; pub(super) const BITMAP_WORD_SIZE: usize = BitmapWord::BITS as usize; @@ -83,7 +82,7 @@ impl PhysicalMemoryManager { STATS.used_pages.fetch_add(1, Ordering::Relaxed); - return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); + return Ok(PhysicalAddress::from_raw(i * L3_PAGE_SIZE + self.offset)); } if self.last_free_bit != 0 { @@ -111,7 +110,7 @@ impl PhysicalMemoryManager { STATS.used_pages.fetch_add(512, Ordering::Relaxed); - return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); + return Ok(PhysicalAddress::from_raw(i * L3_PAGE_SIZE + self.offset)); } if self.last_free_bit != 0 { @@ -138,7 +137,7 @@ impl PhysicalMemoryManager { STATS.used_pages.fetch_add(count, Ordering::Relaxed); - return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); + return Ok(PhysicalAddress::from_raw(i * L3_PAGE_SIZE + self.offset)); } if self.last_free_bit != 0 { @@ -157,7 +156,7 @@ impl PhysicalMemoryManager { pub unsafe fn free_page(&mut self, page: PhysicalAddress) { let page: usize = page.into_raw(); assert!(page >= self.offset); - let index = (page - self.offset) / 0x1000; + let index = (page - self.offset) / L3_PAGE_SIZE; STATS.used_pages.fetch_sub(1, Ordering::Relaxed); @@ -173,7 +172,7 @@ impl PhysicalMemoryManager { pub fn add_available_page(&mut self, page: PhysicalAddress) { let page: usize = page.into_raw(); assert!(page >= self.offset); - let index = (page - self.offset) / 0x1000; + let index = (page - self.offset) / L3_PAGE_SIZE; STATS.available_pages.fetch_add(1, Ordering::Relaxed); @@ -191,7 +190,7 @@ impl PhysicalMemoryManager { total_usable_pages: available, allocated_pages: used, free_pages: free, - page_size: L3::SIZE, + page_size: L3_PAGE_SIZE, } } } diff --git a/src/mem/phys/mod.rs b/libk-mm/src/phys/mod.rs similarity index 71% rename from src/mem/phys/mod.rs rename to libk-mm/src/phys/mod.rs index 00dbae0b..82449298 100644 --- a/src/mem/phys/mod.rs +++ b/libk-mm/src/phys/mod.rs @@ -1,33 +1,65 @@ -//! Physical memory management utilities +use core::ops::Range; -use abi::{error::Error, system::SystemMemoryStats}; use kernel_arch::absolute_address; -use libk_mm::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, - phys::PhysicalMemoryRegion, -}; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; +use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; use crate::{ - arch::{Platform, PLATFORM}, - mem::phys::reserved::is_reserved, + address::{FromRaw, IntoRaw}, + phys::{ + manager::BITMAP_WORD_SIZE, + reserved::{is_reserved, reserve_region}, + }, + L2_PAGE_SIZE, L3_PAGE_SIZE, }; -use self::{ - manager::{PhysicalMemoryManager, BITMAP_WORD_SIZE, TRACKED_PAGE_LIMIT}, - reserved::reserve_region, -}; +use self::manager::{PhysicalMemoryManager, TRACKED_PAGE_LIMIT}; -// 8 * 4096 bits per page, 1 page per bit -const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PAGE_LIMIT * 4096); +use super::address::PhysicalAddress; mod manager; pub mod reserved; +/// Defines an usable memory region +#[derive(Clone, Copy, Debug)] +pub struct PhysicalMemoryRegion { + /// Start of the region + pub base: PhysicalAddress, + /// Length of the region + pub size: usize, +} + +// 8 * 4096 bits per page, 1 page per bit +const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PAGE_LIMIT * 4096); + /// Global physical memory manager pub static PHYSICAL_MEMORY: OneTimeInit> = OneTimeInit::new(); +impl PhysicalMemoryRegion { + /// Returns the end address of the region + pub const fn end(&self) -> PhysicalAddress { + self.base.add(self.size) + } + + /// Returns an address range covered by the region + pub fn range(&self) -> Range { + self.base..self.end() + } + + /// Constrains the [PhysicalMemoryRegion] to global memory limits set in the kernel + pub fn clamp(self, limit: PhysicalAddress) -> Option<(PhysicalAddress, PhysicalAddress)> { + let start = self.base.min(limit); + let end = self.end().min(limit); + + if start < end { + Some((start, end)) + } else { + None + } + } +} + /// Allocates a single physical page from the global manager pub fn alloc_page() -> Result { PHYSICAL_MEMORY.get().lock().alloc_page() @@ -88,7 +120,7 @@ pub fn find_contiguous_region>( let mut collected = 0; let mut base_addr = None; - for addr in (reg_start..reg_end).step_by(0x1000) { + for addr in (reg_start..reg_end).step_by(L3_PAGE_SIZE) { if is_reserved(addr) { collected = 0; base_addr = None; @@ -115,19 +147,23 @@ pub fn find_contiguous_region>( /// /// The caller must ensure this function has not been called before and that the regions /// are valid and actually available. -pub unsafe fn init_from_iter + Clone>( +pub unsafe fn init_from_iter< + I: Iterator + Clone, + Map: FnOnce(I, PhysicalAddress, PhysicalAddress) -> Result<(), Error>, +>( it: I, + map_physical_memory: Map, ) -> Result<(), Error> { // Map the physical memory let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap(); reserve_region("kernel", kernel_physical_memory_region()); - PLATFORM.map_physical_memory(it.clone(), phys_start, phys_end)?; + map_physical_memory(it.clone(), phys_start, phys_end)?; - let total_count = (phys_end - phys_start) / 0x1000; + let total_count = (phys_end - phys_start) / L3_PAGE_SIZE; let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / (BITMAP_WORD_SIZE / 8); - let page_bitmap_page_count = (page_bitmap_size + 0xFFF) / 0x1000; + let page_bitmap_page_count = (page_bitmap_size + L3_PAGE_SIZE - 1) / L3_PAGE_SIZE; let page_bitmap_phys_base = find_contiguous_region(it.clone(), page_bitmap_page_count).unwrap(); @@ -135,11 +171,11 @@ pub unsafe fn init_from_iter + Clone>( "page-bitmap", PhysicalMemoryRegion { base: page_bitmap_phys_base, - size: page_bitmap_page_count * 0x1000, + size: page_bitmap_page_count * L3_PAGE_SIZE, }, ); - if IntoRaw::::into_raw(phys_start) & 0x1FFFFFF != 0 { + if IntoRaw::::into_raw(phys_start) & (L2_PAGE_SIZE - 1) != 0 { todo!(); } @@ -149,7 +185,7 @@ pub unsafe fn init_from_iter + Clone>( const MAX_MEMORY: usize = 64 * 1024; for (start, end) in it.into_iter().filter_map(|r| r.clamp(MEMORY_UPPER_LIMIT)) { - for page in (start..end).step_by(0x1000) { + for page in (start..end).step_by(L3_PAGE_SIZE) { if collected >= MAX_MEMORY { break; } diff --git a/src/mem/phys/reserved.rs b/libk-mm/src/phys/reserved.rs similarity index 91% rename from src/mem/phys/reserved.rs rename to libk-mm/src/phys/reserved.rs index 5515a936..43d70757 100644 --- a/src/mem/phys/reserved.rs +++ b/libk-mm/src/phys/reserved.rs @@ -1,8 +1,9 @@ //! Utilities for handling reserved memory regions -use libk_mm::{address::PhysicalAddress, phys::PhysicalMemoryRegion}; use libk_util::StaticVector; +use crate::{address::PhysicalAddress, phys::PhysicalMemoryRegion}; + static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); /// Marks a region of physical memory as reserved. diff --git a/libk/src/api.rs b/libk/src/api.rs index 05abfe75..98286166 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -2,7 +2,6 @@ use core::time::Duration; use alloc::{string::String, sync::Arc}; use device_api::interrupt::{InterruptHandler, IrqOptions, MessageInterruptController}; -use libk_mm::address::PhysicalAddress; use yggdrasil_abi::{ error::Error, process::{ExitCode, Signal}, @@ -14,14 +13,6 @@ extern "Rust" { pub fn __cpu_index() -> usize; pub fn __cpu_count() -> usize; - pub fn __allocate_2m_page() -> Result; - pub fn __allocate_page() -> Result; - pub fn __allocate_contiguous_pages(count: usize) -> Result; - pub fn __free_page(page: PhysicalAddress); - - pub fn __virtualize(phys: u64) -> usize; - pub fn __physicalize(virt: usize) -> u64; - // SAFETY: Both the kernel-side and api-side Threads are sized and Arc<___> has the same value // for them pub fn __create_kthread( diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 131ea423..353d5017 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -11,18 +11,16 @@ use kernel_fs::devfs; use libk::runtime; use libk_mm::{ address::{IntoRaw, PhysicalAddress}, + phys, table::EntryLevel, }; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use super::{ - exception, mem::load_fixed_tables, smp::CPU_COUNT, AArch64, BootStack, BOOT_STACK_SIZE, - PLATFORM, + exception, mem::load_fixed_tables, smp::CPU_COUNT, BootStack, BOOT_STACK_SIZE, PLATFORM, }; use crate::{ - arch::{aarch64::mem::table::L3, Platform}, - kernel_main, kernel_secondary_main, - mem::{phys, KERNEL_VIRT_OFFSET}, + arch::aarch64::mem::table::L3, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, }; unsafe fn pre_init_mmu() { diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index ef5b114a..a2b456f7 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -3,12 +3,9 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use libk_mm::address::PhysicalAddress; +use libk_mm::{address::PhysicalAddress, phys}; -use crate::{ - mem::phys, - task::{context::TaskContextImpl, thread::Thread}, -}; +use crate::task::{context::TaskContextImpl, thread::Thread}; struct StackBuilder { base: usize, diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index b784f86f..e4ab320d 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -17,7 +17,7 @@ use kernel_arch::{Architecture, ArchitectureImpl}; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{Platform, PlatformImpl}, + arch::Platform, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, thread::Thread}, diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index f71a72fd..8f204195 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -4,12 +4,12 @@ use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, + phys, pointer::PhysicalRefMut, table::{EntryLevel, EntryLevelExt}, }; use crate::mem::{ - phys, process::ProcessAddressSpaceManager, table::{EntryLevelDrop, MapAttributes, NextPageTable}, }; diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs index f87c59e9..d46e0f5e 100644 --- a/src/arch/aarch64/mem/table.rs +++ b/src/arch/aarch64/mem/table.rs @@ -7,14 +7,12 @@ use abi::error::Error; use bitflags::bitflags; use libk_mm::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + phys, pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevel, }; -use crate::mem::{ - phys, - table::{EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel}, -}; +use crate::mem::table::{EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel}; bitflags! { #[derive(Clone, Copy, PartialEq, Eq)] diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 910a0fed..14a98c50 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -4,7 +4,7 @@ pub mod mem; use core::sync::atomic::Ordering; -use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF}; +use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use device_api::{ interrupt::{ @@ -20,11 +20,12 @@ use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, phys::PhysicalMemoryRegion, + phys::{self, reserved::reserve_region}, pointer::PhysicalRef, table::{EntryLevel, EntryLevelExt}, }; use libk_util::OneTimeInit; -use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use tock_registers::interfaces::Writeable; use ygg_driver_pci::PciBusManager; use crate::{ @@ -35,10 +36,7 @@ use crate::{ debug, device::{self, power::arm_psci::Psci}, fs::{Initrd, INITRD_DATA}, - mem::{ - heap, - phys::{self, reserved::reserve_region}, - }, + mem::heap, }; use self::mem::{table::L1, EarlyMapping}; @@ -109,29 +107,6 @@ impl Platform for AArch64 { mem::unmap_device_memory(map) } - fn map_physical_memory + Clone>( - &self, - _it: I, - _memory_start: PhysicalAddress, - memory_end: PhysicalAddress, - ) -> Result<(), Error> { - let end_l1i = memory_end.page_align_up::().page_index::(); - if end_l1i > mem::RAM_MAPPING_L1_COUNT { - todo!() - } - - // Map 1GiB chunks - for index in 0..end_l1i { - unsafe { - mem::map_ram_l1(index); - } - } - - mem::MEMORY_LIMIT.init(memory_end.into_raw()); - - Ok(()) - } - fn virtualize(address: u64) -> usize { let address = address as usize; if address < *mem::MEMORY_LIMIT.get() { @@ -246,6 +221,28 @@ impl AArch64 { Some((initrd_start, initrd_end)) } + fn map_physical_memory + Clone>( + _it: I, + _memory_start: PhysicalAddress, + memory_end: PhysicalAddress, + ) -> Result<(), Error> { + let end_l1i = memory_end.page_align_up::().page_index::(); + if end_l1i > mem::RAM_MAPPING_L1_COUNT { + todo!() + } + + // Map 1GiB chunks + for index in 0..end_l1i { + unsafe { + mem::map_ram_l1(index); + } + } + + mem::MEMORY_LIMIT.init(memory_end.into_raw()); + + Ok(()) + } + unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> { // 16x2MiB const HEAP_PAGES: usize = 16; @@ -291,7 +288,7 @@ impl AArch64 { // Initialize the physical memory let regions = FdtMemoryRegionIter::new(&dt); - phys::init_from_iter(regions)?; + phys::init_from_iter(regions, Self::map_physical_memory)?; // Setup the heap for i in 0..HEAP_PAGES { diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 51532302..dafeec45 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -17,7 +17,6 @@ use kernel_arch::{Architecture, ArchitectureImpl}; use libk_mm::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - phys::PhysicalMemoryRegion, table::EntryLevel, }; use libk_util::sync::IrqGuard; @@ -92,15 +91,6 @@ pub trait Platform { /// The caller must ensure the mapping is and will no longer be used. unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping); - /// Maps the physical memory regions into the kernel space so they can later be accessed by the - /// kernel - fn map_physical_memory + Clone>( - &self, - it: I, - memory_start: PhysicalAddress, - memory_end: PhysicalAddress, - ) -> Result<(), Error>; - /// Converts a physical address to a virtual one, so it can be accessed by the kernel fn virtualize(address: u64) -> usize; diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 5ab83730..1272e0f7 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -3,11 +3,13 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use libk_mm::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; +use libk_mm::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + phys, +}; use crate::{ arch::x86_64::mem::KERNEL_TABLES, - mem::phys, task::{context::TaskContextImpl, thread::Thread}, }; diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index a6bac2ea..52795c15 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,6 +1,7 @@ //! x86-64-specific process address space management functions use libk_mm::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + phys, pointer::PhysicalRefMut, table::EntryLevelExt, }; @@ -9,7 +10,6 @@ use yggdrasil_abi::error::Error; use crate::{ arch::x86_64::intrinsics, mem::{ - phys, process::ProcessAddressSpaceManager, table::{EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable}, }, diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index b74329ff..eec58785 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -8,12 +8,12 @@ use abi::error::Error; use bitflags::bitflags; use libk_mm::{ address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, + phys, pointer::{PhysicalRef, PhysicalRefMut}, }; -use crate::mem::{ - phys, - table::{EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel}, +use crate::mem::table::{ + EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel, }; bitflags! { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 9f9283b8..962e00eb 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -14,7 +14,7 @@ use kernel_fs::devfs; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - phys::PhysicalMemoryRegion, + phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, table::EntryLevelExt, }; use libk_util::{sync::SpinFence, OneTimeInit}; @@ -51,11 +51,7 @@ use crate::{ display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, }, fs::{Initrd, INITRD_DATA}, - mem::{ - heap, - phys::{self, reserved::reserve_region}, - table::EntryLevel, - }, + mem::{heap, table::EntryLevel}, }; use self::{ @@ -149,88 +145,6 @@ impl Platform for X86_64 { mem::unmap_device_memory(map) } - fn map_physical_memory + Clone>( - &self, - it: I, - _memory_start: PhysicalAddress, - memory_end: PhysicalAddress, - ) -> Result<(), Error> { - let end_l1i = (IntoRaw::::into_raw(memory_end) + (1 << 30) - 1) >> 30; - - if end_l1i > 512 { - todo!( - "Cannot handle {}GiB of RAM", - end_l1i * L1::SIZE / (1024 * 1024 * 1024) - ); - } - - MEMORY_LIMIT.init(memory_end.into_raw()); - - // Check if 1GiB pages are supported - if PROCESSOR_FEATURES - .get() - .contains(ProcessorFeatures::PDPE1GB) - { - // Just map gigabytes of RAM - for l1i in 0..end_l1i { - // TODO NX - unsafe { - RAM_MAPPING_L1[l1i] = PageEntry::::block( - PhysicalAddress::from_raw(l1i << 30), - PageAttributes::WRITABLE, - ); - } - } - } else { - // Allocate the intermediate tables first - let l2_tables_start = phys::find_contiguous_region(it, end_l1i) - .expect("Could not allocate the memory for RAM mapping L2 tables"); - - unsafe { - reserve_region( - "ram-l2-tables", - PhysicalMemoryRegion { - base: l2_tables_start, - size: end_l1i * 0x1000, - }, - ); - } - - // Fill in the tables - for l1i in 0..end_l1i { - let l2_phys_addr = l2_tables_start.add(l1i * 0x1000); - - // TODO (minor) the slice is uninitialized, maybe find some way to deal with that - // case nicely - // Safety: ok, the mapping is done to the memory obtained from - // find_contiguous_region() - let mut l2_data = - unsafe { EarlyMapping::<[PageEntry; 512]>::map(l2_phys_addr)? }; - // Safety: ok, the slice comes from EarlyMapping of a page-aligned region - let l2 = unsafe { PageTable::from_raw_slice_mut(l2_data.deref_mut()) }; - - for l2i in 0..512 { - // TODO NX - l2[l2i] = PageEntry::::block( - PhysicalAddress::from_raw((l1i << 30) | (l2i << 21)), - PageAttributes::WRITABLE, - ); - } - - // Point the L1 entry to the L2 table - unsafe { - RAM_MAPPING_L1[l1i] = - PageEntry::::table(l2_phys_addr, PageAttributes::WRITABLE) - }; - - intrinsics::flush_cpu_cache(); - // The EarlyMapping is then dropped - } - } - - Ok(()) - } - #[inline] fn virtualize(address: u64) -> usize { let address = address as usize; @@ -307,16 +221,102 @@ impl X86_64 { self.boot_data.init(data); } + fn map_physical_memory + Clone>( + it: I, + _memory_start: PhysicalAddress, + memory_end: PhysicalAddress, + ) -> Result<(), Error> { + let end_l1i = IntoRaw::::into_raw(memory_end) + .page_align_up::() + .page_index::(); + + if end_l1i > 512 { + todo!( + "Cannot handle {}GiB of RAM", + end_l1i * L1::SIZE / (1024 * 1024 * 1024) + ); + } + + MEMORY_LIMIT.init(memory_end.into_raw()); + + // Check if 1GiB pages are supported + if PROCESSOR_FEATURES + .get() + .contains(ProcessorFeatures::PDPE1GB) + { + // Just map gigabytes of RAM + for l1i in 0..end_l1i { + // TODO NX + unsafe { + RAM_MAPPING_L1[l1i] = PageEntry::::block( + PhysicalAddress::from_raw(l1i * L1::SIZE), + PageAttributes::WRITABLE, + ); + } + } + } else { + // Allocate the intermediate tables first + let l2_tables_start = phys::find_contiguous_region(it, end_l1i) + .expect("Could not allocate the memory for RAM mapping L2 tables"); + + unsafe { + reserve_region( + "ram-l2-tables", + PhysicalMemoryRegion { + base: l2_tables_start, + size: end_l1i * L3::SIZE, + }, + ); + } + + // Fill in the tables + for l1i in 0..end_l1i { + let l2_phys_addr = l2_tables_start.add(l1i * L3::SIZE); + + // TODO (minor) the slice is uninitialized, maybe find some way to deal with that + // case nicely + // Safety: ok, the mapping is done to the memory obtained from + // find_contiguous_region() + let mut l2_data = + unsafe { EarlyMapping::<[PageEntry; 512]>::map(l2_phys_addr)? }; + // Safety: ok, the slice comes from EarlyMapping of a page-aligned region + let l2 = unsafe { PageTable::from_raw_slice_mut(l2_data.deref_mut()) }; + + for l2i in 0..512 { + // TODO NX + l2[l2i] = PageEntry::::block( + PhysicalAddress::from_raw((l1i * L1::SIZE) | (l2i * L2::SIZE)), + PageAttributes::WRITABLE, + ); + } + + // Point the L1 entry to the L2 table + unsafe { + RAM_MAPPING_L1[l1i] = + PageEntry::::table(l2_phys_addr, PageAttributes::WRITABLE) + }; + + intrinsics::flush_cpu_cache(); + // The EarlyMapping is then dropped + } + } + + Ok(()) + } + unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) -> Result<(), Error> { let mmap = EarlyMapping::::map_slice( PhysicalAddress::from_raw(data.memory_map.address), data.memory_map.len as usize, )?; - phys::init_from_iter(mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion { - base: PhysicalAddress::from_raw(reg.start_address), - size: reg.page_count as usize * 0x1000, - })) + phys::init_from_iter( + mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion { + base: PhysicalAddress::from_raw(reg.start_address), + size: reg.page_count as usize * L3::SIZE, + }), + Self::map_physical_memory, + ) } unsafe fn init_memory_management(&self) -> Result<(), Error> { diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index c594cf89..fb94e415 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -5,6 +5,7 @@ use acpi_lib::platform::{ProcessorInfo, ProcessorState}; use kernel_arch::{Architecture, ArchitectureImpl}; use libk_mm::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + phys, pointer::PhysicalRefMut, }; @@ -20,7 +21,6 @@ use crate::{ }, CpuAccess, }, - mem::phys, task::Cpu, }; diff --git a/src/device/power/arm_psci.rs b/src/device/power/arm_psci.rs index 140b44c0..5868b2fd 100644 --- a/src/device/power/arm_psci.rs +++ b/src/device/power/arm_psci.rs @@ -6,7 +6,7 @@ use device_api::{CpuBringupDevice, Device, ResetDevice}; use device_tree::{device_tree_driver, dt::DevTreeIndexNodePropGet}; use kernel_arch::{Architecture, ArchitectureImpl}; -use crate::arch::{Platform, PlatformImpl, PLATFORM}; +use crate::arch::PLATFORM; enum CallMethod { Hvc, diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 20d1a218..57c91747 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -3,13 +3,13 @@ use core::ptr::NonNull; use kernel_fs::devfs; -use libk_mm::address::PhysicalAddress; +use libk_mm::{address::PhysicalAddress, phys}; use libk_util::OneTimeInit; use memfs::block::{self, BlockAllocator}; use vfs::{impls::read_fn_node, NodeRef}; use yggdrasil_abi::{error::Error, io::MountOptions}; -use crate::{mem::phys, proc::random}; +use crate::proc::random; // pub mod devfs; pub mod sysfs; diff --git a/src/fs/sysfs.rs b/src/fs/sysfs.rs index 2fccba57..6ca38d8f 100644 --- a/src/fs/sysfs.rs +++ b/src/fs/sysfs.rs @@ -2,13 +2,14 @@ use abi::error::Error; use git_version::git_version; +use libk_mm::phys; use libk_util::OneTimeInit; use vfs::{ impls::{const_value_node, mdir, read_fn_node, ReadOnlyFnValueNode}, NodeRef, }; -use crate::{debug, mem::phys, util}; +use crate::{debug, util}; static ROOT: OneTimeInit = OneTimeInit::new(); diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 7ee410ae..dc4b082c 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -9,11 +9,10 @@ use core::{ use abi::error::Error; use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; -use crate::arch::{PlatformImpl, Platform}; +use crate::arch::{Platform, PlatformImpl}; pub mod address; pub mod heap; -pub mod phys; pub mod process; pub mod table; diff --git a/src/mem/process.rs b/src/mem/process.rs index a7cfe5a1..0e67fe8a 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -6,6 +6,7 @@ use abi::error::Error; use cfg_if::cfg_if; use libk_mm::{ address::PhysicalAddress, + phys, pointer::{PhysicalRef, PhysicalRefMut}, table::EntryLevelExt, PageProvider, @@ -14,7 +15,7 @@ use libk_util::sync::IrqSafeSpinlock; use vfs::FileRef; use vmalloc::{RangeData, VirtualMemoryAllocator}; -use crate::{arch::L3, mem::phys}; +use crate::arch::L3; use super::table::MapAttributes; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index edc3d940..faae3695 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -19,7 +19,7 @@ use abi::{ }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use libk::{block, runtime}; -use libk_mm::table::EntryLevelExt; +use libk_mm::{phys, table::EntryLevelExt}; use libk_util::sync::IrqSafeSpinlockGuard; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; @@ -29,7 +29,7 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::{phys, process::VirtualRangeBacking, table::MapAttributes}, + mem::{process::VirtualRangeBacking, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, From 07bd75fec5399d57ced98ba1a027b1d9e28fb841 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 6 Feb 2024 12:27:02 +0200 Subject: [PATCH 182/211] libk: move arch-specific mem management to arch/ --- Cargo.toml | 2 + arch/aarch64/Cargo.toml | 5 + arch/aarch64/src/lib.rs | 5 + arch/aarch64/src/mem/mod.rs | 407 ++++++++++++++++++ arch/aarch64/src/mem/table.rs | 351 +++++++++++++++ arch/interface/Cargo.toml | 1 + arch/interface/src/lib.rs | 5 + arch/interface/src/mem/address.rs | 0 arch/interface/src/mem/mod.rs | 111 +++++ arch/interface/src/mem/table.rs | 0 arch/src/lib.rs | 6 +- arch/x86_64/Cargo.toml | 7 + arch/x86_64/src/lib.rs | 8 + arch/x86_64/src/mem/mod.rs | 404 +++++++++++++++++ .../x86_64 => arch/x86_64/src}/mem/table.rs | 42 +- .../mod.rs => arch/x86_64/src/registers.rs | 0 libk-mm/Cargo.toml | 1 + libk-mm/interface/Cargo.toml | 13 + libk-mm/interface/src/address.rs | 160 +++++++ libk-mm/interface/src/lib.rs | 50 +++ libk-mm/interface/src/pointer.rs | 163 +++++++ libk-mm/interface/src/table.rs | 144 +++++++ libk-mm/src/address.rs | 160 +------ libk-mm/src/api.rs | 18 - libk-mm/src/device.rs | 105 +---- libk-mm/src/lib.rs | 36 +- libk-mm/src/phys/manager.rs | 16 +- libk-mm/src/phys/mod.rs | 4 +- libk-mm/src/phys/reserved.rs | 3 +- libk-mm/src/pointer.rs | 145 +------ libk-mm/src/table.rs | 79 ---- src/arch/aarch64/boot/mod.rs | 16 +- src/arch/aarch64/context.rs | 9 +- src/arch/aarch64/gic/mod.rs | 4 +- src/arch/aarch64/mem/mod.rs | 374 ---------------- src/arch/aarch64/mem/process.rs | 26 +- src/arch/aarch64/mem/table.rs | 335 -------------- src/arch/aarch64/mod.rs | 55 +-- src/arch/mod.rs | 48 --- src/arch/x86_64/acpi.rs | 4 +- src/arch/x86_64/apic/local.rs | 5 +- src/arch/x86_64/boot/mod.rs | 5 +- src/arch/x86_64/context.rs | 16 +- src/arch/x86_64/cpu.rs | 3 +- src/arch/x86_64/cpuid.rs | 3 +- src/arch/x86_64/exception.rs | 6 +- src/arch/x86_64/intrinsics.rs | 5 - src/arch/x86_64/mem/mod.rs | 362 ---------------- src/arch/x86_64/mem/process.rs | 35 +- src/arch/x86_64/mod.rs | 64 +-- src/arch/x86_64/smp.rs | 26 +- src/arch/x86_64/syscall.rs | 4 +- src/device/display/linear_fb.rs | 4 +- src/fs/mod.rs | 7 +- src/mem/address.rs | 39 -- src/mem/mod.rs | 7 +- src/mem/process.rs | 4 +- src/mem/table.rs | 52 --- src/proc/elf.rs | 7 +- src/proc/exec.rs | 3 +- src/syscall/mod.rs | 7 +- 61 files changed, 2040 insertions(+), 1946 deletions(-) create mode 100644 arch/aarch64/src/mem/mod.rs create mode 100644 arch/aarch64/src/mem/table.rs create mode 100644 arch/interface/src/mem/address.rs create mode 100644 arch/interface/src/mem/mod.rs create mode 100644 arch/interface/src/mem/table.rs create mode 100644 arch/x86_64/src/mem/mod.rs rename {src/arch/x86_64 => arch/x86_64/src}/mem/table.rs (88%) rename src/arch/x86_64/registers/mod.rs => arch/x86_64/src/registers.rs (100%) create mode 100644 libk-mm/interface/Cargo.toml create mode 100644 libk-mm/interface/src/address.rs create mode 100644 libk-mm/interface/src/lib.rs create mode 100644 libk-mm/interface/src/pointer.rs create mode 100644 libk-mm/interface/src/table.rs delete mode 100644 libk-mm/src/api.rs delete mode 100644 libk-mm/src/table.rs diff --git a/Cargo.toml b/Cargo.toml index 084d3dd8..2363a7eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ features = ["no_std_stream"] [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.3.1" device-tree = { path = "lib/device-tree" } +kernel-arch-aarch64 = { path = "arch/aarch64" } [target.'cfg(target_arch = "x86_64")'.dependencies] yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } @@ -64,6 +65,7 @@ acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } # TODO currently only supported here xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } ygg_driver_nvme = { path = "driver/block/nvme" } +kernel-arch-x86_64 = { path = "arch/x86_64" } [features] default = ["fb_console"] diff --git a/arch/aarch64/Cargo.toml b/arch/aarch64/Cargo.toml index 50af2b8a..6e5a7d95 100644 --- a/arch/aarch64/Cargo.toml +++ b/arch/aarch64/Cargo.toml @@ -6,7 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../interface" } +libk-mm-interface = { path = "../../libk-mm/interface" } +memtables = { path = "../../lib/memtables" } +bitflags = "2.3.3" +static_assertions = "1.1.0" aarch64-cpu = "9.3.1" tock-registers = "0.8.1" diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index 7f85495b..6f861732 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -1,9 +1,14 @@ #![no_std] +#![feature(effects, strict_provenance)] use aarch64_cpu::registers::DAIF; use kernel_arch_interface::Architecture; use tock_registers::interfaces::{ReadWriteable, Readable}; +pub mod mem; + +pub use mem::KernelTableManagerImpl; + pub struct ArchitectureImpl; impl Architecture for ArchitectureImpl { diff --git a/arch/aarch64/src/mem/mod.rs b/arch/aarch64/src/mem/mod.rs new file mode 100644 index 00000000..c2eaf9b7 --- /dev/null +++ b/arch/aarch64/src/mem/mod.rs @@ -0,0 +1,407 @@ +use core::{ + alloc::Layout, + ops::{Deref, DerefMut}, + ptr::addr_of, + sync::atomic::AtomicUsize, + sync::atomic::Ordering, +}; + +use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1}; +use kernel_arch_interface::{ + mem::{DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping}, + KERNEL_VIRT_OFFSET, +}; +use libk_mm_interface::{ + address::{FromRaw, PhysicalAddress}, + table::{EntryLevel, EntryLevelExt}, + KernelImageObject, +}; +use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; +use static_assertions::const_assert_eq; +use tock_registers::interfaces::Writeable; +use yggdrasil_abi::error::Error; + +use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}; + +pub mod table; + +#[derive(Debug)] +pub struct KernelTableManagerImpl; + +// TODO eliminate this requirement by using precomputed indices +const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET; +const KERNEL_PHYS_BASE: usize = 0x40080000; + +// Precomputed mappings +const KERNEL_L1_INDEX: usize = (KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE).page_index::(); +const KERNEL_START_L2_INDEX: usize = (KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE).page_index::(); +const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT; + +// Must not be zero, should be at 4MiB +const_assert_eq!(KERNEL_START_L2_INDEX, 0); +// From static mapping +const_assert_eq!(KERNEL_L1_INDEX, 1); + +// Runtime mappings +// 2MiB max +const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1; +// 1GiB max +const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; +// 1GiB max +const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; +const DEVICE_MAPPING_L3_COUNT: usize = 4; +// 16GiB max +const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3; +pub const RAM_MAPPING_L1_COUNT: usize = 16; + +// 2MiB for early mappings +const EARLY_MAPPING_OFFSET: usize = + MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE); +static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); +// 1GiB for heap mapping +pub const HEAP_MAPPING_OFFSET: usize = MAPPING_OFFSET | (HEAP_MAPPING_L1I * L1::SIZE); +pub static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); +// 1GiB for device MMIO mapping +const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE); +static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); +static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = + [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; +// 16GiB for RAM mapping +pub const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE); +pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0); + +#[link_section = ".data.tables"] +pub static mut KERNEL_TABLES: KernelImageObject = + unsafe { KernelImageObject::new(FixedTables::zeroed()) }; + +impl KernelTableManager for KernelTableManagerImpl { + fn virtualize(address: u64) -> usize { + let address = address as usize; + if address < MEMORY_LIMIT.load(Ordering::Acquire) { + address + RAM_MAPPING_OFFSET + } else { + panic!("Invalid physical address: {:#x}", address); + } + } + + fn physicalize(address: usize) -> u64 { + if address < RAM_MAPPING_OFFSET + || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire) + { + panic!("Not a virtualized physical address: {:#x}", address); + } + + (address - RAM_MAPPING_OFFSET) as _ + } + + unsafe fn map_device_pages( + base: u64, + count: usize, + attrs: DeviceMemoryAttributes, + ) -> Result, Error> { + map_device_memory(PhysicalAddress::from_raw(base), count, attrs) + } + + unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping) { + unmap_device_memory(mapping) + } +} + +/// Memory mapping which may be used for performing early kernel initialization +pub struct EarlyMapping<'a, T: ?Sized> { + value: &'a mut T, + page_count: usize, +} + +impl<'a, T: Sized> EarlyMapping<'a, T> { + pub unsafe fn map_slice( + physical: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); + + Ok(EarlyMapping { value, page_count }) + } +} + +impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { + fn drop(&mut self) { + let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); + + for i in 0..self.page_count { + let page = address + i * L3::SIZE; + + unsafe { + unmap_early_page(page); + } + } + } +} + +fn kernel_table_flags() -> PageAttributes { + PageAttributes::TABLE + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | PageAttributes::PRESENT +} + +fn ram_block_flags() -> PageAttributes { + // TODO UXN, PXN + PageAttributes::BLOCK + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | PageAttributes::PRESENT +} + +// Early mappings +unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { + for l3i in 0..512 { + let mut taken = false; + for i in 0..count { + if EARLY_MAPPING_L3[i + l3i].is_present() { + taken = true; + break; + } + } + + if taken { + continue; + } + + for i in 0..count { + let page = physical.add(i * L3::SIZE); + // TODO NX, NC + EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty()); + } + + return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn unmap_early_page(address: usize) { + if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { + panic!("Tried to unmap invalid early mapping: {:#x}", address); + } + + let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); + + assert!(EARLY_MAPPING_L3[l3i].is_present()); + EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; + + // TODO invalidate tlb +} + +pub unsafe fn map_ram_l1(index: usize) { + if index >= RAM_MAPPING_L1_COUNT { + todo!() + } + assert_eq!(KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I], 0); + + KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I] = + ((index * L1::SIZE) as u64) | ram_block_flags().bits(); +} + +pub unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) { + if index >= 512 { + todo!() + } + assert!(!HEAP_MAPPING_L2[index].is_present()); + // TODO UXN, PXN + HEAP_MAPPING_L2[index] = PageEntry::normal_block(page, PageAttributes::empty()); +} + +// Device mappings +unsafe fn map_device_memory_l3( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { + // TODO don't map pages if already mapped + + 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { + continue 'l0; + } + } + + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + // TODO NX, NC + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE)); + } + + return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn map_device_memory_l2( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { + 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { + for j in 0..count { + if DEVICE_MAPPING_L2[i + j].is_present() { + continue 'l0; + } + } + + for j in 0..count { + DEVICE_MAPPING_L2[i + j] = PageEntry::::device_block(base.add(j * L2::SIZE)); + } + + // log::debug!( + // "map l2s: base={:#x}, count={} -> {:#x}", + // base, + // count, + // DEVICE_MAPPING_OFFSET + i * L2::SIZE + // ); + return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); + } + + Err(Error::OutOfMemory) +} + +pub unsafe fn map_device_memory( + base: PhysicalAddress, + size: usize, + attrs: DeviceMemoryAttributes, +) -> Result, Error> { + // debugln!("Map {}B @ {:#x}", size, base); + let l3_aligned = base.page_align_down::(); + let l3_offset = base.page_offset::(); + let page_count = (l3_offset + size).page_count::(); + + if page_count > 256 { + // Large mapping, use L2 mapping instead + let l2_aligned = base.page_align_down::(); + let l2_offset = base.page_offset::(); + let page_count = (l2_offset + size).page_count::(); + + let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; + let address = base_address + l2_offset; + + Ok(RawDeviceMemoryMapping::from_raw_parts( + address, + base_address, + page_count, + L2::SIZE, + )) + } else { + // Just map the pages directly + let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; + let address = base_address + l3_offset; + + Ok(RawDeviceMemoryMapping::from_raw_parts( + address, + base_address, + page_count, + L3::SIZE, + )) + } +} + +pub unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { + // debugln!( + // "Unmap {}B @ {:#x}", + // map.page_count * map.page_size, + // map.base_address + // ); + match map.page_size { + L3::SIZE => { + for i in 0..map.page_count { + let page = map.base_address + i * L3::SIZE; + let l2i = page.page_index::(); + let l3i = page.page_index::(); + assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; + + tlb_flush_vaae1(page); + } + } + L2::SIZE => todo!(), + _ => unimplemented!(), + } +} + +#[inline] +pub fn tlb_flush_vaae1(mut page: usize) { + page >>= 12; + unsafe { + core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); + } +} + +/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half" +/// +/// # Safety +/// +/// Unsafe, must only be called by BSP during its early init while still in "lower-half" +pub unsafe fn load_fixed_tables() { + let ttbr0 = KERNEL_TABLES.l1.data.as_ptr() as u64; + TTBR0_EL1.set(ttbr0); + TTBR1_EL1.set(ttbr0); +} + +/// Sets up additional translation tables for kernel usage +/// +/// # Safety +/// +/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half" +pub unsafe fn init_fixed_tables() { + // TODO this could be built in compile-time too? + let early_mapping_l3_phys = addr_of!(EARLY_MAPPING_L3) as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = addr_of!(DEVICE_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = addr_of!(HEAP_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + + for i in 0..DEVICE_MAPPING_L3_COUNT { + let device_mapping_l3_phys = PhysicalAddress::from_raw( + &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, + ); + DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty()); + } + + assert_eq!(KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I], 0); + KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I] = + (early_mapping_l3_phys as u64) | kernel_table_flags().bits(); + + assert_eq!(KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I], 0); + KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I] = + (heap_mapping_l2_phys as u64) | kernel_table_flags().bits(); + + assert_eq!(KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I], 0); + KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I] = + (device_mapping_l2_phys as u64) | kernel_table_flags().bits(); +} diff --git a/arch/aarch64/src/mem/table.rs b/arch/aarch64/src/mem/table.rs new file mode 100644 index 00000000..6fd34b31 --- /dev/null +++ b/arch/aarch64/src/mem/table.rs @@ -0,0 +1,351 @@ +use core::{ + marker::PhantomData, + ops::{Index, IndexMut, Range}, +}; + +use bitflags::bitflags; +use libk_mm_interface::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + pointer::{PhysicalRef, PhysicalRefMut}, + table::{ + EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel, + TableAllocator, + }, +}; +use yggdrasil_abi::error::Error; + +use crate::{ArchitectureImpl, KernelTableManagerImpl}; + +// use abi::error::Error; +// use bitflags::bitflags; +// use libk_mm::{ +// address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, +// phys, +// pointer::{PhysicalRef, PhysicalRefMut}, +// table::EntryLevel, +// }; + +bitflags! { + #[derive(Clone, Copy, PartialEq, Eq)] + pub struct PageAttributes: u64 { + const PRESENT = 1 << 0; + + const TABLE = 1 << 1; + const PAGE = 1 << 1; + const BLOCK = 0 << 1; + + const ACCESS = 1 << 10; + + const AP_KERNEL_READWRITE = 0 << 6; + const AP_BOTH_READWRITE = 1 << 6; + const AP_KERNEL_READONLY = 2 << 6; + const AP_BOTH_READONLY = 3 << 6; + const AP_ACCESS_MASK = 3 << 6; + + const SH_OUTER = 2 << 8; + const SH_INNER = 3 << 8; + + const PAGE_ATTR_NORMAL = 0 << 2; + const PAGE_ATTR_DEVICE = 1 << 2; + + const NON_GLOBAL = 1 << 11; + + const PXN = 1 << 53; + const UXN = 1 << 54; + } +} + +#[derive(Clone, Copy)] +#[repr(C, align(0x1000))] +pub struct PageTable { + entries: [PageEntry; 512], +} + +#[derive(Clone, Copy)] +pub struct PageEntry(u64, PhantomData); + +#[derive(Clone, Copy)] +pub struct L1; +#[derive(Clone, Copy)] +pub struct L2; +#[derive(Clone, Copy)] +pub struct L3; + +impl NonTerminalEntryLevel for L1 { + type NextLevel = L2; +} + +impl NonTerminalEntryLevel for L2 { + type NextLevel = L3; +} + +impl EntryLevel for L1 { + const SHIFT: usize = 30; +} + +impl EntryLevel for L2 { + const SHIFT: usize = 21; +} + +impl EntryLevel for L3 { + const SHIFT: usize = 12; +} + +impl PageTable { + pub const fn zeroed() -> Self { + Self { + entries: [PageEntry::INVALID; 512], + } + } + + pub fn new_zeroed<'a, TA: TableAllocator>( + ) -> Result, Error> { + let physical = TA::allocate_page_table()?; + let mut table = + unsafe { PhysicalRefMut::<'a, Self, KernelTableManagerImpl>::map(physical) }; + + for i in 0..512 { + table[i] = PageEntry::INVALID; + } + + Ok(table) + } +} + +impl PageEntry { + pub const INVALID: Self = Self(0, PhantomData); + + pub const fn is_present(self) -> bool { + self.0 & PageAttributes::PRESENT.bits() != 0 + } + + pub fn attributes(self) -> PageAttributes { + PageAttributes::from_bits_retain(self.0) + } +} + +impl NextPageTable for PageTable { + type NextLevel = PageTable; + type TableRef = PhysicalRef<'static, PageTable, KernelTableManagerImpl>; + type TableRefMut = PhysicalRefMut<'static, PageTable, KernelTableManagerImpl>; + + fn get(&self, index: usize) -> Option { + self[index] + .as_table() + .map(|phys| unsafe { PhysicalRef::map(phys) }) + } + + fn get_mut(&mut self, index: usize) -> Option { + self[index] + .as_table() + .map(|phys| unsafe { PhysicalRefMut::map(phys) }) + } + + fn get_mut_or_alloc( + &mut self, + index: usize, + ) -> Result { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + Ok(unsafe { PhysicalRefMut::map(table) }) + } else { + let table = PageTable::new_zeroed::()?; + self[index] = PageEntry::::table( + unsafe { table.as_physical_address() }, + PageAttributes::empty(), + ); + Ok(table) + } + } +} + +impl EntryLevelDrop for PageTable { + const FULL_RANGE: Range = 0..512; + + // Do nothing + unsafe fn drop_range(&mut self, _range: Range) {} +} + +impl EntryLevelDrop for PageTable +where + PageTable: EntryLevelDrop, +{ + const FULL_RANGE: Range = 0..512; + + unsafe fn drop_range(&mut self, range: Range) { + for index in range { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + let mut table_ref: PhysicalRefMut, KernelTableManagerImpl> = + PhysicalRefMut::map(table); + + table_ref.drop_all::(); + + // Drop the table + drop(table_ref); + + TA::free_page_table(table); + } else if entry.is_present() { + // Memory must've been cleared beforehand, so no non-table entries must be present + panic!( + "Expected a table containing only tables, got table[{}] = {:#x?}", + index, entry.0 + ); + } + + self[index] = PageEntry::INVALID; + } + } +} + +impl PageEntry { + pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), + PhantomData, + ) + } + + pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_block(phys: PhysicalAddress) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | PageAttributes::UXN + | PageAttributes::PXN) + .bits(), + PhantomData, + ) + } + + /// Returns the physical address of the table this entry refers to, returning None if it + /// does not + pub fn as_table(self) -> Option { + if self.0 & PageAttributes::PRESENT.bits() != 0 + && self.0 & PageAttributes::BLOCK.bits() == 0 + { + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) + } else { + None + } + } +} + +impl PageEntry { + pub fn normal_page(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_page(phys: PhysicalAddress) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | PageAttributes::UXN + | PageAttributes::PXN) + .bits(), + PhantomData, + ) + } + + pub fn as_page(&self) -> Option { + let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); + if self.0 & mask == mask { + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) + } else { + None + } + } +} + +impl Index for PageTable { + type Output = PageEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +impl From for PageAttributes { + fn from(value: MapAttributes) -> Self { + let mut out = PageAttributes::empty(); + // TODO kernel cannot write el0 readonly pages + if value.contains(MapAttributes::USER_WRITE) { + // Read/write + out |= PageAttributes::AP_BOTH_READWRITE; + } else if value.contains(MapAttributes::USER_READ) { + // Read only + out |= PageAttributes::AP_BOTH_READONLY; + } else { + // No read/write + out |= PageAttributes::AP_KERNEL_READONLY; + } + + if value.contains(MapAttributes::NON_GLOBAL) { + out |= PageAttributes::NON_GLOBAL; + } + + out + } +} + +impl From for MapAttributes { + fn from(value: PageAttributes) -> Self { + let mut out = MapAttributes::empty(); + + out |= match value.intersection(PageAttributes::AP_ACCESS_MASK) { + PageAttributes::AP_BOTH_READWRITE => { + MapAttributes::USER_WRITE | MapAttributes::USER_READ + } + PageAttributes::AP_BOTH_READONLY => MapAttributes::USER_READ, + PageAttributes::AP_KERNEL_READONLY => MapAttributes::empty(), + PageAttributes::AP_KERNEL_READWRITE => panic!("This variant cannot be constructed"), + _ => unreachable!(), + }; + + if value.contains(PageAttributes::NON_GLOBAL) { + out |= MapAttributes::NON_GLOBAL; + } + + out + } +} diff --git a/arch/interface/Cargo.toml b/arch/interface/Cargo.toml index 8714e738..7161facb 100644 --- a/arch/interface/Cargo.toml +++ b/arch/interface/Cargo.toml @@ -6,3 +6,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs index f1aed0c8..65c3e138 100644 --- a/arch/interface/src/lib.rs +++ b/arch/interface/src/lib.rs @@ -1,4 +1,9 @@ #![no_std] +#![feature(step_trait, effects, const_trait_impl)] + +pub mod mem; + +pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; pub trait Architecture { fn interrupt_mask() -> bool; diff --git a/arch/interface/src/mem/address.rs b/arch/interface/src/mem/address.rs new file mode 100644 index 00000000..e69de29b diff --git a/arch/interface/src/mem/mod.rs b/arch/interface/src/mem/mod.rs new file mode 100644 index 00000000..5dcec697 --- /dev/null +++ b/arch/interface/src/mem/mod.rs @@ -0,0 +1,111 @@ +use core::{fmt, marker::PhantomData, mem::size_of, ptr::NonNull}; + +use yggdrasil_abi::error::Error; + +pub mod address; +pub mod table; + +#[derive(Debug, Default, Clone, Copy)] +pub enum DeviceMemoryCaching { + #[default] + None, + Cacheable, +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct DeviceMemoryAttributes { + pub caching: DeviceMemoryCaching, +} + +/// Describes a single device memory mapping +#[derive(Debug)] +pub struct RawDeviceMemoryMapping { + /// Virtual address of the mapped object + pub address: usize, + /// Base address of the mapping start + pub base_address: usize, + /// Page size used for the mapping + pub page_size: usize, + /// Number of pages used to map the object + pub page_count: usize, + + _manager: PhantomData, +} + +pub trait KernelTableManager: Sized + fmt::Debug { + fn virtualize(phys: u64) -> usize; + fn physicalize(virt: usize) -> u64; + + unsafe fn map_device_pages( + base: u64, + count: usize, + attrs: DeviceMemoryAttributes, + ) -> Result, Error>; + unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping); +} + +impl RawDeviceMemoryMapping { + /// Maps a region of physical memory as device memory of given size. + /// + /// # Safety + /// + /// The caller must ensure proper access synchronization, as well as the address' origin. + #[inline] + pub unsafe fn map( + base: u64, + size: usize, + attrs: DeviceMemoryAttributes, + ) -> Result { + A::map_device_pages(base, size, attrs) + } + + /// Consumes the device mapping, leaking its address without deallocating the translation + /// mapping itself + pub fn leak(self) -> usize { + let address = self.address; + core::mem::forget(self); + address + } + + pub fn into_raw_parts(self) -> (usize, usize, usize, usize) { + let address = self.address; + let base_address = self.base_address; + let page_count = self.page_count; + let page_size = self.page_size; + + core::mem::forget(self); + + (address, base_address, page_count, page_size) + } + + pub unsafe fn from_raw_parts( + address: usize, + base_address: usize, + page_count: usize, + page_size: usize, + ) -> Self { + Self { + address, + base_address, + page_count, + page_size, + _manager: PhantomData, + } + } + + /// "Casts" the mapping to a specific type T and returns a [NonNull] pointer to it + pub unsafe fn as_non_null(&self) -> NonNull { + if self.page_size * self.page_count < size_of::() { + panic!(); + } + NonNull::new_unchecked(self.address as *mut T) + } +} + +impl Drop for RawDeviceMemoryMapping { + fn drop(&mut self) { + unsafe { + A::unmap_device_pages(self); + } + } +} diff --git a/arch/interface/src/mem/table.rs b/arch/interface/src/mem/table.rs new file mode 100644 index 00000000..e69de29b diff --git a/arch/src/lib.rs b/arch/src/lib.rs index 1a53ef3e..39c6d94d 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -2,8 +2,6 @@ use cfg_if::cfg_if; -pub use kernel_arch_interface::Architecture; - /// Returns an absolute address to the given symbol #[macro_export] macro_rules! absolute_address { @@ -31,4 +29,6 @@ cfg_if! { } } -pub use imp::ArchitectureImpl; +pub use imp::{ArchitectureImpl, KernelTableManagerImpl}; + +pub use kernel_arch_interface::{mem, Architecture}; diff --git a/arch/x86_64/Cargo.toml b/arch/x86_64/Cargo.toml index 9501db4a..a78d7d10 100644 --- a/arch/x86_64/Cargo.toml +++ b/arch/x86_64/Cargo.toml @@ -6,4 +6,11 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../interface" } +libk-mm-interface = { path = "../../libk-mm/interface" } +memtables = { path = "../../lib/memtables" } + +bitflags = "2.3.3" +static_assertions = "1.1.0" +tock-registers = "0.8.1" diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index bff7e580..f14e013d 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -1,9 +1,17 @@ #![no_std] +#![feature(effects, strict_provenance)] use kernel_arch_interface::Architecture; +pub mod mem; +pub mod registers; + +pub use mem::KernelTableManagerImpl; + pub struct ArchitectureImpl; +pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; + impl Architecture for ArchitectureImpl { fn interrupt_mask() -> bool { let mut flags: u64; diff --git a/arch/x86_64/src/mem/mod.rs b/arch/x86_64/src/mem/mod.rs new file mode 100644 index 00000000..0d469b54 --- /dev/null +++ b/arch/x86_64/src/mem/mod.rs @@ -0,0 +1,404 @@ +use core::{ + alloc::Layout, + ops::{Deref, DerefMut}, + ptr::addr_of, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use kernel_arch_interface::mem::{ + DeviceMemoryAttributes, KernelTableManager, RawDeviceMemoryMapping, +}; +use libk_mm_interface::{ + address::{FromRaw, PhysicalAddress}, + table::{EntryLevel, EntryLevelExt}, + KernelImageObject, +}; +use memtables::x86_64::FixedTables; +use static_assertions::{const_assert_eq, const_assert_ne}; +use yggdrasil_abi::error::Error; + +use crate::{registers::CR3, KERNEL_VIRT_OFFSET}; + +use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3}; + +pub mod table; + +#[derive(Debug)] +pub struct KernelTableManagerImpl; + +const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; +const KERNEL_PHYS_BASE: usize = 0x200000; + +// Mapped at compile time +const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE; +const KERNEL_L0_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); +const KERNEL_L1_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); +const KERNEL_START_L2_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); + +// Must not be zero, should be at 4MiB +const_assert_ne!(KERNEL_START_L2_INDEX, 0); +// From static mapping +const_assert_eq!(KERNEL_L0_INDEX, 511); +const_assert_eq!(KERNEL_L1_INDEX, 0); + +// Mapped at boot +const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1; +const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; +const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; +const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1; + +const DEVICE_MAPPING_L3_COUNT: usize = 4; + +#[link_section = ".data.tables"] +pub static mut KERNEL_TABLES: KernelImageObject = + unsafe { KernelImageObject::new(FixedTables::zeroed()) }; + +// 2MiB for early mappings +const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK + | (KERNEL_L0_INDEX * L0::SIZE) + | (KERNEL_L1_INDEX * L1::SIZE) + | (EARLY_MAPPING_L2I * L2::SIZE); +static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); +// 1GiB for heap mapping +pub const HEAP_MAPPING_OFFSET: usize = + CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (HEAP_MAPPING_L1I * L1::SIZE); +pub(super) static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); +// 1GiB for device MMIO mapping +const DEVICE_MAPPING_OFFSET: usize = + CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE); +static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); +static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = + [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; +// 512GiB for whole RAM mapping +pub const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE); +pub static MEMORY_LIMIT: AtomicUsize = AtomicUsize::new(0); +pub static mut RAM_MAPPING_L1: PageTable = PageTable::zeroed(); + +impl KernelTableManager for KernelTableManagerImpl { + fn virtualize(address: u64) -> usize { + let address = address as usize; + if address < MEMORY_LIMIT.load(Ordering::Acquire) { + address + RAM_MAPPING_OFFSET + } else { + panic!("Invalid physical address: {:#x}", address); + } + } + + fn physicalize(address: usize) -> u64 { + if address < RAM_MAPPING_OFFSET + || address - RAM_MAPPING_OFFSET >= MEMORY_LIMIT.load(Ordering::Acquire) + { + panic!("Not a virtualized physical address: {:#x}", address); + } + + (address - RAM_MAPPING_OFFSET) as _ + } + + unsafe fn map_device_pages( + base: u64, + count: usize, + attrs: DeviceMemoryAttributes, + ) -> Result, Error> { + map_device_memory(PhysicalAddress::from_raw(base), count, attrs) + } + + unsafe fn unmap_device_pages(mapping: &RawDeviceMemoryMapping) { + unmap_device_memory(mapping) + } +} + +// Early mappings +unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { + for l3i in 0..512 { + let mut taken = false; + for i in 0..count { + if EARLY_MAPPING_L3[i + l3i].is_present() { + taken = true; + break; + } + } + + if taken { + continue; + } + + for i in 0..count { + // TODO NX, NC + EARLY_MAPPING_L3[i + l3i] = + PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE); + } + + return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn unmap_early_page(address: usize) { + if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { + panic!("Tried to unmap invalid early mapping: {:#x}", address); + } + + let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); + + assert!(EARLY_MAPPING_L3[l3i].is_present()); + EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; +} + +// Device mappings +unsafe fn map_device_memory_l3( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { + // TODO don't map pages if already mapped + + 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { + continue 'l0; + } + } + + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + // TODO NX, NC + DEVICE_MAPPING_L3S[l2i][l3i] = + PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE); + } + + return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn map_device_memory_l2( + base: PhysicalAddress, + count: usize, + _attrs: DeviceMemoryAttributes, +) -> Result { + 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { + for j in 0..count { + if DEVICE_MAPPING_L2[i + j].is_present() { + continue 'l0; + } + } + + for j in 0..count { + DEVICE_MAPPING_L2[i + j] = + PageEntry::::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE); + } + + // debugln!( + // "map l2s: base={:#x}, count={} -> {:#x}", + // base, + // count, + // DEVICE_MAPPING_OFFSET + i * L2::SIZE + // ); + return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn map_device_memory( + base: PhysicalAddress, + size: usize, + attrs: DeviceMemoryAttributes, +) -> Result, Error> { + // debugln!("Map {}B @ {:#x}", size, base); + let l3_aligned = base.page_align_down::(); + let l3_offset = base.page_offset::(); + let page_count = (l3_offset + size).page_count::(); + + if page_count > 256 { + // Large mapping, use L2 mapping instead + let l2_aligned = base.page_align_down::(); + let l2_offset = base.page_offset::(); + let page_count = (l2_offset + size).page_count::(); + + let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; + let address = base_address + l2_offset; + + Ok(RawDeviceMemoryMapping::from_raw_parts( + address, + base_address, + page_count, + L2::SIZE, + )) + } else { + // Just map the pages directly + let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; + let address = base_address + l3_offset; + + Ok(RawDeviceMemoryMapping::from_raw_parts( + address, + base_address, + page_count, + L3::SIZE, + )) + } +} + +unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { + // debugln!( + // "Unmap {}B @ {:#x}", + // map.page_count * map.page_size, + // map.base_address + // ); + match map.page_size { + L3::SIZE => { + for i in 0..map.page_count { + let page = map.base_address + i * L3::SIZE; + let l2i = page.page_index::(); + let l3i = page.page_index::(); + assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; + flush_tlb_entry(page); + } + } + L2::SIZE => todo!(), + _ => unimplemented!(), + } +} + +pub unsafe fn map_heap_block(index: usize, page: PhysicalAddress) { + if !page.is_page_aligned_for::() { + panic!("Attempted to map a misaligned 2MiB page"); + } + assert!(index < 512); + + if HEAP_MAPPING_L2[index].is_present() { + panic!("Page is already mappged: {:#x}", page); + } + + // TODO NX + HEAP_MAPPING_L2[index] = PageEntry::::block(page, PageAttributes::WRITABLE); +} + +/// Memory mapping which may be used for performing early kernel initialization +pub struct EarlyMapping<'a, T: ?Sized> { + value: &'a mut T, + page_count: usize, +} + +impl<'a, T: Sized> EarlyMapping<'a, T> { + pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { + let layout = Layout::new::(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = &mut *((virt + offset) as *mut T); + + Ok(EarlyMapping { value, page_count }) + } + + pub unsafe fn map_slice( + physical: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); + + Ok(EarlyMapping { value, page_count }) + } +} + +impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { + fn drop(&mut self) { + let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); + + for i in 0..self.page_count { + let page = address + i * L3::SIZE; + + unsafe { + unmap_early_page(page); + } + } + } +} + +pub fn clone_kernel_tables(dst: &mut PageTable) { + unsafe { + dst[KERNEL_L0_INDEX] = PageEntry::from_raw(KERNEL_TABLES.l0.data[KERNEL_L0_INDEX]); + dst[RAM_MAPPING_L0I] = PageEntry::from_raw(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I]); + } +} + +/// Sets up the following memory map: +/// ...: KERNEL_TABLES.l0: +/// * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000 : RAM_MAPPING_L1 +/// * 0xFFFFFF8000000000 .. ... : KERNEL_TABLES.kernel_l1: +/// * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000 : KERNEL_TABLES.kernel_l2 +/// * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000 : --- +/// * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000 : EARLY_MAPPING_L3 +/// * 0xFFFFFF8000400000 .. ... : KERNEL_TABLES.kernel_l3s +/// * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000 : HEAP_MAPPING_L2 +/// * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000 : DEVICE_MAPPING_L2 +/// * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000 : DEVICE_MAPPING_L3S +/// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ... +pub unsafe fn init_fixed_tables() { + // TODO this could be built in compile-time too? + let early_mapping_l3_phys = addr_of!(EARLY_MAPPING_L3) as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = addr_of!(DEVICE_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = addr_of!(HEAP_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; + let ram_mapping_l1_phys = addr_of!(RAM_MAPPING_L1) as usize - KERNEL_VIRT_OFFSET; + + for i in 0..DEVICE_MAPPING_L3_COUNT { + let device_mapping_l3_phys = PhysicalAddress::from_raw( + &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, + ); + DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE); + } + + assert_eq!(KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I], 0); + KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64) + | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + assert_eq!(KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I], 0); + KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I] = + (heap_mapping_l2_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + assert_eq!(KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I], 0); + KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64) + | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + assert_eq!(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I], 0); + KERNEL_TABLES.l0.data[RAM_MAPPING_L0I] = + (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); + + // TODO ENABLE EFER.NXE + let cr3 = &KERNEL_TABLES.l0 as *const _ as usize - KERNEL_VIRT_OFFSET; + CR3.set_address(cr3); +} + +#[inline] +pub unsafe fn flush_tlb_entry(address: usize) { + core::arch::asm!("invlpg ({0})", in(reg) address, options(att_syntax)); +} diff --git a/src/arch/x86_64/mem/table.rs b/arch/x86_64/src/mem/table.rs similarity index 88% rename from src/arch/x86_64/mem/table.rs rename to arch/x86_64/src/mem/table.rs index eec58785..354b3fec 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/arch/x86_64/src/mem/table.rs @@ -4,17 +4,18 @@ use core::{ ops::{Index, IndexMut, Range}, }; -use abi::error::Error; use bitflags::bitflags; -use libk_mm::{ +use libk_mm_interface::{ address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, - phys, pointer::{PhysicalRef, PhysicalRefMut}, + table::{ + EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel, + TableAllocator, + }, }; +use yggdrasil_abi::error::Error; -use crate::mem::table::{ - EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel, -}; +use crate::KernelTableManagerImpl; bitflags! { /// Describes how each page table entry is mapped @@ -199,9 +200,11 @@ impl PageTable { } /// Allocates a new page table, filling it with non-preset entries - pub fn new_zeroed<'a>() -> Result, Error> { - let physical = phys::alloc_page()?; - let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) }; + pub fn new_zeroed<'a, TA: TableAllocator>( + ) -> Result, Error> { + let physical = TA::allocate_page_table()?; + let mut table = + unsafe { PhysicalRefMut::<'a, Self, KernelTableManagerImpl>::map(physical) }; for i in 0..512 { table[i] = PageEntry::INVALID; @@ -218,8 +221,8 @@ impl PageTable { impl NextPageTable for PageTable { type NextLevel = PageTable; - type TableRef = PhysicalRef<'static, Self::NextLevel>; - type TableRefMut = PhysicalRefMut<'static, Self::NextLevel>; + type TableRef = PhysicalRef<'static, Self::NextLevel, KernelTableManagerImpl>; + type TableRefMut = PhysicalRefMut<'static, Self::NextLevel, KernelTableManagerImpl>; fn get(&self, index: usize) -> Option { self[index] @@ -233,13 +236,16 @@ impl NextPageTable for PageTable { .map(|addr| unsafe { PhysicalRefMut::map(addr) }) } - fn get_mut_or_alloc(&mut self, index: usize) -> Result { + fn get_mut_or_alloc( + &mut self, + index: usize, + ) -> Result { let entry = self[index]; if let Some(table) = entry.as_table() { Ok(unsafe { PhysicalRefMut::map(table) }) } else { - let table = PageTable::new_zeroed()?; + let table = PageTable::new_zeroed::()?; self[index] = PageEntry::::table( unsafe { table.as_physical_address() }, PageAttributes::WRITABLE | PageAttributes::USER, @@ -253,7 +259,7 @@ impl EntryLevelDrop for PageTable { const FULL_RANGE: Range = 0..512; // Do nothing - unsafe fn drop_range(&mut self, _range: Range) {} + unsafe fn drop_range(&mut self, _range: Range) {} } impl EntryLevelDrop for PageTable @@ -262,20 +268,20 @@ where { const FULL_RANGE: Range = 0..512; - unsafe fn drop_range(&mut self, range: Range) { + unsafe fn drop_range(&mut self, range: Range) { for index in range { let entry = self[index]; if let Some(table) = entry.as_table() { - let mut table_ref: PhysicalRefMut> = + let mut table_ref: PhysicalRefMut, KernelTableManagerImpl> = PhysicalRefMut::map(table); - table_ref.drop_all(); + table_ref.drop_all::(); // Drop the table drop(table_ref); - phys::free_page(table); + TA::free_page_table(table); } else if entry.is_present() { // Memory must've been cleared beforehand, so no non-table entries must be present panic!( diff --git a/src/arch/x86_64/registers/mod.rs b/arch/x86_64/src/registers.rs similarity index 100% rename from src/arch/x86_64/registers/mod.rs rename to arch/x86_64/src/registers.rs diff --git a/libk-mm/Cargo.toml b/libk-mm/Cargo.toml index 7971cc2e..59527852 100644 --- a/libk-mm/Cargo.toml +++ b/libk-mm/Cargo.toml @@ -9,5 +9,6 @@ edition = "2021" yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch = { path = "../arch" } libk-util = { path = "../libk-util" } +libk-mm-interface = { path = "interface" } log = "0.4.20" diff --git a/libk-mm/interface/Cargo.toml b/libk-mm/interface/Cargo.toml new file mode 100644 index 00000000..2efbb6a6 --- /dev/null +++ b/libk-mm/interface/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "libk-mm-interface" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } + +kernel-arch-interface = { path = "../../arch/interface" } + +bitflags = "2.3.3" diff --git a/libk-mm/interface/src/address.rs b/libk-mm/interface/src/address.rs new file mode 100644 index 00000000..4fac2a4f --- /dev/null +++ b/libk-mm/interface/src/address.rs @@ -0,0 +1,160 @@ +use core::{ + fmt, + iter::Step, + mem::align_of, + ops::{Add, Sub}, +}; + +use kernel_arch_interface::{mem::KernelTableManager, Architecture}; + +/// Wrapper type to represent a physical memory address +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[repr(transparent)] +pub struct PhysicalAddress(pub(crate) u64); + +/// Interface for converting addresses from their raw values to more specific types +#[const_trait] +pub trait FromRaw { + /// Converts a raw value into the address wrapper type + fn from_raw(value: T) -> Self; +} + +/// Interface for converting wrapper types into their raw address representations +#[const_trait] +pub trait IntoRaw { + /// Converts a wrapper type value into its raw address + fn into_raw(self) -> T; +} + +/// Interface for obtaining physical addresses of values +pub trait AsPhysicalAddress { + /// Returns the value's physical address. + /// + /// # Safety + /// + /// The caller must ensure the value has been constructed and obtained through proper means. + unsafe fn as_physical_address(&self) -> PhysicalAddress; +} + +impl PhysicalAddress { + /// Physical address of zero + pub const ZERO: Self = Self(0); + + /// Maximum representable physical address + pub const MAX: Self = Self(u64::MAX); + /// Minumum representable physical address + pub const MIN: Self = Self(u64::MIN); + + /// Applies an offset to the address + pub const fn add(self, offset: usize) -> Self { + Self(self.0 + offset as u64) + } + + /// Returns `true` if the address is zero + #[inline(always)] + pub const fn is_zero(self) -> bool { + self.0 == 0 + } + + /// Returns `true` if the address is aligned to a boundary of a page at level `L` + #[inline] + pub const fn is_aligned_for(self) -> bool { + self.0 as usize % align_of::() == 0 + } + + /// Converts a previously virtualized physical address back into its physical form. + /// + /// # Safety + /// + /// The caller must ensure the function only receives addresses obtained through + /// [PhysicalAddress::virtualize_raw] or + /// [super::pointer::PhysicalRef]/[super::pointer::PhysicalRefMut] facilities. + #[inline] + pub unsafe fn raw_from_virtualized(address: usize) -> Self { + PhysicalAddress(K::physicalize(address)) + } + + /// Converts the physical address to a virtual one + #[inline] + pub fn raw_virtualize(self) -> usize { + K::virtualize(self.0) + } +} + +impl Add for PhysicalAddress { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl Sub for PhysicalAddress { + type Output = usize; + + fn sub(self, rhs: Self) -> Self::Output { + (self.0 - rhs.0) as usize + } +} + +// Conversions + +impl const FromRaw for PhysicalAddress { + fn from_raw(value: u64) -> Self { + Self(value) + } +} + +impl const FromRaw for PhysicalAddress { + fn from_raw(value: usize) -> Self { + Self(value as u64) + } +} + +impl const IntoRaw for PhysicalAddress { + fn into_raw(self) -> u64 { + self.0 + } +} + +impl const IntoRaw for PhysicalAddress { + fn into_raw(self) -> usize { + self.0 as usize + } +} + +impl From for u64 { + fn from(addr: PhysicalAddress) -> u64 { + addr.0 + } +} + +impl From for usize { + fn from(addr: PhysicalAddress) -> usize { + addr.0 as usize + } +} + +// Ranges + +impl Step for PhysicalAddress { + fn steps_between(_start: &Self, _end: &Self) -> Option { + todo!() + } + + fn forward_checked(start: Self, count: usize) -> Option { + start.0.checked_add(count as u64).map(Self) + } + + fn backward_checked(_start: Self, _count: usize) -> Option { + todo!() + } +} + +// fmt + +impl fmt::LowerHex for PhysicalAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} diff --git a/libk-mm/interface/src/lib.rs b/libk-mm/interface/src/lib.rs new file mode 100644 index 00000000..d2c5feae --- /dev/null +++ b/libk-mm/interface/src/lib.rs @@ -0,0 +1,50 @@ +#![no_std] +#![feature(step_trait, const_trait_impl, effects, strict_provenance)] + +use core::ops::{Deref, DerefMut}; + +use address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; +use kernel_arch_interface::KERNEL_VIRT_OFFSET; + +pub mod address; +pub mod pointer; +pub mod table; + +/// Wrapper type to represent an object residing within the kernel +#[repr(transparent)] +pub struct KernelImageObject { + inner: T, +} + +// KernelImageObject wrapper for objects inside the kernel + +impl KernelImageObject { + /// Wraps a value in the [KernelImageObject], allowing its physical address calculation. + /// + /// # Safety + /// + /// The caller must ensure the T is a `static (mut)` binding inside the kernel. + pub const unsafe fn new(inner: T) -> Self { + Self { inner } + } +} + +impl AsPhysicalAddress for KernelImageObject { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::from_raw(&self.inner as *const _ as usize - KERNEL_VIRT_OFFSET) + } +} + +impl Deref for KernelImageObject { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for KernelImageObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} diff --git a/libk-mm/interface/src/pointer.rs b/libk-mm/interface/src/pointer.rs new file mode 100644 index 00000000..0efea751 --- /dev/null +++ b/libk-mm/interface/src/pointer.rs @@ -0,0 +1,163 @@ +use core::{ + fmt, + marker::PhantomData, + ops::{Deref, DerefMut}, +}; + +use kernel_arch_interface::mem::KernelTableManager; + +use crate::address::{AsPhysicalAddress, PhysicalAddress}; + +/// Wrapper for immutably accessing a value at a physical address +#[repr(transparent)] +pub struct PhysicalRef<'a, T: ?Sized, K: KernelTableManager> { + value: &'a T, + _pd: PhantomData, +} + +/// Wrapper for mutably accessing a value at a physical address +#[repr(transparent)] +pub struct PhysicalRefMut<'a, T: ?Sized, K: KernelTableManager> { + value: &'a mut T, + _pd: PhantomData, +} + +// PhysicalRefMut wrapper for safe mutable access to physical addresses + +impl<'a, T: Sized, K: KernelTableManager> PhysicalRefMut<'a, T, K> { + /// Maps a physical address into the kernel space as &mut T, allowing mmutable access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains T. The caller must also take care of access synchronization and make sure no + /// aliasing occurs. + pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRefMut<'a, T, K> { + let value = virtualize_raw::<_, K>(physical); + PhysicalRefMut { + value, + _pd: PhantomData, + } + } + + /// Maps a physical address into the kernel space as &mut [T], allowing mmutable access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains [T; len]. The caller must also take care of access synchronization and make + /// sure no aliasing occurs. + pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRefMut<'a, [T], K> { + let value = virtualize_slice_raw::<_, K>(physical, len); + PhysicalRefMut { + value, + _pd: PhantomData, + } + } +} + +impl PhysicalRefMut<'_, T, K> { + /// Returns the "address" part of the reference + #[inline] + pub fn as_address(&self) -> usize { + (self.value as *const T).addr() + } +} + +impl AsPhysicalAddress for PhysicalRefMut<'_, T, K> { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::raw_from_virtualized::(PhysicalRefMut::::as_address(self)) + } +} + +impl Deref for PhysicalRefMut<'_, T, K> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl DerefMut for PhysicalRefMut<'_, T, K> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl fmt::Pointer for PhysicalRefMut<'_, T, K> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Pointer::fmt(&self.value, f) + } +} + +// PhysicalRef: same as PhysicalRefMut, except immutable + +impl<'a, T: Sized, K: KernelTableManager> PhysicalRef<'a, T, K> { + /// Maps a physical address into the kernel space as &T, allowing immutable access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains T. + pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRef<'a, T, K> { + let value = virtualize_raw::<_, K>(physical); + PhysicalRef { + value, + _pd: PhantomData, + } + } + + /// Maps a physical address into the kernel space as &[T] of given len, allowing immutable + /// access to it. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address as well that it actually + /// contains [T; len]. + pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRef<'a, [T], K> { + let value = virtualize_slice_raw::<_, K>(physical, len); + PhysicalRef { + value, + _pd: PhantomData, + } + } +} + +impl PhysicalRef<'_, T, K> { + /// Returns the "address" part of the reference + #[inline] + pub fn as_address(&self) -> usize { + (self.value as *const T).addr() + } +} + +impl AsPhysicalAddress for PhysicalRef<'_, T, K> { + unsafe fn as_physical_address(&self) -> PhysicalAddress { + PhysicalAddress::raw_from_virtualized::(PhysicalRef::::as_address(self)) + } +} + +impl Deref for PhysicalRef<'_, T, K> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +unsafe fn virtualize_raw<'a, T: Sized, K: KernelTableManager>( + physical: PhysicalAddress, +) -> &'a mut T { + // TODO check align + let address = physical.raw_virtualize::(); + &mut *(address as *mut T) +} + +unsafe fn virtualize_slice_raw<'a, T: Sized, K: KernelTableManager>( + physical: PhysicalAddress, + len: usize, +) -> &'a mut [T] { + // TODO check align + let address = physical.raw_virtualize::(); + core::slice::from_raw_parts_mut(address as *mut T, len) +} diff --git a/libk-mm/interface/src/table.rs b/libk-mm/interface/src/table.rs new file mode 100644 index 00000000..e1f5323a --- /dev/null +++ b/libk-mm/interface/src/table.rs @@ -0,0 +1,144 @@ +use core::ops::{Deref, DerefMut, Range}; + +use bitflags::bitflags; +use yggdrasil_abi::error::Error; + +use super::address::PhysicalAddress; + +/// Interface for a single level of address translation +pub trait EntryLevel: Copy { + /// The right shift needed to obtain an index of an entry at this level from an address + const SHIFT: usize; + /// The size of a page at this entry level + const SIZE: usize = 1 << Self::SHIFT; +} + +pub trait TableAllocator { + fn allocate_page_table() -> Result; + unsafe fn free_page_table(address: PhysicalAddress); +} + +// TODO EXECUTABLE +bitflags! { + /// Describes how a page translation mapping should behave + #[derive(Clone, Copy)] + pub struct MapAttributes: u64 { + /// The data mapped can be read by the user process + const USER_READ = 1 << 0; + /// The data mapped can be written to by the user process + const USER_WRITE = 1 << 1; + /// The mapping is not global across the address spaces + const NON_GLOBAL = 1 << 2; + } +} + +#[const_trait] +pub trait EntryLevelExt: Sized { + fn page_index(&self) -> usize; + fn page_offset(&self) -> usize; + fn page_count(&self) -> usize; + fn page_align_up(&self) -> Self; + fn page_align_down(&self) -> Self; + fn is_page_aligned_for(&self) -> bool; +} + +#[const_trait] +trait AddressLike: Sized + Copy { + fn into_usize(self) -> usize; + fn from_usize(v: usize) -> Self; +} + +/// Interface for destroying memory translation tables +pub trait EntryLevelDrop { + /// Range covering the whole table + const FULL_RANGE: Range; + + /// Recursively destroys the specified range within the table + unsafe fn drop_range(&mut self, range: Range); + + /// Recursively destroys all the entries in within the table + unsafe fn drop_all(&mut self) { + self.drop_range::(Self::FULL_RANGE) + } +} + +/// Interface for non-terminal tables to retrieve the next level of address translation tables +pub trait NextPageTable { + /// Type for the next-level page table + type NextLevel; + /// Type for an immutable reference to the next-level page table + type TableRef: Deref; + /// Type for a mutable reference to the next-level page table + type TableRefMut: DerefMut; + + /// Tries looking up a next-level table at given index, allocating and mapping one if it is not + /// present there + fn get_mut_or_alloc( + &mut self, + index: usize, + ) -> Result; + /// Returns a mutable reference to a next-level table at `index`, if present + fn get_mut(&mut self, index: usize) -> Option; + /// Returns an immutable reference to a next-level table at `index`, if present + fn get(&self, index: usize) -> Option; +} + +/// Tag trait to mark that the page table level may point to a next-level table +pub trait NonTerminalEntryLevel: EntryLevel { + /// Tag type of the level this entry level may point to + type NextLevel: EntryLevel; +} + +impl const AddressLike for usize { + #[inline(always)] + fn into_usize(self) -> usize { + self + } + + #[inline(always)] + fn from_usize(v: usize) -> Self { + v + } +} + +impl const AddressLike for PhysicalAddress { + fn from_usize(v: usize) -> Self { + Self(v as _) + } + + fn into_usize(self) -> usize { + self.0 as _ + } +} + +impl const EntryLevelExt for T { + #[inline(always)] + fn page_index(&self) -> usize { + (self.into_usize() >> L::SHIFT) & 0x1FF + } + + #[inline(always)] + fn page_offset(&self) -> usize { + self.into_usize() & (L::SIZE - 1) + } + + #[inline(always)] + fn page_count(&self) -> usize { + (self.into_usize() + L::SIZE - 1) / L::SIZE + } + + #[inline(always)] + fn page_align_up(&self) -> Self { + Self::from_usize((self.into_usize() + L::SIZE - 1) & !(L::SIZE - 1)) + } + + #[inline(always)] + fn page_align_down(&self) -> Self { + Self::from_usize(self.into_usize() & !(L::SIZE - 1)) + } + + #[inline(always)] + fn is_page_aligned_for(&self) -> bool { + self.page_offset::() == 0 + } +} diff --git a/libk-mm/src/address.rs b/libk-mm/src/address.rs index e37584ce..04994a98 100644 --- a/libk-mm/src/address.rs +++ b/libk-mm/src/address.rs @@ -1,157 +1,17 @@ -use crate::api::{__physicalize, __virtualize}; -use core::{ - fmt, - iter::Step, - mem::align_of, - ops::{Add, Sub}, -}; +use kernel_arch::{mem::KernelTableManager, KernelTableManagerImpl}; +pub use libk_mm_interface::address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}; -/// Wrapper type to represent a physical memory address -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] -#[repr(transparent)] -pub struct PhysicalAddress(pub(crate) u64); - -/// Interface for converting addresses from their raw values to more specific types -#[const_trait] -pub trait FromRaw { - /// Converts a raw value into the address wrapper type - fn from_raw(value: T) -> Self; +pub trait Virtualize { + fn virtualize(self) -> usize; + fn from_virtualized(value: usize) -> Self; } -/// Interface for converting wrapper types into their raw address representations -#[const_trait] -pub trait IntoRaw { - /// Converts a wrapper type value into its raw address - fn into_raw(self) -> T; -} - -/// Interface for obtaining physical addresses of values -pub trait AsPhysicalAddress { - /// Returns the value's physical address. - /// - /// # Safety - /// - /// The caller must ensure the value has been constructed and obtained through proper means. - unsafe fn as_physical_address(&self) -> PhysicalAddress; -} - -impl PhysicalAddress { - /// Physical address of zero - pub const ZERO: Self = Self(0); - - /// Maximum representable physical address - pub const MAX: Self = Self(u64::MAX); - /// Minumum representable physical address - pub const MIN: Self = Self(u64::MIN); - - /// Applies an offset to the address - pub const fn add(self, offset: usize) -> Self { - Self(self.0 + offset as u64) +impl Virtualize for PhysicalAddress { + fn virtualize(self) -> usize { + KernelTableManagerImpl::virtualize(self.into_raw()) } - /// Returns `true` if the address is zero - #[inline(always)] - pub const fn is_zero(self) -> bool { - self.0 == 0 - } - - /// Returns `true` if the address is aligned to a boundary of a page at level `L` - #[inline] - pub const fn is_aligned_for(self) -> bool { - self.0 as usize % align_of::() == 0 - } - - /// Converts a previously virtualized physical address back into its physical form. - /// - /// # Safety - /// - /// The caller must ensure the function only receives addresses obtained through - /// [PhysicalAddress::virtualize_raw] or - /// [super::pointer::PhysicalRef]/[super::pointer::PhysicalRefMut] facilities. - pub unsafe fn from_virtualized(address: usize) -> Self { - Self(__physicalize(address)) - } - - /// Converts the physical address to a virtual one - pub fn virtualize_raw(self) -> usize { - unsafe { __virtualize(self.0) } - } -} - -impl Add for PhysicalAddress { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -impl Sub for PhysicalAddress { - type Output = usize; - - fn sub(self, rhs: Self) -> Self::Output { - (self.0 - rhs.0) as usize - } -} - -// Conversions - -impl const FromRaw for PhysicalAddress { - fn from_raw(value: u64) -> Self { - Self(value) - } -} - -impl const FromRaw for PhysicalAddress { - fn from_raw(value: usize) -> Self { - Self(value as u64) - } -} - -impl const IntoRaw for PhysicalAddress { - fn into_raw(self) -> u64 { - self.0 - } -} - -impl const IntoRaw for PhysicalAddress { - fn into_raw(self) -> usize { - self.0 as usize - } -} - -impl From for u64 { - fn from(addr: PhysicalAddress) -> u64 { - addr.0 - } -} - -impl From for usize { - fn from(addr: PhysicalAddress) -> usize { - addr.0 as usize - } -} - -// Ranges - -impl Step for PhysicalAddress { - fn steps_between(_start: &Self, _end: &Self) -> Option { - todo!() - } - - fn forward_checked(start: Self, count: usize) -> Option { - start.0.checked_add(count as u64).map(Self) - } - - fn backward_checked(_start: Self, _count: usize) -> Option { - todo!() - } -} - -// fmt - -impl fmt::LowerHex for PhysicalAddress { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::LowerHex::fmt(&self.0, f) + fn from_virtualized(value: usize) -> Self { + PhysicalAddress::from_raw(KernelTableManagerImpl::physicalize(value)) } } diff --git a/libk-mm/src/api.rs b/libk-mm/src/api.rs deleted file mode 100644 index 0aaf8a32..00000000 --- a/libk-mm/src/api.rs +++ /dev/null @@ -1,18 +0,0 @@ -use yggdrasil_abi::error::Error; - -use crate::{ - address::PhysicalAddress, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, -}; - -extern "Rust" { - pub fn __virtualize(phys: u64) -> usize; - pub fn __physicalize(virt: usize) -> u64; - - pub fn __map_device_pages( - base: PhysicalAddress, - count: usize, - attrs: DeviceMemoryAttributes, - ) -> Result; - pub fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping); -} diff --git a/libk-mm/src/device.rs b/libk-mm/src/device.rs index 45647cd9..8ead089b 100644 --- a/libk-mm/src/device.rs +++ b/libk-mm/src/device.rs @@ -2,40 +2,16 @@ use core::{ alloc::Layout, mem::size_of, ops::{Deref, DerefMut}, - ptr::NonNull, }; use alloc::sync::Arc; +use kernel_arch::KernelTableManagerImpl; +use libk_mm_interface::address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}; use yggdrasil_abi::error::Error; -use crate::api::{__map_device_pages, __unmap_device_pages}; +pub use kernel_arch::mem::{DeviceMemoryAttributes, DeviceMemoryCaching}; -use super::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; - -/// Describes a single device memory mapping -#[derive(Debug)] -pub struct RawDeviceMemoryMapping { - /// Virtual address of the mapped object - pub address: usize, - /// Base address of the mapping start - pub base_address: usize, - /// Page size used for the mapping - pub page_size: usize, - /// Number of pages used to map the object - pub page_count: usize, -} - -#[derive(Debug, Default, Clone, Copy)] -pub enum DeviceMemoryCaching { - #[default] - None, - Cacheable, -} - -#[derive(Default, Debug, Clone, Copy)] -pub struct DeviceMemoryAttributes { - pub caching: DeviceMemoryCaching, -} +pub type RawDeviceMemoryMapping = kernel_arch::mem::RawDeviceMemoryMapping; /// Describes a single untyped device memory mapping #[derive(Clone, Debug)] @@ -61,71 +37,6 @@ pub struct DeviceMemoryIoMut<'a, T: ?Sized> { value: &'a mut T, } -impl RawDeviceMemoryMapping { - /// Maps a region of physical memory as device memory of given size. - /// - /// # Safety - /// - /// The caller must ensure proper access synchronization, as well as the address' origin. - #[inline] - pub unsafe fn map( - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, - ) -> Result { - __map_device_pages(base, size, attrs) - } - - /// Consumes the device mapping, leaking its address without deallocating the translation - /// mapping itself - pub fn leak(self) -> usize { - let address = self.address; - core::mem::forget(self); - address - } - - pub fn into_raw_parts(self) -> (usize, usize, usize, usize) { - let address = self.address; - let base_address = self.base_address; - let page_count = self.page_count; - let page_size = self.page_size; - - core::mem::forget(self); - - (address, base_address, page_count, page_size) - } - - pub unsafe fn from_raw_parts( - address: usize, - base_address: usize, - page_count: usize, - page_size: usize, - ) -> Self { - Self { - address, - base_address, - page_count, - page_size, - } - } - - /// "Casts" the mapping to a specific type T and returns a [NonNull] pointer to it - pub unsafe fn as_non_null(&self) -> NonNull { - if self.page_size * self.page_count < size_of::() { - panic!(); - } - NonNull::new_unchecked(self.address as *mut T) - } -} - -impl Drop for RawDeviceMemoryMapping { - fn drop(&mut self) { - unsafe { - __unmap_device_pages(self); - } - } -} - impl DeviceMemoryMapping { /// Maps a region of physical memory as device memory of given size. /// @@ -139,7 +50,7 @@ impl DeviceMemoryMapping { size: usize, attrs: DeviceMemoryAttributes, ) -> Result { - let inner = RawDeviceMemoryMapping::map(base, size, attrs)?; + let inner = RawDeviceMemoryMapping::map(base.into_raw(), size, attrs)?; let address = inner.address; Ok(Self { inner: Arc::new(inner), @@ -183,7 +94,7 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { attrs: DeviceMemoryAttributes, ) -> Result, Error> { let layout = Layout::array::(count).unwrap(); - let inner = RawDeviceMemoryMapping::map(base, layout.size(), attrs)?; + let inner = RawDeviceMemoryMapping::map(base.into_raw(), layout.size(), attrs)?; let value = core::slice::from_raw_parts(inner.address as *mut T, count); Ok(DeviceMemoryIo { @@ -202,7 +113,7 @@ impl<'a, T: Sized> DeviceMemoryIo<'a, T> { base: PhysicalAddress, attrs: DeviceMemoryAttributes, ) -> Result, Error> { - let inner = RawDeviceMemoryMapping::map(base, size_of::(), attrs)?; + let inner = RawDeviceMemoryMapping::map(base.into_raw(), size_of::(), attrs)?; let value = &*(inner.address as *const T); Ok(DeviceMemoryIo { @@ -262,7 +173,7 @@ impl<'a, T: Sized> DeviceMemoryIoMut<'a, T> { attrs: DeviceMemoryAttributes, ) -> Result, Error> { let layout = Layout::array::(len).unwrap(); - let inner = RawDeviceMemoryMapping::map(base, layout.size(), attrs)?; + let inner = RawDeviceMemoryMapping::map(base.into_raw(), layout.size(), attrs)?; let value = core::slice::from_raw_parts_mut(inner.address as *mut T, len); Ok(DeviceMemoryIoMut { inner, value }) diff --git a/libk-mm/src/lib.rs b/libk-mm/src/lib.rs index 5c96f56b..bd67664e 100644 --- a/libk-mm/src/lib.rs +++ b/libk-mm/src/lib.rs @@ -17,17 +17,31 @@ use core::{ ops::{Deref, DerefMut}, }; +use address::Virtualize; +use libk_mm_interface::{ + address::{AsPhysicalAddress, PhysicalAddress}, + table::TableAllocator, +}; use yggdrasil_abi::error::Error; -use self::address::{AsPhysicalAddress, PhysicalAddress}; - -pub(crate) mod api; - pub mod address; pub mod device; pub mod phys; pub mod pointer; -pub mod table; + +pub use libk_mm_interface::table; + +pub struct TableAllocatorImpl; + +impl TableAllocator for TableAllocatorImpl { + fn allocate_page_table() -> Result { + phys::alloc_page() + } + + unsafe fn free_page_table(address: PhysicalAddress) { + phys::free_page(address) + } +} // TODO find a way to integrate this nicely with Architecture? pub const L3_PAGE_SIZE: usize = 1 << 12; @@ -60,7 +74,7 @@ impl PageBox { pub fn new(init: T) -> Result, Error> { let (base, page_count) = Self::alloc()?; - let value = base.virtualize_raw() as *mut T; + let value = base.virtualize() as *mut T; unsafe { value.write(init); @@ -76,7 +90,7 @@ impl PageBox { T: Copy, { let (base, page_count) = Self::alloc_slice(count)?; - let base_virt_ptr = base.virtualize_raw() as *mut T; + let base_virt_ptr = base.virtualize() as *mut T; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); for i in 0..count { @@ -102,7 +116,7 @@ impl PageBox { pub fn new_uninit() -> Result>, Error> { let (base, page_count) = PageBox::>::alloc()?; - let value = base.virtualize_raw() as *mut MaybeUninit; + let value = base.virtualize() as *mut MaybeUninit; let result = PageBox { value, page_count }; result.trace_created(); Ok(result) @@ -110,7 +124,7 @@ impl PageBox { pub fn new_uninit_slice(count: usize) -> Result]>, Error> { let (base, page_count) = PageBox::>::alloc_slice(count)?; - let base_virt_ptr = base.virtualize_raw() as *mut MaybeUninit; + let base_virt_ptr = base.virtualize() as *mut MaybeUninit; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); let result = PageBox { value, page_count }; result.trace_created(); @@ -205,7 +219,7 @@ impl PageBox<[MaybeUninit]> { impl AsPhysicalAddress for PageBox { #[inline] unsafe fn as_physical_address(&self) -> PhysicalAddress { - PhysicalAddress(api::__physicalize(self.value.addr())) + PhysicalAddress::from_virtualized(self.value.addr()) } } @@ -232,7 +246,7 @@ impl Drop for PageBox { core::ptr::drop_in_place(self.value); } // SAFETY: Safe, pointer obtained through "virtualize" - let base = PhysicalAddress(unsafe { api::__physicalize(self.value.addr()) }); + let base = PhysicalAddress::from_virtualized(self.value.addr()); for i in 0..self.page_count { // SAFETY: Safe, page allocated only by this PageBox unsafe { diff --git a/libk-mm/src/phys/manager.rs b/libk-mm/src/phys/manager.rs index b7842e14..44ca720e 100644 --- a/libk-mm/src/phys/manager.rs +++ b/libk-mm/src/phys/manager.rs @@ -1,13 +1,14 @@ //! Physical memory manager implementation use core::sync::atomic::{AtomicUsize, Ordering}; -use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; - -use crate::{ +use kernel_arch::KernelTableManagerImpl; +use libk_mm_interface::{ address::{FromRaw, IntoRaw, PhysicalAddress}, pointer::PhysicalRefMut, - L3_PAGE_SIZE, }; +use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; + +use crate::L3_PAGE_SIZE; pub type BitmapWord = u64; @@ -30,7 +31,7 @@ static STATS: MemoryStats = MemoryStats { /// Physical memory management interface pub struct PhysicalMemoryManager { - bitmap: PhysicalRefMut<'static, [u64]>, + bitmap: PhysicalRefMut<'static, [u64], KernelTableManagerImpl>, last_free_bit: usize, offset: usize, page_count: usize, @@ -43,7 +44,10 @@ impl PhysicalMemoryManager { page_count: usize, ) -> PhysicalMemoryManager { let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE; - let mut bitmap = PhysicalRefMut::<'static, u64>::map_slice(bitmap_phys_base, bitmap_len); + let mut bitmap = PhysicalRefMut::<'static, u64, KernelTableManagerImpl>::map_slice( + bitmap_phys_base, + bitmap_len, + ); bitmap.fill(BitmapWord::MAX); diff --git a/libk-mm/src/phys/mod.rs b/libk-mm/src/phys/mod.rs index 82449298..f531b346 100644 --- a/libk-mm/src/phys/mod.rs +++ b/libk-mm/src/phys/mod.rs @@ -1,11 +1,11 @@ use core::ops::Range; use kernel_arch::absolute_address; +use libk_mm_interface::address::{FromRaw, IntoRaw, PhysicalAddress}; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; use crate::{ - address::{FromRaw, IntoRaw}, phys::{ manager::BITMAP_WORD_SIZE, reserved::{is_reserved, reserve_region}, @@ -15,8 +15,6 @@ use crate::{ use self::manager::{PhysicalMemoryManager, TRACKED_PAGE_LIMIT}; -use super::address::PhysicalAddress; - mod manager; pub mod reserved; diff --git a/libk-mm/src/phys/reserved.rs b/libk-mm/src/phys/reserved.rs index 43d70757..bfe3b75b 100644 --- a/libk-mm/src/phys/reserved.rs +++ b/libk-mm/src/phys/reserved.rs @@ -1,8 +1,9 @@ //! Utilities for handling reserved memory regions +use libk_mm_interface::address::PhysicalAddress; use libk_util::StaticVector; -use crate::{address::PhysicalAddress, phys::PhysicalMemoryRegion}; +use crate::phys::PhysicalMemoryRegion; static mut RESERVED_MEMORY: StaticVector = StaticVector::new(); diff --git a/libk-mm/src/pointer.rs b/libk-mm/src/pointer.rs index db89f6df..1f681f33 100644 --- a/libk-mm/src/pointer.rs +++ b/libk-mm/src/pointer.rs @@ -1,141 +1,6 @@ -use core::{ - fmt, - ops::{Deref, DerefMut}, -}; +use kernel_arch::KernelTableManagerImpl; -use super::address::{AsPhysicalAddress, PhysicalAddress}; - -/// Wrapper for immutably accessing a value at a physical address -#[repr(transparent)] -pub struct PhysicalRef<'a, T: ?Sized> { - value: &'a T, -} - -/// Wrapper for mutably accessing a value at a physical address -#[repr(transparent)] -pub struct PhysicalRefMut<'a, T: ?Sized> { - value: &'a mut T, -} - -// PhysicalRefMut wrapper for safe mutable access to physical addresses - -impl<'a, T: Sized> PhysicalRefMut<'a, T> { - /// Maps a physical address into the kernel space as &mut T, allowing mmutable access to it. - /// - /// # Safety - /// - /// The caller must ensure the correct origin of the physical address as well that it actually - /// contains T. The caller must also take care of access synchronization and make sure no - /// aliasing occurs. - pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRefMut<'a, T> { - let value = virtualize_raw(physical); - PhysicalRefMut { value } - } - - /// Maps a physical address into the kernel space as &mut [T], allowing mmutable access to it. - /// - /// # Safety - /// - /// The caller must ensure the correct origin of the physical address as well that it actually - /// contains [T; len]. The caller must also take care of access synchronization and make - /// sure no aliasing occurs. - pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRefMut<'a, [T]> { - let value = virtualize_slice_raw(physical, len); - PhysicalRefMut { value } - } -} - -impl PhysicalRefMut<'_, T> { - /// Returns the "address" part of the reference - #[inline] - pub fn as_address(&self) -> usize { - (self.value as *const T).addr() - } -} - -impl AsPhysicalAddress for PhysicalRefMut<'_, T> { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - PhysicalAddress::from_virtualized(self.as_address()) - } -} - -impl Deref for PhysicalRefMut<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.value - } -} - -impl DerefMut for PhysicalRefMut<'_, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.value - } -} - -impl fmt::Pointer for PhysicalRefMut<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Pointer::fmt(&self.value, f) - } -} - -// PhysicalRef: same as PhysicalRefMut, except immutable - -impl<'a, T: Sized> PhysicalRef<'a, T> { - /// Maps a physical address into the kernel space as &T, allowing immutable access to it. - /// - /// # Safety - /// - /// The caller must ensure the correct origin of the physical address as well that it actually - /// contains T. - pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRef<'a, T> { - let value = virtualize_raw(physical); - PhysicalRef { value } - } - - /// Maps a physical address into the kernel space as &[T] of given len, allowing immutable - /// access to it. - /// - /// # Safety - /// - /// The caller must ensure the correct origin of the physical address as well that it actually - /// contains [T; len]. - pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRef<'a, [T]> { - let value = virtualize_slice_raw(physical, len); - PhysicalRef { value } - } -} - -impl PhysicalRef<'_, T> { - /// Returns the "address" part of the reference - #[inline] - pub fn as_address(&self) -> usize { - (self.value as *const T).addr() - } -} - -impl AsPhysicalAddress for PhysicalRef<'_, T> { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - PhysicalAddress::from_virtualized(self.as_address()) - } -} - -impl Deref for PhysicalRef<'_, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.value - } -} - -unsafe fn virtualize_raw<'a, T: Sized>(physical: PhysicalAddress) -> &'a mut T { - // TODO check align - let address = physical.virtualize_raw(); - &mut *(address as *mut T) -} - -unsafe fn virtualize_slice_raw<'a, T: Sized>(physical: PhysicalAddress, len: usize) -> &'a mut [T] { - // TODO check align - let address = physical.virtualize_raw(); - core::slice::from_raw_parts_mut(address as *mut T, len) -} +pub type PhysicalRef<'a, T> = + libk_mm_interface::pointer::PhysicalRef<'a, T, KernelTableManagerImpl>; +pub type PhysicalRefMut<'a, T> = + libk_mm_interface::pointer::PhysicalRefMut<'a, T, KernelTableManagerImpl>; diff --git a/libk-mm/src/table.rs b/libk-mm/src/table.rs deleted file mode 100644 index 2254d107..00000000 --- a/libk-mm/src/table.rs +++ /dev/null @@ -1,79 +0,0 @@ -use super::address::PhysicalAddress; - -/// Interface for a single level of address translation -pub trait EntryLevel: Copy { - /// The right shift needed to obtain an index of an entry at this level from an address - const SHIFT: usize; - /// The size of a page at this entry level - const SIZE: usize = 1 << Self::SHIFT; -} - -#[const_trait] -pub trait EntryLevelExt: Sized { - fn page_index(&self) -> usize; - fn page_offset(&self) -> usize; - fn page_count(&self) -> usize; - fn page_align_up(&self) -> Self; - fn page_align_down(&self) -> Self; - fn is_page_aligned_for(&self) -> bool; -} - -#[const_trait] -trait AddressLike: Sized + Copy { - fn into_usize(self) -> usize; - fn from_usize(v: usize) -> Self; -} - -impl const AddressLike for usize { - #[inline(always)] - fn into_usize(self) -> usize { - self - } - - #[inline(always)] - fn from_usize(v: usize) -> Self { - v - } -} - -impl const AddressLike for PhysicalAddress { - fn from_usize(v: usize) -> Self { - Self(v as _) - } - - fn into_usize(self) -> usize { - self.0 as _ - } -} - -impl const EntryLevelExt for T { - #[inline(always)] - fn page_index(&self) -> usize { - (self.into_usize() >> L::SHIFT) & 0x1FF - } - - #[inline(always)] - fn page_offset(&self) -> usize { - self.into_usize() & (L::SIZE - 1) - } - - #[inline(always)] - fn page_count(&self) -> usize { - (self.into_usize() + L::SIZE - 1) / L::SIZE - } - - #[inline(always)] - fn page_align_up(&self) -> Self { - Self::from_usize((self.into_usize() + L::SIZE - 1) & !(L::SIZE - 1)) - } - - #[inline(always)] - fn page_align_down(&self) -> Self { - Self::from_usize(self.into_usize() & !(L::SIZE - 1)) - } - - #[inline(always)] - fn is_page_aligned_for(&self) -> bool { - self.page_offset::() == 0 - } -} diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index 353d5017..a8a75853 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -10,18 +10,14 @@ use kernel_arch::{Architecture, ArchitectureImpl}; use kernel_fs::devfs; use libk::runtime; use libk_mm::{ - address::{IntoRaw, PhysicalAddress}, + address::{IntoRaw, PhysicalAddress, Virtualize}, phys, table::EntryLevel, }; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use super::{ - exception, mem::load_fixed_tables, smp::CPU_COUNT, BootStack, BOOT_STACK_SIZE, PLATFORM, -}; -use crate::{ - arch::aarch64::mem::table::L3, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, -}; +use super::{exception, smp::CPU_COUNT, BootStack, BOOT_STACK_SIZE, PLATFORM}; +use crate::{arch::L3, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET}; unsafe fn pre_init_mmu() { if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { @@ -82,7 +78,7 @@ unsafe extern "C" fn __aarch64_el1_bsp_lower_entry(dtb: PhysicalAddress) -> ! { // Setup MMU to jump to "higher-half" address space pre_init_mmu(); - load_fixed_tables(); + kernel_arch_aarch64::mem::load_fixed_tables(); enable_mmu(); // Safety: SP points to the .bss section, so it's +offset mapped @@ -125,11 +121,11 @@ unsafe extern "C" fn __aarch64_el1_ap_lower_entry() -> ! { CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); pre_init_mmu(); - load_fixed_tables(); + kernel_arch_aarch64::mem::load_fixed_tables(); enable_mmu(); let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES).unwrap(); - let stack_base = stack_pages.virtualize_raw(); + let stack_base = stack_pages.virtualize(); let sp = stack_base + L3::SIZE * AP_STACK_PAGES; let elr = absolute_address!(__aarch64_ap_upper_entry); diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index a2b456f7..4c61955d 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -3,7 +3,10 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; -use libk_mm::{address::PhysicalAddress, phys}; +use libk_mm::{ + address::{PhysicalAddress, Virtualize}, + phys, +}; use crate::task::{context::TaskContextImpl, thread::Thread}; @@ -85,7 +88,7 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 8; let stack_base_phys = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize_raw(); + let stack_base = stack_base_phys.virtualize(); let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -115,7 +118,7 @@ impl TaskContextImpl for TaskContext { ) -> Result { const USER_TASK_PAGES: usize = 16; let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize_raw(); + let stack_base = stack_base_phys.virtualize(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 46c0c0d1..c012fedd 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -15,7 +15,7 @@ use device_api::{ }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use libk_mm::{ - address::{FromRaw, PhysicalAddress}, + address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, RawDeviceMemoryMapping}, }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; @@ -51,7 +51,7 @@ impl Device for Gic { unsafe fn init(&'static self) -> Result<(), Error> { let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map( - self.gicd_base, + self.gicd_base.into_raw(), 0x1000, Default::default(), )?); diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs index b77693e6..4d21fdf0 100644 --- a/src/arch/aarch64/mem/mod.rs +++ b/src/arch/aarch64/mem/mod.rs @@ -1,378 +1,4 @@ //! AArch64-specific memory management interfaces and functions -use core::{ - alloc::Layout, - ops::{Deref, DerefMut}, - ptr::addr_of, -}; - -use abi::error::Error; -use cfg_if::cfg_if; -use libk_mm::{ - address::{FromRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - table::{EntryLevel, EntryLevelExt}, -}; -use libk_util::OneTimeInit; -use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; - -use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1}; -use static_assertions::const_assert_eq; -use tock_registers::interfaces::Writeable; - -use crate::mem::{address::KernelImageObject, KERNEL_VIRT_OFFSET}; - -use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}; pub mod process; pub(super) mod table; - -// TODO eliminate this requirement by using precomputed indices -const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET; -cfg_if! { - if #[cfg(feature = "aarch64_qemu")] { - const KERNEL_PHYS_BASE: usize = 0x40080000; - } else { - const KERNEL_PHYS_BASE: usize = 0x40080000; - } -} - -// Precomputed mappings -const KERNEL_L1_INDEX: usize = (KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE).page_index::(); -const KERNEL_START_L2_INDEX: usize = (KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE).page_index::(); -const KERNEL_END_L2_INDEX: usize = KERNEL_START_L2_INDEX + KERNEL_L3_COUNT; - -// Must not be zero, should be at 4MiB -const_assert_eq!(KERNEL_START_L2_INDEX, 0); -// From static mapping -const_assert_eq!(KERNEL_L1_INDEX, 1); - -// Runtime mappings -// 2MiB max -const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1; -// 1GiB max -const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; -// 1GiB max -const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; -const DEVICE_MAPPING_L3_COUNT: usize = 4; -// 16GiB max -const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3; -pub(super) const RAM_MAPPING_L1_COUNT: usize = 16; - -// 2MiB for early mappings -const EARLY_MAPPING_OFFSET: usize = - MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE); -static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); -// 1GiB for heap mapping -pub(super) const HEAP_MAPPING_OFFSET: usize = MAPPING_OFFSET | (HEAP_MAPPING_L1I * L1::SIZE); -pub(super) static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); -// 1GiB for device MMIO mapping -const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE); -static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); -static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = - [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; -// 16GiB for RAM mapping -pub(super) const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE); -pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); - -#[link_section = ".data.tables"] -pub(super) static mut KERNEL_TABLES: KernelImageObject = - unsafe { KernelImageObject::new(FixedTables::zeroed()) }; - -/// Memory mapping which may be used for performing early kernel initialization -pub struct EarlyMapping<'a, T: ?Sized> { - value: &'a mut T, - page_count: usize, -} - -impl<'a, T: Sized> EarlyMapping<'a, T> { - pub(super) unsafe fn map_slice( - physical: PhysicalAddress, - len: usize, - ) -> Result, Error> { - let layout = Layout::array::(len).unwrap(); - let aligned = physical.page_align_down::(); - let offset = physical.page_offset::(); - let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; - - let virt = map_early_pages(aligned, page_count)?; - let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); - - Ok(EarlyMapping { value, page_count }) - } -} - -impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.value - } -} - -impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.value - } -} - -impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { - fn drop(&mut self) { - let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); - - for i in 0..self.page_count { - let page = address + i * L3::SIZE; - - unsafe { - unmap_early_page(page); - } - } - } -} - -fn kernel_table_flags() -> PageAttributes { - PageAttributes::TABLE - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | PageAttributes::PRESENT -} - -fn ram_block_flags() -> PageAttributes { - // TODO UXN, PXN - PageAttributes::BLOCK - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | PageAttributes::PRESENT -} - -// Early mappings -unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { - for l3i in 0..512 { - let mut taken = false; - for i in 0..count { - if EARLY_MAPPING_L3[i + l3i].is_present() { - taken = true; - break; - } - } - - if taken { - continue; - } - - for i in 0..count { - let page = physical.add(i * L3::SIZE); - // TODO NX, NC - EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty()); - } - - return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn unmap_early_page(address: usize) { - if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { - panic!("Tried to unmap invalid early mapping: {:#x}", address); - } - - let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); - - assert!(EARLY_MAPPING_L3[l3i].is_present()); - EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; - - // TODO invalidate tlb -} - -pub(super) unsafe fn map_ram_l1(index: usize) { - if index >= RAM_MAPPING_L1_COUNT { - todo!() - } - assert_eq!(KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I], 0); - - KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I] = - ((index * L1::SIZE) as u64) | ram_block_flags().bits(); -} - -pub(super) unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) { - if index >= 512 { - todo!() - } - assert!(!HEAP_MAPPING_L2[index].is_present()); - // TODO UXN, PXN - HEAP_MAPPING_L2[index] = PageEntry::normal_block(page, PageAttributes::empty()); -} - -// Device mappings -unsafe fn map_device_memory_l3( - base: PhysicalAddress, - count: usize, - _attrs: DeviceMemoryAttributes, -) -> Result { - // TODO don't map pages if already mapped - - 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { - for j in 0..count { - let l2i = (i + j) / 512; - let l3i = (i + j) % 512; - - if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { - continue 'l0; - } - } - - for j in 0..count { - let l2i = (i + j) / 512; - let l3i = (i + j) % 512; - - // TODO NX, NC - DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE)); - } - - return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn map_device_memory_l2( - base: PhysicalAddress, - count: usize, - _attrs: DeviceMemoryAttributes, -) -> Result { - 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { - for j in 0..count { - if DEVICE_MAPPING_L2[i + j].is_present() { - continue 'l0; - } - } - - for j in 0..count { - DEVICE_MAPPING_L2[i + j] = PageEntry::::device_block(base.add(j * L2::SIZE)); - } - - debugln!( - "map l2s: base={:#x}, count={} -> {:#x}", - base, - count, - DEVICE_MAPPING_OFFSET + i * L2::SIZE - ); - return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); - } - - Err(Error::OutOfMemory) -} - -pub(super) unsafe fn map_device_memory( - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, -) -> Result { - // debugln!("Map {}B @ {:#x}", size, base); - let l3_aligned = base.page_align_down::(); - let l3_offset = base.page_offset::(); - let page_count = (l3_offset + size).page_count::(); - - if page_count > 256 { - // Large mapping, use L2 mapping instead - let l2_aligned = base.page_align_down::(); - let l2_offset = base.page_offset::(); - let page_count = (l2_offset + size).page_count::(); - - let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; - let address = base_address + l2_offset; - - Ok(RawDeviceMemoryMapping { - address, - base_address, - page_count, - page_size: L2::SIZE, - }) - } else { - // Just map the pages directly - let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; - let address = base_address + l3_offset; - - Ok(RawDeviceMemoryMapping { - address, - base_address, - page_count, - page_size: L3::SIZE, - }) - } -} - -pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { - // debugln!( - // "Unmap {}B @ {:#x}", - // map.page_count * map.page_size, - // map.base_address - // ); - match map.page_size { - L3::SIZE => { - for i in 0..map.page_count { - let page = map.base_address + i * L3::SIZE; - let l2i = page.page_index::(); - let l3i = page.page_index::(); - assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); - DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; - - tlb_flush_vaae1(page); - } - } - L2::SIZE => todo!(), - _ => unimplemented!(), - } -} - -#[inline] -fn tlb_flush_vaae1(mut page: usize) { - page >>= 12; - unsafe { - core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); - } -} - -/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half" -/// -/// # Safety -/// -/// Unsafe, must only be called by BSP during its early init while still in "lower-half" -pub(super) unsafe fn load_fixed_tables() { - let ttbr0 = KERNEL_TABLES.l1.data.as_ptr() as u64; - TTBR0_EL1.set(ttbr0); - TTBR1_EL1.set(ttbr0); -} - -/// Sets up additional translation tables for kernel usage -/// -/// # Safety -/// -/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half" -pub(super) unsafe fn init_fixed_tables() { - // TODO this could be built in compile-time too? - let early_mapping_l3_phys = addr_of!(EARLY_MAPPING_L3) as usize - KERNEL_VIRT_OFFSET; - let device_mapping_l2_phys = addr_of!(DEVICE_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; - let heap_mapping_l2_phys = addr_of!(HEAP_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; - - for i in 0..DEVICE_MAPPING_L3_COUNT { - let device_mapping_l3_phys = PhysicalAddress::from_raw( - &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, - ); - DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty()); - } - - assert_eq!(KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I], 0); - KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I] = - (early_mapping_l3_phys as u64) | kernel_table_flags().bits(); - - assert_eq!(KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I], 0); - KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I] = - (heap_mapping_l2_phys as u64) | kernel_table_flags().bits(); - - assert_eq!(KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I], 0); - KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I] = - (device_mapping_l2_phys as u64) | kernel_table_flags().bits(); -} diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs index 8f204195..14fc0a13 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/src/arch/aarch64/mem/process.rs @@ -2,22 +2,19 @@ use core::sync::atomic::{AtomicU8, Ordering}; use abi::error::Error; +use kernel_arch_aarch64::mem::{ + table::{PageEntry, PageTable, L1, L2, L3}, + tlb_flush_vaae1, +}; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, phys, pointer::PhysicalRefMut, - table::{EntryLevel, EntryLevelExt}, + table::{EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable}, + TableAllocatorImpl, }; -use crate::mem::{ - process::ProcessAddressSpaceManager, - table::{EntryLevelDrop, MapAttributes, NextPageTable}, -}; - -use super::{ - table::{PageEntry, PageTable, L1, L2, L3}, - tlb_flush_vaae1, -}; +use crate::mem::process::ProcessAddressSpaceManager; /// AArch64 implementation of a process address space table #[repr(C)] @@ -72,8 +69,9 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { } unsafe fn clear(&mut self) { - self.l1 - .drop_range(0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::())); + self.l1.drop_range::( + 0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::()), + ); } } @@ -89,8 +87,8 @@ impl ProcessAddressSpaceImpl { let l2i = virt.page_index::(); let l3i = virt.page_index::(); - let mut l2 = self.l1.get_mut_or_alloc(l1i)?; - let mut l3 = l2.get_mut_or_alloc(l2i)?; + let mut l2 = self.l1.get_mut_or_alloc::(l1i)?; + let mut l3 = l2.get_mut_or_alloc::(l2i)?; if l3[l3i].is_present() && !overwrite { todo!(); diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs index d46e0f5e..e69de29b 100644 --- a/src/arch/aarch64/mem/table.rs +++ b/src/arch/aarch64/mem/table.rs @@ -1,335 +0,0 @@ -use core::{ - marker::PhantomData, - ops::{Index, IndexMut, Range}, -}; - -use abi::error::Error; -use bitflags::bitflags; -use libk_mm::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - table::EntryLevel, -}; - -use crate::mem::table::{EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel}; - -bitflags! { - #[derive(Clone, Copy, PartialEq, Eq)] - pub struct PageAttributes: u64 { - const PRESENT = 1 << 0; - - const TABLE = 1 << 1; - const PAGE = 1 << 1; - const BLOCK = 0 << 1; - - const ACCESS = 1 << 10; - - const AP_KERNEL_READWRITE = 0 << 6; - const AP_BOTH_READWRITE = 1 << 6; - const AP_KERNEL_READONLY = 2 << 6; - const AP_BOTH_READONLY = 3 << 6; - const AP_ACCESS_MASK = 3 << 6; - - const SH_OUTER = 2 << 8; - const SH_INNER = 3 << 8; - - const PAGE_ATTR_NORMAL = 0 << 2; - const PAGE_ATTR_DEVICE = 1 << 2; - - const NON_GLOBAL = 1 << 11; - - const PXN = 1 << 53; - const UXN = 1 << 54; - } -} - -#[derive(Clone, Copy)] -#[repr(C, align(0x1000))] -pub struct PageTable { - entries: [PageEntry; 512], -} - -#[derive(Clone, Copy)] -pub struct PageEntry(u64, PhantomData); - -#[derive(Clone, Copy)] -pub struct L1; -#[derive(Clone, Copy)] -pub struct L2; -#[derive(Clone, Copy)] -pub struct L3; - -impl NonTerminalEntryLevel for L1 { - type NextLevel = L2; -} - -impl NonTerminalEntryLevel for L2 { - type NextLevel = L3; -} - -impl EntryLevel for L1 { - const SHIFT: usize = 30; -} - -impl EntryLevel for L2 { - const SHIFT: usize = 21; -} - -impl EntryLevel for L3 { - const SHIFT: usize = 12; -} - -impl PageTable { - pub const fn zeroed() -> Self { - Self { - entries: [PageEntry::INVALID; 512], - } - } - - pub fn new_zeroed<'a>() -> Result, Error> { - let physical = phys::alloc_page()?; - let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) }; - - for i in 0..512 { - table[i] = PageEntry::INVALID; - } - - Ok(table) - } -} - -impl PageEntry { - pub const INVALID: Self = Self(0, PhantomData); - - pub const fn is_present(self) -> bool { - self.0 & PageAttributes::PRESENT.bits() != 0 - } - - pub fn attributes(self) -> PageAttributes { - PageAttributes::from_bits_retain(self.0) - } -} - -impl NextPageTable for PageTable { - type NextLevel = PageTable; - type TableRef = PhysicalRef<'static, PageTable>; - type TableRefMut = PhysicalRefMut<'static, PageTable>; - - fn get(&self, index: usize) -> Option { - self[index] - .as_table() - .map(|phys| unsafe { PhysicalRef::map(phys) }) - } - - fn get_mut(&mut self, index: usize) -> Option { - self[index] - .as_table() - .map(|phys| unsafe { PhysicalRefMut::map(phys) }) - } - - fn get_mut_or_alloc(&mut self, index: usize) -> Result { - let entry = self[index]; - - if let Some(table) = entry.as_table() { - Ok(unsafe { PhysicalRefMut::map(table) }) - } else { - let table = PageTable::new_zeroed()?; - self[index] = PageEntry::::table( - unsafe { table.as_physical_address() }, - PageAttributes::empty(), - ); - Ok(table) - } - } -} - -impl EntryLevelDrop for PageTable { - const FULL_RANGE: Range = 0..512; - - // Do nothing - unsafe fn drop_range(&mut self, _range: Range) {} -} - -impl EntryLevelDrop for PageTable -where - PageTable: EntryLevelDrop, -{ - const FULL_RANGE: Range = 0..512; - - unsafe fn drop_range(&mut self, range: Range) { - for index in range { - let entry = self[index]; - - if let Some(table) = entry.as_table() { - let mut table_ref: PhysicalRefMut> = - PhysicalRefMut::map(table); - - table_ref.drop_all(); - - // Drop the table - drop(table_ref); - - phys::free_page(table); - } else if entry.is_present() { - // Memory must've been cleared beforehand, so no non-table entries must be present - panic!( - "Expected a table containing only tables, got table[{}] = {:#x?}", - index, entry.0 - ); - } - - self[index] = PageEntry::INVALID; - } - } -} - -impl PageEntry { - pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self { - Self( - IntoRaw::::into_raw(phys) - | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), - PhantomData, - ) - } - - pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { - Self( - IntoRaw::::into_raw(phys) - | (PageAttributes::BLOCK - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | attrs) - .bits(), - PhantomData, - ) - } - - pub fn device_block(phys: PhysicalAddress) -> Self { - Self( - IntoRaw::::into_raw(phys) - | (PageAttributes::BLOCK - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_OUTER - | PageAttributes::PAGE_ATTR_DEVICE - | PageAttributes::UXN - | PageAttributes::PXN) - .bits(), - PhantomData, - ) - } - - /// Returns the physical address of the table this entry refers to, returning None if it - /// does not - pub fn as_table(self) -> Option { - if self.0 & PageAttributes::PRESENT.bits() != 0 - && self.0 & PageAttributes::BLOCK.bits() == 0 - { - Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) - } else { - None - } - } -} - -impl PageEntry { - pub fn normal_page(phys: PhysicalAddress, attrs: PageAttributes) -> Self { - Self( - IntoRaw::::into_raw(phys) - | (PageAttributes::PAGE - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | attrs) - .bits(), - PhantomData, - ) - } - - pub fn device_page(phys: PhysicalAddress) -> Self { - Self( - IntoRaw::::into_raw(phys) - | (PageAttributes::PAGE - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_OUTER - | PageAttributes::PAGE_ATTR_DEVICE - | PageAttributes::UXN - | PageAttributes::PXN) - .bits(), - PhantomData, - ) - } - - pub fn as_page(&self) -> Option { - let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); - if self.0 & mask == mask { - Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) - } else { - None - } - } -} - -impl Index for PageTable { - type Output = PageEntry; - - fn index(&self, index: usize) -> &Self::Output { - &self.entries[index] - } -} - -impl IndexMut for PageTable { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.entries[index] - } -} - -impl From for PageAttributes { - fn from(value: MapAttributes) -> Self { - let mut out = PageAttributes::empty(); - // TODO kernel cannot write el0 readonly pages - if value.contains(MapAttributes::USER_WRITE) { - // Read/write - out |= PageAttributes::AP_BOTH_READWRITE; - } else if value.contains(MapAttributes::USER_READ) { - // Read only - out |= PageAttributes::AP_BOTH_READONLY; - } else { - // No read/write - out |= PageAttributes::AP_KERNEL_READONLY; - } - - if value.contains(MapAttributes::NON_GLOBAL) { - out |= PageAttributes::NON_GLOBAL; - } - - out - } -} - -impl From for MapAttributes { - fn from(value: PageAttributes) -> Self { - let mut out = MapAttributes::empty(); - - out |= match value.intersection(PageAttributes::AP_ACCESS_MASK) { - PageAttributes::AP_BOTH_READWRITE => { - MapAttributes::USER_WRITE | MapAttributes::USER_READ - } - PageAttributes::AP_BOTH_READONLY => MapAttributes::USER_READ, - PageAttributes::AP_KERNEL_READONLY => MapAttributes::empty(), - PageAttributes::AP_KERNEL_READWRITE => panic!("This variant cannot be constructed"), - _ => unreachable!(), - }; - - if value.contains(PageAttributes::NON_GLOBAL) { - out |= MapAttributes::NON_GLOBAL; - } - - out - } -} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 14a98c50..b33901d4 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -16,6 +16,10 @@ use device_api::{ }; use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; use git_version::git_version; +use kernel_arch_aarch64::mem::{ + table::{L1, L2, L3}, + EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, RAM_MAPPING_OFFSET, +}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, @@ -29,18 +33,13 @@ use tock_registers::interfaces::Writeable; use ygg_driver_pci::PciBusManager; use crate::{ - arch::aarch64::{ - cpu::Cpu, - mem::table::{L2, L3}, - }, + arch::aarch64::cpu::Cpu, debug, device::{self, power::arm_psci::Psci}, fs::{Initrd, INITRD_DATA}, mem::heap, }; -use self::mem::{table::L1, EarlyMapping}; - use super::{CpuMessage, Platform}; pub mod boot; @@ -94,38 +93,6 @@ impl Platform for AArch64 { smp::CPU_COUNT.load(Ordering::Acquire) } - unsafe fn map_device_memory( - &self, - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, - ) -> Result { - mem::map_device_memory(base, size, attrs) - } - - unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) { - mem::unmap_device_memory(map) - } - - fn virtualize(address: u64) -> usize { - let address = address as usize; - if address < *mem::MEMORY_LIMIT.get() { - address + mem::RAM_MAPPING_OFFSET - } else { - panic!("Invalid physical address: {:#x}", address); - } - } - - fn physicalize(address: usize) -> u64 { - if address < mem::RAM_MAPPING_OFFSET - || address - mem::RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() - { - panic!("Not a virtualized physical address: {:#x}", address); - } - - (address - mem::RAM_MAPPING_OFFSET) as _ - } - fn local_interrupt_controller( &'static self, ) -> &'static dyn LocalInterruptController { @@ -227,18 +194,18 @@ impl AArch64 { memory_end: PhysicalAddress, ) -> Result<(), Error> { let end_l1i = memory_end.page_align_up::().page_index::(); - if end_l1i > mem::RAM_MAPPING_L1_COUNT { + if end_l1i > RAM_MAPPING_L1_COUNT { todo!() } // Map 1GiB chunks for index in 0..end_l1i { unsafe { - mem::map_ram_l1(index); + kernel_arch_aarch64::mem::map_ram_l1(index); } } - mem::MEMORY_LIMIT.init(memory_end.into_raw()); + MEMORY_LIMIT.store(memory_end.into_raw(), Ordering::Release); Ok(()) } @@ -248,7 +215,7 @@ impl AArch64 { const HEAP_PAGES: usize = 16; // Initialize the runtime mappings - mem::init_fixed_tables(); + kernel_arch_aarch64::mem::init_fixed_tables(); // Extract the size of the device tree let dtb_size = { @@ -293,10 +260,10 @@ impl AArch64 { // Setup the heap for i in 0..HEAP_PAGES { let l2_page = phys::alloc_2m_page()?; - mem::map_heap_l2(i, l2_page); + kernel_arch_aarch64::mem::map_heap_l2(i, l2_page); } - heap::init_heap(mem::HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE); + heap::init_heap(HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE); // EarlyMapping for DTB no longer needed, it lives in physical memory and can be obtained // through PhysicalRef diff --git a/src/arch/mod.rs b/src/arch/mod.rs index dafeec45..13f3f33d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -72,31 +72,6 @@ pub trait Platform { /// Only safe to call once during system init. unsafe fn start_application_processors(&self) {} - /// Allocates a virtual mapping for the specified physical memory region. - /// - /// # Safety - /// - /// The caller must ensure the validity of the provided region. - unsafe fn map_device_memory( - &self, - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, - ) -> Result; - - /// Removes the provided mapping from the kernel's translation tables. - /// - /// # Safety - /// - /// The caller must ensure the mapping is and will no longer be used. - unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping); - - /// Converts a physical address to a virtual one, so it can be accessed by the kernel - fn virtualize(address: u64) -> usize; - - /// Converts a virtual address created by [Architecture::virtualize] back to its physical form - fn physicalize(address: usize) -> u64; - // Architecture intrinsics /// Returns the count of present CPUs, including the BSP @@ -270,29 +245,6 @@ fn __cpu_count() -> usize { PlatformImpl::cpu_count() } -#[no_mangle] -fn __virtualize(addr: u64) -> usize { - PlatformImpl::virtualize(addr) -} - -#[no_mangle] -fn __physicalize(addr: usize) -> u64 { - PlatformImpl::physicalize(addr) -} - -#[no_mangle] -fn __map_device_pages( - base: PhysicalAddress, - count: usize, - attrs: DeviceMemoryAttributes, -) -> Result { - unsafe { PLATFORM.map_device_memory(base, count, attrs) } -} -#[no_mangle] -fn __unmap_device_pages(mapping: &RawDeviceMemoryMapping) { - unsafe { PLATFORM.unmap_device_memory(mapping) } -} - #[no_mangle] fn __monotonic_timestamp() -> Result { PLATFORM.monotonic_timer().monotonic_timestamp() diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 323f7d70..0eee7955 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -16,7 +16,7 @@ use device_api::{ Device, }; use libk_mm::{ - address::{FromRaw, PhysicalAddress}, + address::{FromRaw, PhysicalAddress, Virtualize}, pointer::PhysicalRef, }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; @@ -324,7 +324,7 @@ impl AcpiHandler for AcpiHandlerImpl { PhysicalMapping::new( physical_address, NonNull::new_unchecked( - PhysicalAddress::from_raw(physical_address).virtualize_raw() as *mut T + PhysicalAddress::from_raw(physical_address).virtualize() as *mut T ), size, size, diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index a987ef2e..9c2d0873 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -10,6 +10,7 @@ use device_api::{ }, Device, }; +use kernel_arch_x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, @@ -27,9 +28,7 @@ use tock_registers::{ use crate::{ arch::{ - x86_64::{ - apic::APIC_MSI_OFFSET, mem::table::L3, registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT, - }, + x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}, CpuAccess, CpuMessage, }, task::Cpu, diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index 8d6c03e8..f1358a21 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,6 +1,7 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; +use kernel_arch_x86_64::registers::MSR_IA32_KERNEL_GS_BASE; use kernel_fs::devfs; use libk::runtime; use tock_registers::interfaces::Writeable; @@ -10,9 +11,7 @@ use yboot_proto::{ }; use crate::{ - arch::x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, smp::CPU_COUNT}, - kernel_main, kernel_secondary_main, - mem::KERNEL_VIRT_OFFSET, + arch::x86_64::smp::CPU_COUNT, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, }; use super::{cpuid::init_cpuid, exception, PLATFORM}; diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index 1272e0f7..d63e2f75 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -3,17 +3,15 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use alloc::sync::Arc; +use kernel_arch_x86_64::{mem::KERNEL_TABLES, registers::FpuContext}; use libk_mm::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress, Virtualize}, phys, }; -use crate::{ - arch::x86_64::mem::KERNEL_TABLES, - task::{context::TaskContextImpl, thread::Thread}, -}; +use crate::task::{context::TaskContextImpl, thread::Thread}; -use super::{registers::FpuContext, syscall::SyscallFrame}; +use super::syscall::SyscallFrame; struct StackBuilder { base: usize, @@ -93,7 +91,7 @@ impl TaskContext { const USER_TASK_PAGES: usize = 8; let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize_raw(); + let stack_base = stack_base_phys.virtualize(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); @@ -144,7 +142,7 @@ impl TaskContextImpl for TaskContext { const KERNEL_TASK_PAGES: usize = 32; let stack_base_phys = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize_raw(); + let stack_base = stack_base_phys.virtualize(); let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -180,7 +178,7 @@ impl TaskContextImpl for TaskContext { const USER_TASK_PAGES: usize = 8; let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize_raw(); + let stack_base = stack_base_phys.virtualize(); let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 0ce96b0b..be67d842 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -6,6 +6,7 @@ use core::{ }; use alloc::{boxed::Box, vec::Vec}; +use kernel_arch_x86_64::registers::MSR_IA32_KERNEL_GS_BASE; use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, OneTimeInit, @@ -14,7 +15,7 @@ use tock_registers::interfaces::Writeable; use crate::{ arch::{ - x86_64::{cpuid, gdt, registers::MSR_IA32_KERNEL_GS_BASE, syscall}, + x86_64::{cpuid, gdt, syscall}, CpuAccess, CpuMessage, LocalCpuAccess, }, task::{sched::CpuQueue, thread::ThreadId}, diff --git a/src/arch/x86_64/cpuid.rs b/src/arch/x86_64/cpuid.rs index d59532e2..a4afc17e 100644 --- a/src/arch/x86_64/cpuid.rs +++ b/src/arch/x86_64/cpuid.rs @@ -1,11 +1,10 @@ //! x86-64 CPUID interface use bitflags::bitflags; +use kernel_arch_x86_64::registers::{CR4, XCR0}; use libk_util::OneTimeInit; use tock_registers::interfaces::ReadWriteable; -use super::registers::{CR4, XCR0}; - bitflags! { pub struct ProcessorFeatures: u64 { const PDPE1GB = 1 << 0; diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 93ad15c4..838604a7 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -2,13 +2,11 @@ use core::{arch::global_asm, mem::size_of, ptr::addr_of}; use abi::{arch::SavedFrame, primitive_enum, process::Signal}; +use kernel_arch_x86_64::registers::CR3; use tock_registers::interfaces::Readable; use crate::{ - arch::{ - x86_64::{apic, registers::CR3}, - CpuAccess, - }, + arch::{x86_64::apic, CpuAccess}, debug, task::{context::TaskFrame, thread::Thread, Cpu}, }; diff --git a/src/arch/x86_64/intrinsics.rs b/src/arch/x86_64/intrinsics.rs index 8fe7991b..b1d9274c 100644 --- a/src/arch/x86_64/intrinsics.rs +++ b/src/arch/x86_64/intrinsics.rs @@ -130,11 +130,6 @@ pub unsafe fn outl(port: u16, value: u32) { core::arch::asm!("outl %eax, %dx", in("dx") port, in("eax") value, options(att_syntax)); } -#[inline] -pub unsafe fn flush_tlb_entry(address: usize) { - core::arch::asm!("invlpg ({0})", in(reg) address, options(att_syntax)); -} - #[inline] pub fn flush_cpu_cache() { unsafe { diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs index a496e97b..a04d6535 100644 --- a/src/arch/x86_64/mem/mod.rs +++ b/src/arch/x86_64/mem/mod.rs @@ -1,365 +1,3 @@ //! x86-64-specific memory management functions and interfaces -use core::{ - alloc::Layout, - ops::{Deref, DerefMut}, - ptr::addr_of, -}; - -use abi::error::Error; -use libk_mm::{ - address::{FromRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - table::EntryLevelExt, -}; -use libk_util::OneTimeInit; -use memtables::FixedTables; -use static_assertions::{const_assert_eq, const_assert_ne}; pub mod process; -pub mod table; - -use crate::{ - arch::x86_64::{intrinsics, mem::table::PageAttributes, registers::CR3}, - mem::{address::KernelImageObject, table::EntryLevel, KERNEL_VIRT_OFFSET}, -}; - -use self::table::{PageEntry, PageTable, L0, L1, L2, L3}; - -const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000; -const KERNEL_PHYS_BASE: usize = 0x200000; - -// Mapped at compile time -const KERNEL_MAPPING_BASE: usize = KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE; -const KERNEL_L0_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); -const KERNEL_L1_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); -const KERNEL_START_L2_INDEX: usize = KERNEL_MAPPING_BASE.page_index::(); - -// Must not be zero, should be at 4MiB -const_assert_ne!(KERNEL_START_L2_INDEX, 0); -// From static mapping -const_assert_eq!(KERNEL_L0_INDEX, 511); -const_assert_eq!(KERNEL_L1_INDEX, 0); - -// Mapped at boot -const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1; -const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; -const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; -const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1; - -const DEVICE_MAPPING_L3_COUNT: usize = 4; - -#[link_section = ".data.tables"] -pub(super) static mut KERNEL_TABLES: KernelImageObject = - unsafe { KernelImageObject::new(FixedTables::zeroed()) }; - -// 2MiB for early mappings -const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK - | (KERNEL_L0_INDEX * L0::SIZE) - | (KERNEL_L1_INDEX * L1::SIZE) - | (EARLY_MAPPING_L2I * L2::SIZE); -static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); -// 1GiB for heap mapping -pub(super) const HEAP_MAPPING_OFFSET: usize = - CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (HEAP_MAPPING_L1I * L1::SIZE); -pub(super) static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); -// 1GiB for device MMIO mapping -const DEVICE_MAPPING_OFFSET: usize = - CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE); -static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); -static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = - [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; -// 512GiB for whole RAM mapping -pub(super) const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE); -pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); -pub(super) static mut RAM_MAPPING_L1: PageTable = PageTable::zeroed(); - -// Early mappings -unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { - for l3i in 0..512 { - let mut taken = false; - for i in 0..count { - if EARLY_MAPPING_L3[i + l3i].is_present() { - taken = true; - break; - } - } - - if taken { - continue; - } - - for i in 0..count { - // TODO NX, NC - EARLY_MAPPING_L3[i + l3i] = - PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE); - } - - return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn unmap_early_page(address: usize) { - if !(EARLY_MAPPING_OFFSET..EARLY_MAPPING_OFFSET + L2::SIZE).contains(&address) { - panic!("Tried to unmap invalid early mapping: {:#x}", address); - } - - let l3i = (address - EARLY_MAPPING_OFFSET).page_index::(); - - assert!(EARLY_MAPPING_L3[l3i].is_present()); - EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; -} - -// Device mappings -unsafe fn map_device_memory_l3( - base: PhysicalAddress, - count: usize, - _attrs: DeviceMemoryAttributes, -) -> Result { - // TODO don't map pages if already mapped - - 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { - for j in 0..count { - let l2i = (i + j) / 512; - let l3i = (i + j) % 512; - - if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { - continue 'l0; - } - } - - for j in 0..count { - let l2i = (i + j) / 512; - let l3i = (i + j) % 512; - - // TODO NX, NC - DEVICE_MAPPING_L3S[l2i][l3i] = - PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE); - } - - return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); - } - - Err(Error::OutOfMemory) -} - -unsafe fn map_device_memory_l2( - base: PhysicalAddress, - count: usize, - _attrs: DeviceMemoryAttributes, -) -> Result { - 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { - for j in 0..count { - if DEVICE_MAPPING_L2[i + j].is_present() { - continue 'l0; - } - } - - for j in 0..count { - DEVICE_MAPPING_L2[i + j] = - PageEntry::::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE); - } - - debugln!( - "map l2s: base={:#x}, count={} -> {:#x}", - base, - count, - DEVICE_MAPPING_OFFSET + i * L2::SIZE - ); - return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); - } - - Err(Error::OutOfMemory) -} - -pub(super) unsafe fn map_device_memory( - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, -) -> Result { - // debugln!("Map {}B @ {:#x}", size, base); - let l3_aligned = base.page_align_down::(); - let l3_offset = base.page_offset::(); - let page_count = (l3_offset + size).page_count::(); - - if page_count > 256 { - // Large mapping, use L2 mapping instead - let l2_aligned = base.page_align_down::(); - let l2_offset = base.page_offset::(); - let page_count = (l2_offset + size).page_count::(); - - let base_address = map_device_memory_l2(l2_aligned, page_count, attrs)?; - let address = base_address + l2_offset; - - Ok(RawDeviceMemoryMapping { - address, - base_address, - page_count, - page_size: L2::SIZE, - }) - } else { - // Just map the pages directly - let base_address = map_device_memory_l3(l3_aligned, page_count, attrs)?; - let address = base_address + l3_offset; - - Ok(RawDeviceMemoryMapping { - address, - base_address, - page_count, - page_size: L3::SIZE, - }) - } -} - -pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { - // debugln!( - // "Unmap {}B @ {:#x}", - // map.page_count * map.page_size, - // map.base_address - // ); - match map.page_size { - L3::SIZE => { - for i in 0..map.page_count { - let page = map.base_address + i * L3::SIZE; - let l2i = page.page_index::(); - let l3i = page.page_index::(); - assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); - DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; - intrinsics::flush_tlb_entry(page); - } - } - L2::SIZE => todo!(), - _ => unimplemented!(), - } -} - -pub(super) unsafe fn map_heap_block(index: usize, page: PhysicalAddress) { - if !page.is_page_aligned_for::() { - panic!("Attempted to map a misaligned 2MiB page"); - } - assert!(index < 512); - - if HEAP_MAPPING_L2[index].is_present() { - panic!("Page is already mappged: {:#x}", page); - } - - // TODO NX - HEAP_MAPPING_L2[index] = PageEntry::::block(page, PageAttributes::WRITABLE); -} - -/// Memory mapping which may be used for performing early kernel initialization -pub struct EarlyMapping<'a, T: ?Sized> { - value: &'a mut T, - page_count: usize, -} - -impl<'a, T: Sized> EarlyMapping<'a, T> { - pub(super) unsafe fn map(physical: PhysicalAddress) -> Result, Error> { - let layout = Layout::new::(); - let aligned = physical.page_align_down::(); - let offset = physical.page_offset::(); - let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; - - let virt = map_early_pages(aligned, page_count)?; - let value = &mut *((virt + offset) as *mut T); - - Ok(EarlyMapping { value, page_count }) - } - - pub(super) unsafe fn map_slice( - physical: PhysicalAddress, - len: usize, - ) -> Result, Error> { - let layout = Layout::array::(len).unwrap(); - let aligned = physical.page_align_down::(); - let offset = physical.page_offset::(); - let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; - - let virt = map_early_pages(aligned, page_count)?; - let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); - - Ok(EarlyMapping { value, page_count }) - } -} - -impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.value - } -} - -impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.value - } -} - -impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { - fn drop(&mut self) { - let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); - - for i in 0..self.page_count { - let page = address + i * L3::SIZE; - - unsafe { - unmap_early_page(page); - } - } - } -} - -fn clone_kernel_tables(dst: &mut PageTable) { - unsafe { - dst[KERNEL_L0_INDEX] = PageEntry::from_raw(KERNEL_TABLES.l0.data[KERNEL_L0_INDEX]); - dst[RAM_MAPPING_L0I] = PageEntry::from_raw(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I]); - } -} - -/// Sets up the following memory map: -/// ...: KERNEL_TABLES.l0: -/// * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000 : RAM_MAPPING_L1 -/// * 0xFFFFFF8000000000 .. ... : KERNEL_TABLES.kernel_l1: -/// * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000 : KERNEL_TABLES.kernel_l2 -/// * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000 : --- -/// * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000 : EARLY_MAPPING_L3 -/// * 0xFFFFFF8000400000 .. ... : KERNEL_TABLES.kernel_l3s -/// * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000 : HEAP_MAPPING_L2 -/// * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000 : DEVICE_MAPPING_L2 -/// * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000 : DEVICE_MAPPING_L3S -/// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ... -pub(super) unsafe fn init_fixed_tables() { - // TODO this could be built in compile-time too? - let early_mapping_l3_phys = addr_of!(EARLY_MAPPING_L3) as usize - KERNEL_VIRT_OFFSET; - let device_mapping_l2_phys = addr_of!(DEVICE_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; - let heap_mapping_l2_phys = addr_of!(HEAP_MAPPING_L2) as usize - KERNEL_VIRT_OFFSET; - let ram_mapping_l1_phys = addr_of!(RAM_MAPPING_L1) as usize - KERNEL_VIRT_OFFSET; - - for i in 0..DEVICE_MAPPING_L3_COUNT { - let device_mapping_l3_phys = PhysicalAddress::from_raw( - &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, - ); - DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE); - } - - assert_eq!(KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I], 0); - KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64) - | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - - assert_eq!(KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I], 0); - KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I] = - (heap_mapping_l2_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - assert_eq!(KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I], 0); - KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64) - | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - - assert_eq!(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I], 0); - KERNEL_TABLES.l0.data[RAM_MAPPING_L0I] = - (ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits(); - - // TODO ENABLE EFER.NXE - let cr3 = &KERNEL_TABLES.l0 as *const _ as usize - KERNEL_VIRT_OFFSET; - CR3.set_address(cr3); -} diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index 52795c15..4454717b 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -1,24 +1,18 @@ //! x86-64-specific process address space management functions +use kernel_arch_x86_64::mem::{ + clone_kernel_tables, flush_tlb_entry, + table::{PageEntry, PageTable, L0, L1, L2, L3}, +}; use libk_mm::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, phys, pointer::PhysicalRefMut, - table::EntryLevelExt, + table::{EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable}, + TableAllocatorImpl, }; use yggdrasil_abi::error::Error; -use crate::{ - arch::x86_64::intrinsics, - mem::{ - process::ProcessAddressSpaceManager, - table::{EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable}, - }, -}; - -use super::{ - clone_kernel_tables, - table::{PageEntry, PageTable, L0, L1, L2, L3}, -}; +use crate::mem::process::ProcessAddressSpaceManager; /// Represents a process or kernel address space. Because x86-64 does not have cool stuff like /// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space. @@ -72,8 +66,9 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { } unsafe fn clear(&mut self) { - self.l0 - .drop_range(0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::())); + self.l0.drop_range::( + 0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::()), + ); } } @@ -90,9 +85,9 @@ impl ProcessAddressSpaceImpl { let l2i = virt.page_index::(); let l3i = virt.page_index::(); - let mut l1 = self.l0.get_mut_or_alloc(l0i)?; - let mut l2 = l1.get_mut_or_alloc(l1i)?; - let mut l3 = l2.get_mut_or_alloc(l2i)?; + let mut l1 = self.l0.get_mut_or_alloc::(l0i)?; + let mut l2 = l1.get_mut_or_alloc::(l1i)?; + let mut l3 = l2.get_mut_or_alloc::(l2i)?; if l3[l3i].is_present() && !overwrite { todo!(); @@ -100,7 +95,7 @@ impl ProcessAddressSpaceImpl { l3[l3i] = entry; unsafe { - intrinsics::flush_tlb_entry(virt); + flush_tlb_entry(virt); } Ok(()) @@ -121,7 +116,7 @@ impl ProcessAddressSpaceImpl { l3[l3i] = PageEntry::INVALID; unsafe { - intrinsics::flush_tlb_entry(virt); + flush_tlb_entry(virt); } Ok(page) diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 962e00eb..5d0622d5 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -10,10 +10,14 @@ use device_api::{ Device, }; use git_version::git_version; +use kernel_arch_x86_64::mem::{ + init_fixed_tables, map_heap_block, + table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}, + EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1, +}; use kernel_fs::devfs; use libk_mm::{ - address::{FromRaw, IntoRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, + address::{FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, table::EntryLevelExt, }; @@ -32,19 +36,11 @@ mod gdt; mod intrinsics; pub mod mem; mod peripherals; -mod registers; mod smp; mod syscall; use crate::{ - arch::x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - mem::{ - map_heap_block, - table::{PageTable, L2}, - HEAP_MAPPING_OFFSET, - }, - }, + arch::x86_64::intrinsics::{IoPort, IoPortAccess}, debug::{self, LogLevel}, device::{ self, @@ -60,11 +56,6 @@ use self::{ boot::BootData, cpu::Cpu, cpuid::{ProcessorFeatures, PROCESSOR_FEATURES}, - mem::{ - init_fixed_tables, - table::{PageAttributes, PageEntry, L1, L3}, - EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET, - }, peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort}, smp::CPU_COUNT, }; @@ -110,7 +101,7 @@ pub static PLATFORM: X86_64 = X86_64 { impl Platform for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; type IrqNumber = IrqNumber; - type L3 = mem::table::L3; + type L3 = kernel_arch_x86_64::mem::table::L3; unsafe fn start_application_processors(&self) { if let Some(acpi) = self.acpi.try_get() { @@ -130,41 +121,6 @@ impl Platform for X86_64 { CPU_COUNT.load(Ordering::Acquire) } - #[inline] - unsafe fn map_device_memory( - &self, - base: PhysicalAddress, - size: usize, - attrs: DeviceMemoryAttributes, - ) -> Result { - mem::map_device_memory(base, size, attrs) - } - - #[inline] - unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) { - mem::unmap_device_memory(map) - } - - #[inline] - fn virtualize(address: u64) -> usize { - let address = address as usize; - if address < *mem::MEMORY_LIMIT.get() { - address + RAM_MAPPING_OFFSET - } else { - panic!("Invalid physical address: {:#x}", address); - } - } - - #[inline] - fn physicalize(address: usize) -> u64 { - if address < RAM_MAPPING_OFFSET || address - RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() - { - panic!("Not a virtualized physical address: {:#x}", address); - } - - (address - RAM_MAPPING_OFFSET) as _ - } - fn external_interrupt_controller( &'static self, ) -> &'static dyn ExternalInterruptController { @@ -237,7 +193,7 @@ impl X86_64 { ); } - MEMORY_LIMIT.init(memory_end.into_raw()); + MEMORY_LIMIT.store(memory_end.into_raw(), Ordering::Release); // Check if 1GiB pages are supported if PROCESSOR_FEATURES @@ -510,7 +466,7 @@ impl X86_64 { let data = unsafe { core::slice::from_raw_parts( - start_aligned.virtualize_raw() as *const u8, + start_aligned.virtualize() as *const u8, initrd_end - initrd_start, ) }; diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index fb94e415..c91fdf6c 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -3,24 +3,20 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; use kernel_arch::{Architecture, ArchitectureImpl}; +use kernel_arch_x86_64::mem::{ + flush_tlb_entry, + table::{PageAttributes, PageEntry, PageTable, L1, L2}, + KERNEL_TABLES, +}; use libk_mm::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys, pointer::PhysicalRefMut, + TableAllocatorImpl, }; use crate::{ - arch::{ - x86_64::{ - boot::__x86_64_ap_entry, - intrinsics::flush_tlb_entry, - mem::{ - table::{PageAttributes, PageEntry, PageTable, L1, L2}, - KERNEL_TABLES, - }, - }, - CpuAccess, - }, + arch::{x86_64::boot::__x86_64_ap_entry, CpuAccess}, task::Cpu, }; @@ -53,7 +49,7 @@ unsafe fn start_ap_core(apic_id: u32) { let cr3 = KERNEL_TABLES.as_physical_address(); let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES) .unwrap() - .virtualize_raw(); + .virtualize(); let stack_size = AP_STACK_PAGES * 0x1000; let data = ApBootstrapData { @@ -92,8 +88,8 @@ pub unsafe fn start_ap_cores(info: &ProcessorInfo) { } // Temporarily identity-map the lowest 2MiB - let mut identity_l1 = PageTable::::new_zeroed().unwrap(); - let mut identity_l2 = PageTable::::new_zeroed().unwrap(); + let mut identity_l1 = PageTable::::new_zeroed::().unwrap(); + let mut identity_l2 = PageTable::::new_zeroed::().unwrap(); identity_l1[0] = PageEntry::::table(identity_l2.as_physical_address(), PageAttributes::WRITABLE); diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 6f5b469a..21c37090 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -3,10 +3,12 @@ use core::arch::global_asm; use abi::{arch::SavedFrame, error::Error, process::SignalEntryData, syscall::SyscallFunction}; +use kernel_arch_x86_64::registers::{ + MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR, +}; use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ - arch::x86_64::registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, syscall::raw_syscall_handler, task::{ context::{ForkFrame, TaskFrame}, diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 387647ce..4cf906c5 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -8,7 +8,7 @@ use core::{ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; use libk_mm::{ - address::PhysicalAddress, + address::{IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, table::EntryLevel, PageProvider, @@ -59,7 +59,7 @@ impl LinearFramebuffer { ) -> Result { let base = unsafe { RawDeviceMemoryMapping::map( - phys_base, + phys_base.into_raw(), size, DeviceMemoryAttributes { caching: DeviceMemoryCaching::Cacheable, diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 57c91747..ce0ff18d 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -3,7 +3,10 @@ use core::ptr::NonNull; use kernel_fs::devfs; -use libk_mm::{address::PhysicalAddress, phys}; +use libk_mm::{ + address::{PhysicalAddress, Virtualize}, + phys, +}; use libk_util::OneTimeInit; use memfs::block::{self, BlockAllocator}; use vfs::{impls::read_fn_node, NodeRef}; @@ -35,7 +38,7 @@ unsafe impl BlockAllocator for FileBlockAllocator { // TODO make this a static assertion assert_eq!(block::SIZE, 4096); let page = phys::alloc_page()?; - Ok(unsafe { NonNull::new_unchecked(page.virtualize_raw() as *mut _) }) + Ok(unsafe { NonNull::new_unchecked(page.virtualize() as *mut _) }) } unsafe fn dealloc(block: NonNull) { diff --git a/src/mem/address.rs b/src/mem/address.rs index 60eb04b9..d6a0f180 100644 --- a/src/mem/address.rs +++ b/src/mem/address.rs @@ -4,42 +4,3 @@ use libk_mm::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; use super::KERNEL_VIRT_OFFSET; use core::ops::{Deref, DerefMut}; - -/// Wrapper type to represent an object residing within the kernel -#[repr(transparent)] -pub struct KernelImageObject { - inner: T, -} - -// KernelImageObject wrapper for objects inside the kernel - -impl KernelImageObject { - /// Wraps a value in the [KernelImageObject], allowing its physical address calculation. - /// - /// # Safety - /// - /// The caller must ensure the T is a `static (mut)` binding inside the kernel. - pub const unsafe fn new(inner: T) -> Self { - Self { inner } - } -} - -impl AsPhysicalAddress for KernelImageObject { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - PhysicalAddress::from_raw(&self.inner as *const _ as usize - KERNEL_VIRT_OFFSET) - } -} - -impl Deref for KernelImageObject { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl DerefMut for KernelImageObject { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} diff --git a/src/mem/mod.rs b/src/mem/mod.rs index dc4b082c..9c320e04 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -7,7 +7,10 @@ use core::{ }; use abi::error::Error; -use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; +use libk_mm::{ + address::{PhysicalAddress, Virtualize}, + device::DeviceMemoryMapping, +}; use crate::arch::{Platform, PlatformImpl}; @@ -139,7 +142,7 @@ impl ForeignPointer for T { .translate(start_page) .expect("Address is not mapped in the target address space"); - let virt_ptr = phys_page.add(page_offset).virtualize_raw() as *mut T; + let virt_ptr = phys_page.add(page_offset).virtualize() as *mut T; virt_ptr.write_volatile(value); } diff --git a/src/mem/process.rs b/src/mem/process.rs index 0e67fe8a..1db8e985 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -8,7 +8,7 @@ use libk_mm::{ address::PhysicalAddress, phys, pointer::{PhysicalRef, PhysicalRefMut}, - table::EntryLevelExt, + table::{EntryLevelExt, MapAttributes}, PageProvider, }; use libk_util::sync::IrqSafeSpinlock; @@ -17,8 +17,6 @@ use vmalloc::{RangeData, VirtualMemoryAllocator}; use crate::arch::L3; -use super::table::MapAttributes; - cfg_if! { if #[cfg(target_arch = "aarch64")] { use crate::arch::aarch64::mem::process::ProcessAddressSpaceImpl; diff --git a/src/mem/table.rs b/src/mem/table.rs index ecd1dd34..8b53a739 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -4,55 +4,3 @@ use core::ops::{Deref, DerefMut, Range}; use abi::error::Error; use bitflags::bitflags; pub use libk_mm::table::EntryLevel; - -// TODO EXECUTABLE -bitflags! { - /// Describes how a page translation mapping should behave - #[derive(Clone, Copy)] - pub struct MapAttributes: u64 { - /// The data mapped can be read by the user process - const USER_READ = 1 << 0; - /// The data mapped can be written to by the user process - const USER_WRITE = 1 << 1; - /// The mapping is not global across the address spaces - const NON_GLOBAL = 1 << 2; - } -} - -/// Interface for destroying memory translation tables -pub trait EntryLevelDrop { - /// Range covering the whole table - const FULL_RANGE: Range; - - /// Recursively destroys the specified range within the table - unsafe fn drop_range(&mut self, range: Range); - - /// Recursively destroys all the entries in within the table - unsafe fn drop_all(&mut self) { - self.drop_range(Self::FULL_RANGE) - } -} - -/// Interface for non-terminal tables to retrieve the next level of address translation tables -pub trait NextPageTable { - /// Type for the next-level page table - type NextLevel; - /// Type for an immutable reference to the next-level page table - type TableRef: Deref; - /// Type for a mutable reference to the next-level page table - type TableRefMut: DerefMut; - - /// Tries looking up a next-level table at given index, allocating and mapping one if it is not - /// present there - fn get_mut_or_alloc(&mut self, index: usize) -> Result; - /// Returns a mutable reference to a next-level table at `index`, if present - fn get_mut(&mut self, index: usize) -> Option; - /// Returns an immutable reference to a next-level table at `index`, if present - fn get(&self, index: usize) -> Option; -} - -/// Tag trait to mark that the page table level may point to a next-level table -pub trait NonTerminalEntryLevel: EntryLevel { - /// Tag type of the level this entry level may point to - type NextLevel: EntryLevel; -} diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 05b061b6..7fa09f97 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -9,17 +9,14 @@ use elf::{ }; use libk_mm::{ pointer::{PhysicalRef, PhysicalRefMut}, - table::EntryLevelExt, + table::{EntryLevelExt, MapAttributes}, }; use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ arch::L3, - mem::{ - process::{ProcessAddressSpace, VirtualRangeBacking}, - table::MapAttributes, - }, + mem::process::{ProcessAddressSpace, VirtualRangeBacking}, task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 37c0f88f..6ecf3535 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -14,13 +14,12 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; -use libk_mm::pointer::PhysicalRefMut; +use libk_mm::{pointer::PhysicalRefMut, table::MapAttributes}; use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ mem::{ process::{ProcessAddressSpace, VirtualRangeBacking}, - table::MapAttributes, ForeignPointer, }, proc, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index faae3695..371ea71c 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -19,7 +19,10 @@ use abi::{ }; use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use libk::{block, runtime}; -use libk_mm::{phys, table::EntryLevelExt}; +use libk_mm::{ + phys, + table::{EntryLevelExt, MapAttributes}, +}; use libk_util::sync::IrqSafeSpinlockGuard; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; @@ -29,7 +32,7 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::{process::VirtualRangeBacking, table::MapAttributes}, + mem::process::VirtualRangeBacking, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, From 8978995e9246750dbb0d7edabda7df55bd58be53 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 6 Feb 2024 16:38:39 +0200 Subject: [PATCH 183/211] libk: move ProcessAddressSpace to libk-mm --- arch/aarch64/src/lib.rs | 2 +- arch/aarch64/src/mem/mod.rs | 1 + .../aarch64/src}/mem/process.rs | 68 +-- arch/aarch64/src/mem/table.rs | 11 +- arch/src/lib.rs | 2 +- arch/x86_64/src/lib.rs | 2 +- arch/x86_64/src/mem/mod.rs | 1 + .../x86_64 => arch/x86_64/src}/mem/process.rs | 62 +-- driver/block/core/src/device.rs | 11 +- lib/vfs/src/file/mod.rs | 11 +- lib/vfs/src/shared_memory.rs | 10 + libk-mm/Cargo.toml | 1 + libk-mm/interface/src/address.rs | 2 +- libk-mm/interface/src/lib.rs | 1 + libk-mm/interface/src/process.rs | 47 ++ libk-mm/src/lib.rs | 9 +- libk-mm/src/process.rs | 357 ++++++++++++++ src/arch/aarch64/mem/mod.rs | 4 - src/arch/aarch64/mem/table.rs | 0 src/arch/aarch64/mod.rs | 5 +- src/arch/mod.rs | 6 +- src/arch/x86_64/mem/mod.rs | 3 - src/arch/x86_64/mod.rs | 5 +- src/device/display/linear_fb.rs | 11 +- src/mem/address.rs | 6 - src/mem/mod.rs | 2 - src/mem/process.rs | 438 +----------------- src/mem/table.rs | 6 - 28 files changed, 562 insertions(+), 522 deletions(-) rename {src/arch/aarch64 => arch/aarch64/src}/mem/process.rs (68%) rename {src/arch/x86_64 => arch/x86_64/src}/mem/process.rs (72%) create mode 100644 libk-mm/interface/src/process.rs create mode 100644 libk-mm/src/process.rs delete mode 100644 src/arch/aarch64/mem/mod.rs delete mode 100644 src/arch/aarch64/mem/table.rs delete mode 100644 src/arch/x86_64/mem/mod.rs delete mode 100644 src/mem/address.rs delete mode 100644 src/mem/table.rs diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index 6f861732..7ce6f156 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -7,7 +7,7 @@ use tock_registers::interfaces::{ReadWriteable, Readable}; pub mod mem; -pub use mem::KernelTableManagerImpl; +pub use mem::{process::ProcessAddressSpaceImpl, KernelTableManagerImpl}; pub struct ArchitectureImpl; diff --git a/arch/aarch64/src/mem/mod.rs b/arch/aarch64/src/mem/mod.rs index c2eaf9b7..e7fc6ae7 100644 --- a/arch/aarch64/src/mem/mod.rs +++ b/arch/aarch64/src/mem/mod.rs @@ -23,6 +23,7 @@ use yggdrasil_abi::error::Error; use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}; +pub mod process; pub mod table; #[derive(Debug)] diff --git a/src/arch/aarch64/mem/process.rs b/arch/aarch64/src/mem/process.rs similarity index 68% rename from src/arch/aarch64/mem/process.rs rename to arch/aarch64/src/mem/process.rs index 14fc0a13..8fb9a4d9 100644 --- a/src/arch/aarch64/mem/process.rs +++ b/arch/aarch64/src/mem/process.rs @@ -1,46 +1,59 @@ //! AArch64-specific process address space management -use core::sync::atomic::{AtomicU8, Ordering}; +use core::{ + marker::PhantomData, + sync::atomic::{AtomicU8, Ordering}, +}; -use abi::error::Error; -use kernel_arch_aarch64::mem::{ - table::{PageEntry, PageTable, L1, L2, L3}, +use libk_mm_interface::{ + address::{AsPhysicalAddress, PhysicalAddress}, + pointer::PhysicalRefMut, + process::ProcessAddressSpaceManager, + table::{ + EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator, + }, +}; +use yggdrasil_abi::error::Error; + +use crate::{mem::table::PageEntry, KernelTableManagerImpl}; + +use super::{ + table::{PageTable, L1, L2, L3}, tlb_flush_vaae1, }; -use libk_mm::{ - address::{AsPhysicalAddress, PhysicalAddress}, - phys, - pointer::PhysicalRefMut, - table::{EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable}, - TableAllocatorImpl, -}; - -use crate::mem::process::ProcessAddressSpaceManager; /// AArch64 implementation of a process address space table #[repr(C)] -pub struct ProcessAddressSpaceImpl { - l1: PhysicalRefMut<'static, PageTable>, +pub struct ProcessAddressSpaceImpl { + l1: PhysicalRefMut<'static, PageTable, KernelTableManagerImpl>, asid: u8, + _alloc: PhantomData, } -impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { - const PAGE_SIZE: usize = L3::SIZE; +impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { const LOWER_LIMIT_PFN: usize = 8; // 16GiB VM limit - const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE; + const UPPER_LIMIT_PFN: usize = (16 << 30) / L3::SIZE; fn new() -> Result { static LAST_ASID: AtomicU8 = AtomicU8::new(1); let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); - let mut l1 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; + let mut l1 = unsafe { + PhysicalRefMut::<'static, PageTable, KernelTableManagerImpl>::map( + TA::allocate_page_table()?, + ) + }; for i in 0..512 { l1[i] = PageEntry::INVALID; } - Ok(Self { l1, asid }) + Ok(Self { + l1, + asid, + _alloc: PhantomData, + }) } fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> { @@ -69,13 +82,12 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { } unsafe fn clear(&mut self) { - self.l1.drop_range::( - 0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::()), - ); + self.l1 + .drop_range::(0..((Self::UPPER_LIMIT_PFN * L3::SIZE).page_index::())); } } -impl ProcessAddressSpaceImpl { +impl ProcessAddressSpaceImpl { // Write a single 4KiB entry fn write_l3_entry( &mut self, @@ -87,8 +99,8 @@ impl ProcessAddressSpaceImpl { let l2i = virt.page_index::(); let l3i = virt.page_index::(); - let mut l2 = self.l1.get_mut_or_alloc::(l1i)?; - let mut l3 = l2.get_mut_or_alloc::(l2i)?; + let mut l2 = self.l1.get_mut_or_alloc::(l1i)?; + let mut l3 = l2.get_mut_or_alloc::(l2i)?; if l3[l3i].is_present() && !overwrite { todo!(); @@ -131,14 +143,14 @@ impl ProcessAddressSpaceImpl { } } -impl Drop for ProcessAddressSpaceImpl { +impl Drop for ProcessAddressSpaceImpl { fn drop(&mut self) { // SAFETY: with safe usage of the ProcessAddressSpaceImpl, clearing and dropping // is safe, no one refers to the memory unsafe { self.clear(); let l1_phys = self.l1.as_physical_address(); - phys::free_page(l1_phys); + TA::free_page_table(l1_phys); } } } diff --git a/arch/aarch64/src/mem/table.rs b/arch/aarch64/src/mem/table.rs index 6fd34b31..3a5fe69b 100644 --- a/arch/aarch64/src/mem/table.rs +++ b/arch/aarch64/src/mem/table.rs @@ -14,16 +14,7 @@ use libk_mm_interface::{ }; use yggdrasil_abi::error::Error; -use crate::{ArchitectureImpl, KernelTableManagerImpl}; - -// use abi::error::Error; -// use bitflags::bitflags; -// use libk_mm::{ -// address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, -// phys, -// pointer::{PhysicalRef, PhysicalRefMut}, -// table::EntryLevel, -// }; +use crate::KernelTableManagerImpl; bitflags! { #[derive(Clone, Copy, PartialEq, Eq)] diff --git a/arch/src/lib.rs b/arch/src/lib.rs index 39c6d94d..c733cde5 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -29,6 +29,6 @@ cfg_if! { } } -pub use imp::{ArchitectureImpl, KernelTableManagerImpl}; +pub use imp::{ArchitectureImpl, KernelTableManagerImpl, ProcessAddressSpaceImpl}; pub use kernel_arch_interface::{mem, Architecture}; diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index f14e013d..a0a98925 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -6,7 +6,7 @@ use kernel_arch_interface::Architecture; pub mod mem; pub mod registers; -pub use mem::KernelTableManagerImpl; +pub use mem::{process::ProcessAddressSpaceImpl, KernelTableManagerImpl}; pub struct ArchitectureImpl; diff --git a/arch/x86_64/src/mem/mod.rs b/arch/x86_64/src/mem/mod.rs index 0d469b54..0d495b65 100644 --- a/arch/x86_64/src/mem/mod.rs +++ b/arch/x86_64/src/mem/mod.rs @@ -21,6 +21,7 @@ use crate::{registers::CR3, KERNEL_VIRT_OFFSET}; use self::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3}; +pub mod process; pub mod table; #[derive(Debug)] diff --git a/src/arch/x86_64/mem/process.rs b/arch/x86_64/src/mem/process.rs similarity index 72% rename from src/arch/x86_64/mem/process.rs rename to arch/x86_64/src/mem/process.rs index 4454717b..bd4a2059 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/arch/x86_64/src/mem/process.rs @@ -1,35 +1,43 @@ //! x86-64-specific process address space management functions -use kernel_arch_x86_64::mem::{ - clone_kernel_tables, flush_tlb_entry, - table::{PageEntry, PageTable, L0, L1, L2, L3}, -}; -use libk_mm::{ +use core::marker::PhantomData; + +use libk_mm_interface::{ address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, - phys, pointer::PhysicalRefMut, - table::{EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable}, - TableAllocatorImpl, + process::ProcessAddressSpaceManager, + table::{ + EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator, + }, }; use yggdrasil_abi::error::Error; -use crate::mem::process::ProcessAddressSpaceManager; +use crate::KernelTableManagerImpl; + +use super::{ + clone_kernel_tables, flush_tlb_entry, + table::{PageEntry, PageTable, L0, L1, L2, L3}, +}; /// Represents a process or kernel address space. Because x86-64 does not have cool stuff like /// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space. #[repr(C)] -pub struct ProcessAddressSpaceImpl { - l0: PhysicalRefMut<'static, PageTable>, +pub struct ProcessAddressSpaceImpl { + l0: PhysicalRefMut<'static, PageTable, KernelTableManagerImpl>, + _alloc: PhantomData, } -impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { - const PAGE_SIZE: usize = L3::SIZE; +impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { // Start with 8GiB - const LOWER_LIMIT_PFN: usize = (8 << 30) / Self::PAGE_SIZE; + const LOWER_LIMIT_PFN: usize = (8 << 30) / L3::SIZE; // 16GiB VM limit - const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE; + const UPPER_LIMIT_PFN: usize = (16 << 30) / L3::SIZE; fn new() -> Result { - let mut l0 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; + let mut l0 = unsafe { + PhysicalRefMut::<'static, PageTable, KernelTableManagerImpl>::map( + TA::allocate_page_table()?, + ) + }; for i in 0..512 { l0[i] = PageEntry::INVALID; @@ -37,7 +45,10 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { clone_kernel_tables(&mut l0); - Ok(Self { l0 }) + Ok(Self { + l0, + _alloc: PhantomData, + }) } #[inline] @@ -66,13 +77,12 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { } unsafe fn clear(&mut self) { - self.l0.drop_range::( - 0..((Self::UPPER_LIMIT_PFN * Self::PAGE_SIZE).page_index::()), - ); + self.l0 + .drop_range::(0..((Self::UPPER_LIMIT_PFN * L3::SIZE).page_index::())); } } -impl ProcessAddressSpaceImpl { +impl ProcessAddressSpaceImpl { // Write a single 4KiB entry fn write_l3_entry( &mut self, @@ -85,9 +95,9 @@ impl ProcessAddressSpaceImpl { let l2i = virt.page_index::(); let l3i = virt.page_index::(); - let mut l1 = self.l0.get_mut_or_alloc::(l0i)?; - let mut l2 = l1.get_mut_or_alloc::(l1i)?; - let mut l3 = l2.get_mut_or_alloc::(l2i)?; + let mut l1 = self.l0.get_mut_or_alloc::(l0i)?; + let mut l2 = l1.get_mut_or_alloc::(l1i)?; + let mut l3 = l2.get_mut_or_alloc::(l2i)?; if l3[l3i].is_present() && !overwrite { todo!(); @@ -138,14 +148,14 @@ impl ProcessAddressSpaceImpl { } } -impl Drop for ProcessAddressSpaceImpl { +impl Drop for ProcessAddressSpaceImpl { fn drop(&mut self) { // SAFETY: with safe usage of the ProcessAddressSpaceImpl, clearing and dropping // is safe, no one refers to the memory unsafe { self.clear(); let l0_phys = self.l0.as_physical_address(); - phys::free_page(l0_phys); + TA::free_page_table(l0_phys); } } } diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index 99a693f8..d924baca 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -8,7 +8,7 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use libk_mm::{address::PhysicalAddress, PageBox, PageProvider}; +use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageBox, PageProvider}; use libk_util::waker::QueueWaker; use yggdrasil_abi::{error::Error, io::DeviceRequest}; @@ -246,6 +246,15 @@ impl<'a, D: NgBlockDevice + 'a> PageProvider for NgBlockDeviceWrapper<'a, D> { fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { todo!() } + + fn clone_page( + &self, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, + ) -> Result { + todo!() + } } impl<'a, D: NgBlockDevice + 'a> BlockDevice for NgBlockDeviceWrapper<'a, D> { diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 061f3ec3..296f6b81 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -9,7 +9,7 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use libk_mm::{address::PhysicalAddress, PageProvider}; +use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider}; use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{ error::Error, @@ -363,6 +363,15 @@ impl PageProvider for File { _ => Err(Error::InvalidOperation), } } + + fn clone_page( + &self, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, + ) -> Result { + todo!() + } } impl Read for File { diff --git a/lib/vfs/src/shared_memory.rs b/lib/vfs/src/shared_memory.rs index 85aac31f..e852a9a7 100644 --- a/lib/vfs/src/shared_memory.rs +++ b/lib/vfs/src/shared_memory.rs @@ -3,6 +3,7 @@ use core::mem::MaybeUninit; use alloc::vec::Vec; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, + table::MapAttributes, PageBox, PageProvider, }; use yggdrasil_abi::error::Error; @@ -40,4 +41,13 @@ impl PageProvider for SharedMemory { // TODO track get/release? Ok(()) } + + fn clone_page( + &self, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, + ) -> Result { + todo!() + } } diff --git a/libk-mm/Cargo.toml b/libk-mm/Cargo.toml index 59527852..b0f9a1c4 100644 --- a/libk-mm/Cargo.toml +++ b/libk-mm/Cargo.toml @@ -10,5 +10,6 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch = { path = "../arch" } libk-util = { path = "../libk-util" } libk-mm-interface = { path = "interface" } +vmalloc = { path = "../lib/vmalloc" } log = "0.4.20" diff --git a/libk-mm/interface/src/address.rs b/libk-mm/interface/src/address.rs index 4fac2a4f..ff27c3b3 100644 --- a/libk-mm/interface/src/address.rs +++ b/libk-mm/interface/src/address.rs @@ -5,7 +5,7 @@ use core::{ ops::{Add, Sub}, }; -use kernel_arch_interface::{mem::KernelTableManager, Architecture}; +use kernel_arch_interface::mem::KernelTableManager; /// Wrapper type to represent a physical memory address #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] diff --git a/libk-mm/interface/src/lib.rs b/libk-mm/interface/src/lib.rs index d2c5feae..55d6ea10 100644 --- a/libk-mm/interface/src/lib.rs +++ b/libk-mm/interface/src/lib.rs @@ -8,6 +8,7 @@ use kernel_arch_interface::KERNEL_VIRT_OFFSET; pub mod address; pub mod pointer; +pub mod process; pub mod table; /// Wrapper type to represent an object residing within the kernel diff --git a/libk-mm/interface/src/process.rs b/libk-mm/interface/src/process.rs new file mode 100644 index 00000000..e44a68fc --- /dev/null +++ b/libk-mm/interface/src/process.rs @@ -0,0 +1,47 @@ +use yggdrasil_abi::error::Error; + +use crate::{ + address::PhysicalAddress, + table::{MapAttributes, TableAllocator}, +}; + +/// Interface for virtual memory address space management +pub trait ProcessAddressSpaceManager: Sized { + /// PFN of a minimum address allowed for virtual region allocation + const LOWER_LIMIT_PFN: usize; + /// PFN of a maximum address allowed for virtual region allocation + const UPPER_LIMIT_PFN: usize; + + /// Constructs a new implementation-specific per-process address space + fn new() -> Result; + + /// Places a single PAGE_SIZE mapping into the address space. + /// + /// # Safety + /// + /// The caller must ensure the correct origin of the physical address being mapped. + unsafe fn map_page( + &mut self, + address: usize, + physical: PhysicalAddress, + flags: MapAttributes, + ) -> Result<(), Error>; + + /// Removes a single PAGE_SIZE mapping from the address space. + /// + /// # Safety + /// + /// The caller must ensure the process to which this address space belongs does not and + /// will not access this page. + unsafe fn unmap_page(&mut self, address: usize) -> Result; + + /// Returns the [PhysicalAddress] and [MapAttributes] associated with given virtual `address`, + /// if one is mapped + fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; + + /// Returns the implementation specific physical address of this space, with ASID applied + fn as_address_with_asid(&self) -> u64; + + /// Clears the address space by dropping and non-global tables + unsafe fn clear(&mut self); +} diff --git a/libk-mm/src/lib.rs b/libk-mm/src/lib.rs index bd67664e..106c961f 100644 --- a/libk-mm/src/lib.rs +++ b/libk-mm/src/lib.rs @@ -20,7 +20,7 @@ use core::{ use address::Virtualize; use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, - table::TableAllocator, + table::{MapAttributes, TableAllocator}, }; use yggdrasil_abi::error::Error; @@ -28,6 +28,7 @@ pub mod address; pub mod device; pub mod phys; pub mod pointer; +pub mod process; pub use libk_mm_interface::table; @@ -50,6 +51,12 @@ pub const L2_PAGE_SIZE: usize = 1 << 21; pub trait PageProvider { fn get_page(&self, offset: u64) -> Result; fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>; + fn clone_page( + &self, + offset: u64, + src_phys: PhysicalAddress, + src_attrs: MapAttributes, + ) -> Result; } pub struct PageBox { diff --git a/libk-mm/src/process.rs b/libk-mm/src/process.rs new file mode 100644 index 00000000..bdb27eb7 --- /dev/null +++ b/libk-mm/src/process.rs @@ -0,0 +1,357 @@ +use core::ops::Range; + +use kernel_arch::ProcessAddressSpaceImpl; +use libk_mm_interface::{ + address::PhysicalAddress, + process::ProcessAddressSpaceManager, + table::{MapAttributes, TableAllocator}, +}; +use libk_util::sync::IrqSafeSpinlock; +use vmalloc::{RangeData, VirtualMemoryAllocator}; +use yggdrasil_abi::error::Error; + +use crate::{phys, PageProvider, TableAllocatorImpl, L3_PAGE_SIZE}; + +struct Inner { + allocator: VirtualMemoryAllocator, + table: ProcessAddressSpaceImpl, +} + +/// Data structure for managing the address translation and allocation for a single process +pub struct ProcessAddressSpace< + PP: PageProvider + RangeData, + TA: TableAllocator = TableAllocatorImpl, +> { + inner: IrqSafeSpinlock>, +} + +impl Inner { + fn try_map_pages( + &mut self, + address: usize, + page_count: usize, + backing: &PP, + attributes: MapAttributes, + ) -> Result<(), (usize, Error)> { + for i in 0..page_count { + let offset = (i * L3_PAGE_SIZE) as u64; + let virt = address + i * L3_PAGE_SIZE; + let phys = match backing.get_page(offset) { + Ok(page) => page, + Err(err) => { + return Err((i, err)); + } + }; + + if let Err(err) = unsafe { self.table.map_page(virt, phys, attributes) } { + backing.release_page(offset, phys).unwrap(); + return Err((i, err)); + } + } + + Ok(()) + } + + unsafe fn rollback_allocation( + &mut self, + start_pfn: usize, + pages_mapped: usize, + region_size: usize, + ) { + let unmap_range = start_pfn..start_pfn + pages_mapped; + self.allocator + .free(start_pfn, region_size, |origin_pfn, pfn_range, backing| { + for pfn in pfn_range { + if unmap_range.contains(&pfn) { + let offset = (pfn - origin_pfn) * L3_PAGE_SIZE; + let virt = pfn * L3_PAGE_SIZE; + + let phys = self.table.unmap_page(virt)?; + + backing.release_page(offset as u64, phys)?; + } + } + + Ok(()) + }) + .unwrap(); + } + + fn map_range( + &mut self, + address: usize, + page_count: usize, + backing: PP, + attributes: MapAttributes, + ) -> Result<(), Error> { + // If inserting fails, the range cannot be mapped + let start_pfn = address / L3_PAGE_SIZE; + self.allocator + .insert(start_pfn, page_count, backing.clone())?; + + if let Err((mapped, error)) = self.try_map_pages(address, page_count, &backing, attributes) + { + debug_assert!(mapped < page_count); + unsafe { + self.rollback_allocation(start_pfn, mapped, page_count); + } + return Err(error); + }; + + Ok(()) + } + + fn map_single( + &mut self, + address: usize, + backing: PP, + attributes: MapAttributes, + ) -> Result { + let start_pfn = address / L3_PAGE_SIZE; + self.allocator.insert(start_pfn, 1, backing.clone())?; + + let phys = match backing.get_page(0) { + Ok(page) => page, + Err(err) => { + // Do nothing, as the page has not been allocated to this range yet + self.allocator.free(start_pfn, 1, |_, _, _| Ok(())).unwrap(); + return Err(err); + } + }; + + if let Err(err) = unsafe { self.table.map_page(address, phys, attributes) } { + self.allocator + .free(start_pfn, 1, |_, _, _| { + // Deallocate the page, but do not unmap, as the mapping failed + unsafe { + phys::free_page(phys); + } + Ok(()) + }) + .unwrap(); + return Err(err); + } + + Ok(phys) + } + + fn alloc_range( + &mut self, + page_count: usize, + backing: PP, + attributes: MapAttributes, + ) -> Result { + let start_pfn = self.allocator.allocate(page_count, backing.clone())?; + let address = start_pfn * L3_PAGE_SIZE; + + if let Err((mapped, error)) = self.try_map_pages(address, page_count, &backing, attributes) + { + debug_assert!(mapped < page_count); + unsafe { + self.rollback_allocation(start_pfn, mapped, page_count); + } + return Err(error); + }; + + Ok(address) + } + + unsafe fn unmap_range(&mut self, start_address: usize, page_count: usize) -> Result<(), Error> { + let start_pfn = start_address / L3_PAGE_SIZE; + + self.allocator + .free(start_pfn, page_count, |origin_pfn, pfn_range, backing| { + for pfn in pfn_range { + let offset = ((pfn - origin_pfn) * L3_PAGE_SIZE) as u64; + + let virt = pfn * L3_PAGE_SIZE; + let phys = self.table.unmap_page(virt)?; + + backing.release_page(offset, phys)?; + } + + Ok(()) + })?; + + Ok(()) + } + + unsafe fn clear(&mut self) -> Result<(), Error> { + self.allocator.clear(|pfn_range, backing| { + let origin_pfn = pfn_range.start; + for pfn in pfn_range { + let offset = ((pfn - origin_pfn) * L3_PAGE_SIZE) as u64; + + let virt = pfn * L3_PAGE_SIZE; + let phys = unsafe { self.table.unmap_page(virt)? }; + + backing.release_page(offset, phys)?; + } + + Ok(()) + })?; + + // Drop the tables + self.table.clear(); + + Ok(()) + } + + fn clone_range( + &mut self, + source: &Self, + pfn_range: Range, + backing: &PP, + ) -> Result<(), Error> { + self.allocator + .insert(pfn_range.start, pfn_range.len(), backing.clone()) + .unwrap(); + + let start = pfn_range.start * L3_PAGE_SIZE; + let end = pfn_range.end * L3_PAGE_SIZE; + + log::debug!("clone_range({:#x?})", start..end); + + for i in pfn_range { + let address = i * L3_PAGE_SIZE; + let offset = (address - start) as u64; + let (src_page, attrs) = source.table.translate(address).unwrap(); + let dst_page = backing.clone_page(offset, src_page, attrs)?; + unsafe { + self.table.map_page(address, dst_page, attrs).unwrap(); + } + } + + Ok(()) + } +} + +impl ProcessAddressSpace { + /// Constructs a new [ProcessAddressSpace] + pub fn new() -> Result { + let table = ProcessAddressSpaceImpl::new()?; + let allocator = VirtualMemoryAllocator::new( + ProcessAddressSpaceImpl::::LOWER_LIMIT_PFN, + ProcessAddressSpaceImpl::::UPPER_LIMIT_PFN, + ); + Ok(Self { + inner: IrqSafeSpinlock::new(Inner { table, allocator }), + }) + } + + /// Performs a "fork" operation of the address space, cloning all the mappings into a new one + pub fn fork(&self) -> Result { + let src_inner = self.inner.lock(); + let new_table = ProcessAddressSpaceImpl::new()?; + let mut new_inner = Inner { + allocator: VirtualMemoryAllocator::new( + ProcessAddressSpaceImpl::::LOWER_LIMIT_PFN, + ProcessAddressSpaceImpl::::UPPER_LIMIT_PFN, + ), + table: new_table, + }; + + log::debug!("fork address space!"); + + for (range, backing) in src_inner.allocator.regions() { + // If they are present in existing allocator, there should be no + // problem adding them to a new one + new_inner.clone_range(&src_inner, range, backing)?; + } + + for (range, _) in new_inner.allocator.regions() { + let start = range.start * L3_PAGE_SIZE; + let end = range.end * L3_PAGE_SIZE; + log::debug!("forked region: {:#x?}", start..end); + } + + Ok(Self { + inner: IrqSafeSpinlock::new(new_inner), + }) + } + + /// Allocates a region of virtual memory within the address space and maps the pages to the + /// ones returned from `get_page` function + pub fn allocate( + &self, + _hint: Option, + size: usize, + backing: PP, + attributes: MapAttributes, + ) -> Result { + assert_eq!(size & (L3_PAGE_SIZE - 1), 0); + + let mut lock = self.inner.lock(); + + lock.alloc_range(size / L3_PAGE_SIZE, backing, attributes) + } + + /// Maps a region of memory in the address space + pub fn map( + &self, + address: usize, + size: usize, + backing: PP, + attributes: MapAttributes, + ) -> Result<(), Error> { + assert_eq!(address & (L3_PAGE_SIZE - 1), 0); + assert_eq!(size & (L3_PAGE_SIZE - 1), 0); + + let mut lock = self.inner.lock(); + + lock.map_range(address, size / L3_PAGE_SIZE, backing, attributes) + } + + /// Adds a single-page mapping to the address space + pub fn map_single( + &self, + address: usize, + backing: PP, + attributes: MapAttributes, + ) -> Result { + assert_eq!(address & (L3_PAGE_SIZE - 1), 0); + + self.inner.lock().map_single(address, backing, attributes) + } + + /// Returns the [PhysicalAddress] associated with given virtual `address`, + /// if one is mapped + pub fn translate(&self, address: usize) -> Result { + // Offset is handled at impl level + self.inner.lock().table.translate(address).map(|e| e.0) + } + + /// Removes a single PAGE_SIZE mapping from the address space. + /// + /// See [ProcessAddressSpaceManager::unmap]. + /// + /// # Safety + /// + /// The caller must ensure the process to which this address space belongs does not and + /// will not access this page. + pub unsafe fn unmap(&self, address: usize, size: usize) -> Result<(), Error> { + assert_eq!(address & (L3_PAGE_SIZE - 1), 0); + assert_eq!(size & (L3_PAGE_SIZE - 1), 0); + + let mut lock = self.inner.lock(); + + lock.unmap_range(address, size / L3_PAGE_SIZE) + } + + /// Returns the physical address of this table, with ASID applied + pub fn as_address_with_asid(&self) -> u64 { + self.inner.lock().table.as_address_with_asid() + } + + /// Removes all allocations and their associated mappings from the address space + pub fn clear(&self) -> Result<(), Error> { + let mut inner = self.inner.lock(); + unsafe { inner.clear() } + } +} + +impl Drop for ProcessAddressSpace { + fn drop(&mut self) { + self.clear().ok(); + } +} diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs deleted file mode 100644 index 4d21fdf0..00000000 --- a/src/arch/aarch64/mem/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! AArch64-specific memory management interfaces and functions - -pub mod process; -pub(super) mod table; diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index b33901d4..3d8ec624 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -1,7 +1,5 @@ //! AArch64 architecture and platforms implementation -pub mod mem; - use core::sync::atomic::Ordering; use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; @@ -18,11 +16,10 @@ use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemor use git_version::git_version; use kernel_arch_aarch64::mem::{ table::{L1, L2, L3}, - EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, RAM_MAPPING_OFFSET, + EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, }; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, phys::PhysicalMemoryRegion, phys::{self, reserved::reserve_region}, pointer::PhysicalRef, diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 13f3f33d..436c3f1d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -14,11 +14,7 @@ use device_api::{ ResetDevice, }; use kernel_arch::{Architecture, ArchitectureImpl}; -use libk_mm::{ - address::PhysicalAddress, - device::{DeviceMemoryAttributes, RawDeviceMemoryMapping}, - table::EntryLevel, -}; +use libk_mm::table::EntryLevel; use libk_util::sync::IrqGuard; use crate::task::{sched::CpuQueue, thread::ThreadId, Cpu}; diff --git a/src/arch/x86_64/mem/mod.rs b/src/arch/x86_64/mem/mod.rs deleted file mode 100644 index a04d6535..00000000 --- a/src/arch/x86_64/mem/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! x86-64-specific memory management functions and interfaces - -pub mod process; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 5d0622d5..8ccf8dc5 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -19,7 +19,7 @@ use kernel_fs::devfs; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, - table::EntryLevelExt, + table::{EntryLevel, EntryLevelExt}, }; use libk_util::{sync::SpinFence, OneTimeInit}; use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1}; @@ -34,7 +34,6 @@ mod cpuid; mod exception; mod gdt; mod intrinsics; -pub mod mem; mod peripherals; mod smp; mod syscall; @@ -47,7 +46,7 @@ use crate::{ display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, }, fs::{Initrd, INITRD_DATA}, - mem::{heap, table::EntryLevel}, + mem::heap, }; use self::{ diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 4cf906c5..bd400a8e 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -10,7 +10,7 @@ use device_api::Device; use libk_mm::{ address::{IntoRaw, PhysicalAddress}, device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, - table::EntryLevel, + table::{EntryLevel, MapAttributes}, PageProvider, }; use libk_util::sync::IrqSafeSpinlock; @@ -170,6 +170,15 @@ impl PageProvider for LinearFramebuffer { fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { Ok(()) } + + fn clone_page( + &self, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, + ) -> Result { + todo!() + } } impl Device for LinearFramebuffer { diff --git a/src/mem/address.rs b/src/mem/address.rs deleted file mode 100644 index d6a0f180..00000000 --- a/src/mem/address.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Address manipulation interfaces and utilities - -use libk_mm::address::{AsPhysicalAddress, FromRaw, PhysicalAddress}; - -use super::KERNEL_VIRT_OFFSET; -use core::ops::{Deref, DerefMut}; diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 9c320e04..e5dd5d33 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -14,10 +14,8 @@ use libk_mm::{ use crate::arch::{Platform, PlatformImpl}; -pub mod address; pub mod heap; pub mod process; -pub mod table; use self::process::ProcessAddressSpace; diff --git a/src/mem/process.rs b/src/mem/process.rs index 1db8e985..779e3af2 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,9 +1,6 @@ //! Process address space structures and management functions -use core::ops::Range; - use abi::error::Error; -use cfg_if::cfg_if; use libk_mm::{ address::PhysicalAddress, phys, @@ -11,62 +8,13 @@ use libk_mm::{ table::{EntryLevelExt, MapAttributes}, PageProvider, }; -use libk_util::sync::IrqSafeSpinlock; use vfs::FileRef; -use vmalloc::{RangeData, VirtualMemoryAllocator}; +use vmalloc::RangeData; use crate::arch::L3; -cfg_if! { - if #[cfg(target_arch = "aarch64")] { - use crate::arch::aarch64::mem::process::ProcessAddressSpaceImpl; - } else if #[cfg(target_arch = "x86_64")] { - use crate::arch::x86_64::mem::process::ProcessAddressSpaceImpl; - } -} - -/// Interface for virtual memory address space management -pub trait ProcessAddressSpaceManager: Sized { - /// Page size used by this implementation - const PAGE_SIZE: usize; - /// PFN of a minimum address allowed for virtual region allocation - const LOWER_LIMIT_PFN: usize; - /// PFN of a maximum address allowed for virtual region allocation - const UPPER_LIMIT_PFN: usize; - - /// Constructs a new implementation-specific per-process address space - fn new() -> Result; - - /// Places a single PAGE_SIZE mapping into the address space. - /// - /// # Safety - /// - /// The caller must ensure the correct origin of the physical address being mapped. - unsafe fn map_page( - &mut self, - address: usize, - physical: PhysicalAddress, - flags: MapAttributes, - ) -> Result<(), Error>; - - /// Removes a single PAGE_SIZE mapping from the address space. - /// - /// # Safety - /// - /// The caller must ensure the process to which this address space belongs does not and - /// will not access this page. - unsafe fn unmap_page(&mut self, address: usize) -> Result; - - /// Returns the [PhysicalAddress] and [MapAttributes] associated with given virtual `address`, - /// if one is mapped - fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; - - /// Returns the implementation specific physical address of this space, with ASID applied - fn as_address_with_asid(&self) -> u64; - - /// Clears the address space by dropping and non-global tables - unsafe fn clear(&mut self); -} +/// Represents a process address space. See [libk_mm::process::ProcessAddressSpace]. +pub type ProcessAddressSpace = libk_mm::process::ProcessAddressSpace; /// Describes how the physical memory is provided for the mapping #[derive(Clone)] @@ -117,6 +65,24 @@ impl PageProvider for VirtualRangeBacking { Self::File(f) => f.file.release_page(f.offset + offset, phys), } } + + fn clone_page( + &self, + _offset: u64, + src_phys: PhysicalAddress, + _src_attrs: MapAttributes, + ) -> Result { + match self { + Self::Anonymous => { + let dst_page = phys::alloc_page()?; + let src_map = unsafe { PhysicalRef::<[u8; 4096]>::map(src_phys) }; + let mut dst_map = unsafe { PhysicalRefMut::<[u8; 4096]>::map(dst_page) }; + dst_map.copy_from_slice(src_map.as_ref()); + Ok(dst_page) + } + Self::File(_) => todo!(), + } + } } impl PartialEq for VirtualRangeBacking { @@ -141,365 +107,3 @@ impl core::fmt::Debug for VirtualRangeBacking { } } } - -struct Inner { - allocator: VirtualMemoryAllocator, - table: ProcessAddressSpaceImpl, -} - -/// Data structure for managing the address translation and allocation for a single process -pub struct ProcessAddressSpace { - inner: IrqSafeSpinlock, -} - -impl Inner { - fn try_map_pages( - &mut self, - address: usize, - page_count: usize, - backing: &VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result<(), (usize, Error)> { - for i in 0..page_count { - let offset = (i * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; - let virt = address + i * ProcessAddressSpaceImpl::PAGE_SIZE; - let phys = match backing.get_page(offset) { - Ok(page) => page, - Err(err) => { - return Err((i, err)); - } - }; - - if let Err(err) = unsafe { self.table.map_page(virt, phys, attributes) } { - backing.release_page(offset, phys).unwrap(); - return Err((i, err)); - } - } - - Ok(()) - } - - unsafe fn rollback_allocation( - &mut self, - start_pfn: usize, - pages_mapped: usize, - region_size: usize, - ) { - let unmap_range = start_pfn..start_pfn + pages_mapped; - self.allocator - .free(start_pfn, region_size, |origin_pfn, pfn_range, backing| { - for pfn in pfn_range { - if unmap_range.contains(&pfn) { - let offset = (pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE; - let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - - let phys = self.table.unmap_page(virt)?; - - backing.release_page(offset as u64, phys)?; - } - } - - Ok(()) - }) - .unwrap(); - } - - fn map_range( - &mut self, - address: usize, - page_count: usize, - backing: VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result<(), Error> { - // If inserting fails, the range cannot be mapped - let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; - self.allocator - .insert(start_pfn, page_count, backing.clone())?; - - if let Err((mapped, error)) = self.try_map_pages(address, page_count, &backing, attributes) - { - debug_assert!(mapped < page_count); - unsafe { - self.rollback_allocation(start_pfn, mapped, page_count); - } - return Err(error); - }; - - Ok(()) - } - - fn map_single( - &mut self, - address: usize, - backing: VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result { - let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; - self.allocator.insert(start_pfn, 1, backing.clone())?; - - let phys = match backing.get_page(0) { - Ok(page) => page, - Err(err) => { - // Do nothing, as the page has not been allocated to this range yet - self.allocator.free(start_pfn, 1, |_, _, _| Ok(())).unwrap(); - return Err(err); - } - }; - - if let Err(err) = unsafe { self.table.map_page(address, phys, attributes) } { - self.allocator - .free(start_pfn, 1, |_, _, _| { - // Deallocate the page, but do not unmap, as the mapping failed - unsafe { - phys::free_page(phys); - } - Ok(()) - }) - .unwrap(); - return Err(err); - } - - Ok(phys) - } - - fn alloc_range( - &mut self, - page_count: usize, - backing: VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result { - let start_pfn = self.allocator.allocate(page_count, backing.clone())?; - let address = start_pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - - if let Err((mapped, error)) = self.try_map_pages(address, page_count, &backing, attributes) - { - debug_assert!(mapped < page_count); - unsafe { - self.rollback_allocation(start_pfn, mapped, page_count); - } - return Err(error); - }; - - Ok(address) - } - - unsafe fn unmap_range(&mut self, start_address: usize, page_count: usize) -> Result<(), Error> { - let start_pfn = start_address / ProcessAddressSpaceImpl::PAGE_SIZE; - - self.allocator - .free(start_pfn, page_count, |origin_pfn, pfn_range, backing| { - for pfn in pfn_range { - let offset = ((pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; - - let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - let phys = self.table.unmap_page(virt)?; - - backing.release_page(offset, phys)?; - } - - Ok(()) - })?; - - Ok(()) - } - - unsafe fn clear(&mut self) -> Result<(), Error> { - self.allocator.clear(|pfn_range, backing| { - let origin_pfn = pfn_range.start; - for pfn in pfn_range { - let offset = ((pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; - - let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - let phys = unsafe { self.table.unmap_page(virt)? }; - - backing.release_page(offset, phys)?; - } - - Ok(()) - })?; - - // Drop the tables - self.table.clear(); - - Ok(()) - } - - fn clone_range( - &mut self, - source: &Self, - pfn_range: Range, - backing: &VirtualRangeBacking, - ) -> Result<(), Error> { - self.allocator - .insert(pfn_range.start, pfn_range.len(), backing.clone()) - .unwrap(); - - match backing { - VirtualRangeBacking::Anonymous => { - let start = pfn_range.start * ProcessAddressSpaceImpl::PAGE_SIZE; - let end = pfn_range.end * ProcessAddressSpaceImpl::PAGE_SIZE; - debugln!("fork: clone_range({:#x?})", start..end); - // Clone the data - for i in pfn_range { - let address = i * ProcessAddressSpaceImpl::PAGE_SIZE; - let (src_page, attrs) = source.table.translate(address).unwrap(); - let dst_page = clone_page(src_page)?; - unsafe { - self.table.map_page(address, dst_page, attrs).unwrap(); - } - } - Ok(()) - } - VirtualRangeBacking::File(_f) => { - todo!(); - } - } - } -} - -fn clone_page(src_phys: PhysicalAddress) -> Result { - let dst_page = phys::alloc_page()?; - let src_map = unsafe { PhysicalRef::<[u8; 4096]>::map(src_phys) }; - let mut dst_map = unsafe { PhysicalRefMut::<[u8; 4096]>::map(dst_page) }; - dst_map.copy_from_slice(src_map.as_ref()); - Ok(dst_page) -} - -impl ProcessAddressSpace { - /// Constructs a new [ProcessAddressSpace] - pub fn new() -> Result { - let table = ProcessAddressSpaceImpl::new()?; - let allocator = VirtualMemoryAllocator::new( - ProcessAddressSpaceImpl::LOWER_LIMIT_PFN, - ProcessAddressSpaceImpl::UPPER_LIMIT_PFN, - ); - Ok(Self { - inner: IrqSafeSpinlock::new(Inner { table, allocator }), - }) - } - - /// Performs a "fork" operation of the address space, cloning all the mappings into a new one - pub fn fork(&self) -> Result { - let src_inner = self.inner.lock(); - let new_table = ProcessAddressSpaceImpl::new()?; - let mut new_inner = Inner { - allocator: VirtualMemoryAllocator::new( - ProcessAddressSpaceImpl::LOWER_LIMIT_PFN, - ProcessAddressSpaceImpl::UPPER_LIMIT_PFN, - ), - table: new_table, - }; - - debugln!("fork address space!"); - - for (range, backing) in src_inner.allocator.regions() { - // If they are present in existing allocator, there should be no - // problem adding them to a new one - new_inner.clone_range(&src_inner, range, backing)?; - } - - for (range, _) in new_inner.allocator.regions() { - let start = range.start * ProcessAddressSpaceImpl::PAGE_SIZE; - let end = range.end * ProcessAddressSpaceImpl::PAGE_SIZE; - debugln!("forked region: {:#x?}", start..end); - } - - Ok(Self { - inner: IrqSafeSpinlock::new(new_inner), - }) - } - - /// Allocates a region of virtual memory within the address space and maps the pages to the - /// ones returned from `get_page` function - pub fn allocate( - &self, - _hint: Option, - size: usize, - backing: VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result { - assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - - let mut lock = self.inner.lock(); - - lock.alloc_range( - size / ProcessAddressSpaceImpl::PAGE_SIZE, - backing, - attributes, - ) - } - - /// Maps a region of memory in the address space - pub fn map( - &self, - address: usize, - size: usize, - backing: VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result<(), Error> { - assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - - let mut lock = self.inner.lock(); - - lock.map_range( - address, - size / ProcessAddressSpaceImpl::PAGE_SIZE, - backing, - attributes, - ) - } - - /// Adds a single-page mapping to the address space - pub fn map_single( - &self, - address: usize, - backing: VirtualRangeBacking, - attributes: MapAttributes, - ) -> Result { - assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - - self.inner.lock().map_single(address, backing, attributes) - } - - /// Returns the [PhysicalAddress] associated with given virtual `address`, - /// if one is mapped - pub fn translate(&self, address: usize) -> Result { - // Offset is handled at impl level - self.inner.lock().table.translate(address).map(|e| e.0) - } - - /// Removes a single PAGE_SIZE mapping from the address space. - /// - /// See [ProcessAddressSpaceManager::unmap]. - /// - /// # Safety - /// - /// The caller must ensure the process to which this address space belongs does not and - /// will not access this page. - pub unsafe fn unmap(&self, address: usize, size: usize) -> Result<(), Error> { - assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - - let mut lock = self.inner.lock(); - - lock.unmap_range(address, size / ProcessAddressSpaceImpl::PAGE_SIZE) - } - - /// Returns the physical address of this table, with ASID applied - pub fn as_address_with_asid(&self) -> u64 { - self.inner.lock().table.as_address_with_asid() - } - - /// Removes all allocations and their associated mappings from the address space - pub fn clear(&self) -> Result<(), Error> { - let mut inner = self.inner.lock(); - unsafe { inner.clear() } - } -} - -impl Drop for ProcessAddressSpace { - fn drop(&mut self) { - self.clear().ok(); - } -} diff --git a/src/mem/table.rs b/src/mem/table.rs deleted file mode 100644 index 8b53a739..00000000 --- a/src/mem/table.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Virtual memory table interface -use core::ops::{Deref, DerefMut, Range}; - -use abi::error::Error; -use bitflags::bitflags; -pub use libk_mm::table::EntryLevel; From 233a92c6e2d1158a526d513a826ac97ae4fe9311 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 7 Feb 2024 13:06:17 +0200 Subject: [PATCH 184/211] device-api: unify Irq numbers --- lib/device-api/src/interrupt.rs | 27 ++++++++++++---- src/arch/aarch64/cpu.rs | 15 +++++---- src/arch/aarch64/gic/mod.rs | 24 ++++++-------- src/arch/aarch64/mod.rs | 35 ++++++-------------- src/arch/aarch64/timer.rs | 14 +++++--- src/arch/mod.rs | 45 ++++++-------------------- src/arch/x86_64/acpi.rs | 10 +++--- src/arch/x86_64/apic/ioapic.rs | 38 +++++++++++++--------- src/arch/x86_64/apic/local.rs | 10 +++--- src/arch/x86_64/cpu.rs | 13 ++++---- src/arch/x86_64/mod.rs | 35 +++++++++----------- src/arch/x86_64/peripherals/i8253.rs | 10 ++++-- src/arch/x86_64/peripherals/ps2/mod.rs | 17 ++++------ src/arch/x86_64/peripherals/serial.rs | 11 +++---- src/device/serial/pl011.rs | 12 ++++--- src/panic.rs | 6 ++-- 16 files changed, 149 insertions(+), 173 deletions(-) diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index bee36904..c4959492 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -2,6 +2,23 @@ use yggdrasil_abi::error::Error; use crate::Device; +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Irq { + Private(u32), + External(u32), +} + +/// Describes messages sent from some CPU to others +#[derive(Clone, Copy, PartialEq, Debug)] +#[repr(u64)] +pub enum IpiMessage { + /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow + Panic, + /// Indicates that the cores should either halt and wait for the caller to shut the system + /// down, or they should shut down by themselves, depending on the platform + Shutdown, +} + #[derive(Default, Clone, Copy, PartialEq, Eq, Debug)] #[repr(u32)] pub enum IrqLevel { @@ -79,19 +96,17 @@ pub trait MessageInterruptController { } pub trait ExternalInterruptController { - type IrqNumber; - /// Performs IRQ delivery method configuration and registers a handler to execute when it is /// fired fn register_irq( &self, - irq: Self::IrqNumber, + irq: Irq, options: IrqOptions, handler: &'static dyn InterruptHandler, ) -> Result<(), Error>; /// Enables the specified IRQ (unmasks it) - fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error>; + fn enable_irq(&self, irq: Irq) -> Result<(), Error>; /// Handles a single pending interrupt on this controller. /// The function is intended for interrupt controllers which have internal registers to track @@ -105,9 +120,7 @@ pub trait ExternalInterruptController { } pub trait LocalInterruptController { - type IpiMessage; - - fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error>; + fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error>; /// Initializes the local interrupt controller for an Application Processor instance. /// diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index c20eed41..dab57df4 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -6,6 +6,7 @@ use core::{ use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; +use device_api::interrupt::IpiMessage; use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, OneTimeInit, @@ -13,7 +14,7 @@ use libk_util::{ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{CpuAccess, CpuMessage, LocalCpuAccess}, + arch::{CpuAccess, LocalCpuAccess}, panic, task::{sched::CpuQueue, thread::ThreadId}, }; @@ -42,7 +43,7 @@ pub struct Cpu { pub struct LocalCpu(&'static mut Cpu, IrqGuard); struct IpiQueue { - data: IrqSafeSpinlock>, + data: IrqSafeSpinlock>, } static IPI_QUEUES: OneTimeInit> = OneTimeInit::new(); @@ -54,14 +55,14 @@ impl IpiQueue { } } - pub fn push(&self, msg: CpuMessage) { + pub fn push(&self, msg: IpiMessage) { let mut lock = self.data.lock(); assert!(lock.is_none()); lock.replace(msg); } - pub fn pop(&self) -> Option { + pub fn pop(&self) -> Option { let mut lock = self.data.lock(); lock.take() } @@ -124,7 +125,7 @@ impl CpuAccess for Cpu { self.thread_id } - fn push_ipi(id: u32, msg: CpuMessage) { + fn push_ipi(id: u32, msg: IpiMessage) { if let Some(q) = IPI_QUEUES.try_get().and_then(|q| q.get(id as usize)) { q.push(msg); } @@ -137,8 +138,8 @@ impl CpuAccess for Cpu { .and_then(|q| q.pop()) { match ipi { - CpuMessage::Panic => panic::panic_secondary(), - CpuMessage::Shutdown => todo!(), + IpiMessage::Panic => panic::panic_secondary(), + IpiMessage::Shutdown => todo!(), } } } diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index c012fedd..bd7c9a86 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -8,8 +8,8 @@ use alloc::{boxed::Box, sync::Arc}; use device_api::{ interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, - IpiDeliveryTarget, IrqOptions, LocalInterruptController, MessageInterruptController, - MsiInfo, + IpiDeliveryTarget, IpiMessage, Irq, IrqOptions, LocalInterruptController, + MessageInterruptController, MsiInfo, }, Device, }; @@ -20,7 +20,7 @@ use libk_mm::{ }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::arch::{aarch64::IrqNumber, CpuAccess, CpuMessage, Platform}; +use crate::arch::{CpuAccess, Platform}; use self::{gicc::Gicc, gicd::Gicd}; @@ -77,11 +77,9 @@ impl Device for Gic { } impl ExternalInterruptController for Gic { - type IrqNumber = IrqNumber; - fn register_irq( &self, - irq: IrqNumber, + irq: Irq, options: IrqOptions, handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { @@ -89,8 +87,8 @@ impl ExternalInterruptController for Gic { let gicd = self.gicd.get(); let index = match irq { - IrqNumber::Shared(i) => i + 32, - IrqNumber::Private(i) => i + 16, + Irq::External(i) => i + 32, + Irq::Private(i) => i + 16, } as usize; debugln!( @@ -108,11 +106,11 @@ impl ExternalInterruptController for Gic { Ok(()) } - fn enable_irq(&self, irq: IrqNumber) -> Result<(), Error> { + fn enable_irq(&self, irq: Irq) -> Result<(), Error> { let gicd = self.gicd.get(); let index = match irq { - IrqNumber::Shared(i) => i + 32, - IrqNumber::Private(i) => i + 16, + Irq::External(i) => i + 32, + Irq::Private(i) => i + 16, } as usize; gicd.enable_irq(index); Ok(()) @@ -161,9 +159,7 @@ impl MessageInterruptController for Gic { } impl LocalInterruptController for Gic { - type IpiMessage = CpuMessage; - - fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> { + fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error> { // TODO message queue insertion should be moved match target { IpiDeliveryTarget::OtherCpus => { diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 3d8ec624..97d67f28 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -6,7 +6,7 @@ use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use device_api::{ interrupt::{ - ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController, + ExternalInterruptController, IpiDeliveryTarget, IpiMessage, Irq, LocalInterruptController, MessageInterruptController, }, timer::MonotonicTimestampProviderDevice, @@ -37,7 +37,7 @@ use crate::{ mem::heap, }; -use super::{CpuMessage, Platform}; +use super::Platform; pub mod boot; pub mod context; @@ -63,8 +63,8 @@ pub struct AArch64 { pub psci: OneTimeInit<&'static Psci>, reset: OneTimeInit<&'static dyn ResetDevice>, - lintc: OneTimeInit<&'static dyn LocalInterruptController>, - xintc: OneTimeInit<&'static dyn ExternalInterruptController>, + lintc: OneTimeInit<&'static dyn LocalInterruptController>, + xintc: OneTimeInit<&'static dyn ExternalInterruptController>, msi_intc: OneTimeInit<&'static dyn MessageInterruptController>, mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, @@ -75,8 +75,6 @@ pub struct AArch64 { impl Platform for AArch64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; - type IrqNumber = IrqNumber; - type L3 = L3; unsafe fn start_application_processors(&self) { @@ -90,15 +88,11 @@ impl Platform for AArch64 { smp::CPU_COUNT.load(Ordering::Acquire) } - fn local_interrupt_controller( - &'static self, - ) -> &'static dyn LocalInterruptController { + fn local_interrupt_controller(&'static self) -> &'static dyn LocalInterruptController { *self.lintc.get() } - fn external_interrupt_controller( - &'static self, - ) -> &'static dyn ExternalInterruptController { + fn external_interrupt_controller(&'static self) -> &'static dyn ExternalInterruptController { *self.xintc.get() } @@ -108,7 +102,7 @@ impl Platform for AArch64 { fn register_local_interrupt_controller( &self, - intc: &'static dyn LocalInterruptController, + intc: &'static dyn LocalInterruptController, ) -> Result<(), Error> { self.lintc.init(intc); Ok(()) @@ -116,7 +110,7 @@ impl Platform for AArch64 { fn register_external_interrupt_controller( &self, - intc: &'static dyn ExternalInterruptController, + intc: &'static dyn ExternalInterruptController, ) -> Result<(), Error> { self.xintc.init(intc); Ok(()) @@ -142,7 +136,7 @@ impl Platform for AArch64 { Ok(()) } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error> { if let Some(local_intc) = self.lintc.try_get() { local_intc.send_ipi(target, msg) } else { @@ -392,7 +386,7 @@ impl AArch64 { // TODO device-tree initialization for this CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); CNTP_TVAL_EL0.set(10000000); - self.xintc.get().enable_irq(IrqNumber::Private(14)).unwrap(); + self.xintc.get().enable_irq(Irq::Private(14)).unwrap(); } Ok(()) @@ -413,12 +407,3 @@ pub static PLATFORM: AArch64 = AArch64 { mtimer: OneTimeInit::new(), }; - -/// AArch64-specific interrupt number -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum IrqNumber { - /// CPU-private interrupt - Private(u32), - /// Shared interrupt - Shared(u32), -} diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index d06113d4..a22a10bd 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -5,18 +5,22 @@ use core::time::Duration; use aarch64_cpu::registers::{CNTFRQ_EL0, CNTPCT_EL0, CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use alloc::boxed::Box; -use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; +use device_api::{ + interrupt::{InterruptHandler, Irq}, + timer::MonotonicTimestampProviderDevice, + Device, +}; use device_tree::device_tree_driver; use libk::runtime; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use crate::arch::{aarch64::IrqNumber, CpuAccess, Platform, PLATFORM}; +use crate::arch::{CpuAccess, Platform, PLATFORM}; use super::cpu::Cpu; /// ARM Generic Timer driver pub struct ArmTimer { - irq: IrqNumber, + irq: Irq, } /// ARM timer tick interval (in some time units?) @@ -107,7 +111,7 @@ impl ArmTimer { /// # Safety /// /// The caller must ensure the function has not been called before. - pub const unsafe fn new(irq: IrqNumber) -> Self { + pub const unsafe fn new(irq: Irq) -> Self { Self { irq } } } @@ -116,6 +120,6 @@ device_tree_driver! { compatible: ["arm,armv8-timer"], probe(_dt) => { // TODO actually get info from the dt - Some(Box::new(unsafe { ArmTimer::new(IrqNumber::Private(14)) })) + Some(Box::new(unsafe { ArmTimer::new(Irq::Private(14)) })) } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 436c3f1d..e0678e34 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -7,8 +7,8 @@ use abi::error::Error; use cfg_if::cfg_if; use device_api::{ interrupt::{ - ExternalInterruptController, InterruptHandler, IpiDeliveryTarget, IrqOptions, - LocalInterruptController, MessageInterruptController, + ExternalInterruptController, InterruptHandler, IpiDeliveryTarget, IpiMessage, Irq, + IrqOptions, LocalInterruptController, MessageInterruptController, }, timer::MonotonicTimestampProviderDevice, ResetDevice, @@ -38,26 +38,12 @@ pub type L3 = ::L3; // Architecture interfaces -/// Describes messages sent from some CPU to others -#[derive(Clone, Copy, PartialEq, Debug)] -#[repr(u64)] -pub enum CpuMessage { - /// Indicates that the sender CPU entered kernel panic and wants other CPUs to follow - Panic, - /// Indicates that the cores should either halt and wait for the caller to shut the system - /// down, or they should shut down by themselves, depending on the platform - Shutdown, -} - /// Interface for an architecture-specific facilities #[allow(unused)] pub trait Platform { /// Address, to which "zero" address is mapped in the virtual address space const KERNEL_VIRT_OFFSET: usize; - /// IRQ number type associated with the architecture - type IrqNumber; - /// Lowest page entry level, usually 4KiB pages type L3: EntryLevel; @@ -76,7 +62,7 @@ pub trait Platform { /// Adds an external interrupt controller to the system fn register_external_interrupt_controller( &self, - intc: &'static dyn ExternalInterruptController, + intc: &'static dyn ExternalInterruptController, ) -> Result<(), Error> { Err(Error::NotImplemented) } @@ -84,7 +70,7 @@ pub trait Platform { /// Adds a local interrupt controller to the system fn register_local_interrupt_controller( &self, - intc: &'static dyn LocalInterruptController, + intc: &'static dyn LocalInterruptController, ) -> Result<(), Error> { Err(Error::NotImplemented) } @@ -112,16 +98,12 @@ pub trait Platform { // TODO only supports 1 extintc per system /// Returns the primary external interrupt controller - fn external_interrupt_controller( - &'static self, - ) -> &'static dyn ExternalInterruptController { + fn external_interrupt_controller(&'static self) -> &'static dyn ExternalInterruptController { unimplemented!() } /// Returns the local interrupt controller - fn local_interrupt_controller( - &'static self, - ) -> &'static dyn LocalInterruptController { + fn local_interrupt_controller(&'static self) -> &'static dyn LocalInterruptController { unimplemented!() } @@ -145,7 +127,7 @@ pub trait Platform { /// # Safety /// /// As the call may alter the flow of execution on CPUs, this function is unsafe. - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { + unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error> { Ok(()) } @@ -217,7 +199,7 @@ pub trait CpuAccess: Sized { unsafe fn set_current_thread_id(&mut self, id: Option); /// Push an IPI message to the CPU's queue - fn push_ipi(id: u32, msg: CpuMessage) { + fn push_ipi(id: u32, msg: IpiMessage) { let _ = id; let _ = msg; // Not implemented by default @@ -259,16 +241,7 @@ fn __register_global_interrupt( ) -> Result<(), Error> { let intc = PLATFORM.external_interrupt_controller(); - let irq = { - #[cfg(target_arch = "aarch64")] - { - aarch64::IrqNumber::Shared(irq) - } - #[cfg(target_arch = "x86_64")] - { - x86_64::IrqNumber::Gsi(irq.try_into().unwrap()) - } - }; + let irq = Irq::External(irq); intc.register_irq(irq, options, handler)?; intc.enable_irq(irq)?; diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 0eee7955..885d3181 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -12,7 +12,7 @@ use acpi_system::{ }; use alloc::boxed::Box; use device_api::{ - interrupt::{InterruptHandler, IpiDeliveryTarget}, + interrupt::{InterruptHandler, IpiDeliveryTarget, IpiMessage, Irq}, Device, }; use libk_mm::{ @@ -24,8 +24,8 @@ use yggdrasil_abi::error::Error; use crate::{ arch::{ - x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE}, - CpuMessage, Platform, PLATFORM, + x86_64::{apic::ioapic::ISA_IRQ_OFFSET, smp::CPU_COUNT, SHUTDOWN_FENCE}, + Platform, PLATFORM, }, mem::{heap::GLOBAL_HEAP, read_memory, write_memory}, }; @@ -171,7 +171,7 @@ impl acpi_system::Handler for AcpiHandlerImpl { let intc = PLATFORM.external_interrupt_controller(); let handler = Box::leak(Box::new(SciHandler)); - let irq = IrqNumber::Isa(irq as _); + let irq = Irq::External(irq + ISA_IRQ_OFFSET); intc.register_irq(irq, Default::default(), handler).unwrap(); intc.enable_irq(irq).unwrap(); @@ -361,7 +361,7 @@ pub fn init_acpi(tables: &'static AcpiTables) -> Result<(), Err unsafe { PLATFORM - .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Shutdown) + .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Shutdown) .unwrap(); } diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index f68ff4b8..1861a0cc 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -3,7 +3,7 @@ use abi::error::Error; use acpi_lib::platform::interrupt::{Apic as AcpiApic, Polarity, TriggerMode}; use device_api::{ interrupt::{ - ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, + ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, Irq, IrqLevel, IrqOptions, IrqTrigger, }, Device, @@ -19,10 +19,12 @@ use tock_registers::{ registers::{ReadWrite, WriteOnly}, }; -use crate::arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber}; +use crate::arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID}; use super::{APIC_EXTERNAL_OFFSET, POPULATED_EXTERNAL_VECTORS}; +pub const ISA_IRQ_OFFSET: u32 = 1024; + // IRQ 0 is timer, IRQ 1 reserved (for now?), +32 offset for exception entries const IO_APIC_VECTOR_OFFSET: u32 = 32 + APIC_EXTERNAL_OFFSET; @@ -164,11 +166,9 @@ impl Device for IoApic { } impl ExternalInterruptController for IoApic { - type IrqNumber = IrqNumber; - fn register_irq( &self, - irq: Self::IrqNumber, + irq: Irq, options: IrqOptions, handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { @@ -187,8 +187,9 @@ impl ExternalInterruptController for IoApic { ); let (gsi, level, trigger) = match irq { - IrqNumber::Isa(irq) => { - if let Some(redir) = self.isa_redirections[irq as usize].as_ref() { + Irq::External(irq) if irq >= ISA_IRQ_OFFSET => { + if let Some(redir) = self.isa_redirections[(irq - ISA_IRQ_OFFSET) as usize].as_ref() + { // Mapped to a (possibly different) GSI, but also with possibly different options ( redir.gsi_index, @@ -197,10 +198,15 @@ impl ExternalInterruptController for IoApic { ) } else { // Directly mapped to a GSI - (irq as u32, options.level, options.trigger) + ( + (irq - ISA_IRQ_OFFSET) as u32, + options.level, + options.trigger, + ) } } - IrqNumber::Gsi(irq) => (irq as u32, options.level, options.trigger), + Irq::External(irq) => (irq as u32, options.level, options.trigger), + Irq::Private(_) => unimplemented!(), }; inner.configure_gsi(gsi, level, trigger); @@ -209,7 +215,7 @@ impl ExternalInterruptController for IoApic { Ok(()) } - fn enable_irq(&self, irq: Self::IrqNumber) -> Result<(), Error> { + fn enable_irq(&self, irq: Irq) -> Result<(), Error> { let mut inner = self.inner.lock(); let gsi = self.translate_irq(irq); inner.set_gsi_enabled(gsi, true); @@ -302,13 +308,15 @@ impl IoApic { }) } - fn translate_irq(&self, irq: IrqNumber) -> u32 { + fn translate_irq(&self, irq: Irq) -> u32 { let redir = &self.isa_redirections; match irq { - IrqNumber::Isa(isa) => redir[isa as usize] - .map(|t| t.gsi_index) - .unwrap_or(isa as u32), - IrqNumber::Gsi(gsi) => gsi as _, + Irq::External(irq) if irq >= ISA_IRQ_OFFSET => { + let isa = irq - ISA_IRQ_OFFSET; + redir[isa as usize].map(|t| t.gsi_index).unwrap_or(isa) + } + Irq::External(irq) => irq, + Irq::Private(_) => unreachable!(), } } } diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 9c2d0873..3909ec71 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -5,8 +5,8 @@ use abi::error::Error; use alloc::{vec, vec::Vec}; use device_api::{ interrupt::{ - InterruptAffinity, InterruptHandler, IpiDeliveryTarget, LocalInterruptController, - MessageInterruptController, MsiInfo, + InterruptAffinity, InterruptHandler, IpiDeliveryTarget, IpiMessage, + LocalInterruptController, MessageInterruptController, MsiInfo, }, Device, }; @@ -29,7 +29,7 @@ use tock_registers::{ use crate::{ arch::{ x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}, - CpuAccess, CpuMessage, + CpuAccess, }, task::Cpu, }; @@ -219,9 +219,7 @@ impl MessageInterruptController for LocalApic { } impl LocalInterruptController for LocalApic { - type IpiMessage = CpuMessage; - - fn send_ipi(&self, target: IpiDeliveryTarget, msg: Self::IpiMessage) -> Result<(), Error> { + fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error> { while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { core::hint::spin_loop(); } diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index be67d842..8419ca23 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -6,6 +6,7 @@ use core::{ }; use alloc::{boxed::Box, vec::Vec}; +use device_api::interrupt::IpiMessage; use kernel_arch_x86_64::registers::MSR_IA32_KERNEL_GS_BASE; use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, @@ -16,7 +17,7 @@ use tock_registers::interfaces::Writeable; use crate::{ arch::{ x86_64::{cpuid, gdt, syscall}, - CpuAccess, CpuMessage, LocalCpuAccess, + CpuAccess, LocalCpuAccess, }, task::{sched::CpuQueue, thread::ThreadId}, }; @@ -49,7 +50,7 @@ pub struct Cpu { pub struct LocalCpu(&'static mut Cpu, IrqGuard); struct IpiQueue { - data: IrqSafeSpinlock>, + data: IrqSafeSpinlock>, } static IPI_QUEUES: OneTimeInit> = OneTimeInit::new(); @@ -61,14 +62,14 @@ impl IpiQueue { } } - pub fn push(&self, msg: CpuMessage) { + pub fn push(&self, msg: IpiMessage) { let mut lock = self.data.lock(); assert!(lock.is_none()); lock.replace(msg); } - pub fn pop(&self) -> Option { + pub fn pop(&self) -> Option { let mut lock = self.data.lock(); lock.take() } @@ -115,7 +116,7 @@ impl Cpu { } /// Inserts an IPI message to the back of the target CPU's message queue - pub fn push_ipi_queue(cpu_id: u32, msg: CpuMessage) { + pub fn push_ipi_queue(cpu_id: u32, msg: IpiMessage) { let ipi_queue = &IPI_QUEUES.get()[cpu_id as usize]; ipi_queue.push(msg); } @@ -125,7 +126,7 @@ impl Cpu { /// # Note /// /// Currently the queue consists of only one entry, so the CPU will only receive the last one. - pub fn get_ipi(&self) -> Option { + pub fn get_ipi(&self) -> Option { let ipi_queue = &IPI_QUEUES.get()[self.id as usize]; ipi_queue.pop() } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 8ccf8dc5..619f08af 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -5,7 +5,7 @@ use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use device_api::{ - interrupt::{ExternalInterruptController, MessageInterruptController}, + interrupt::{ExternalInterruptController, Irq, MessageInterruptController}, timer::MonotonicTimestampProviderDevice, Device, }; @@ -39,7 +39,10 @@ mod smp; mod syscall; use crate::{ - arch::x86_64::intrinsics::{IoPort, IoPortAccess}, + arch::x86_64::{ + apic::ioapic::ISA_IRQ_OFFSET, + intrinsics::{IoPort, IoPortAccess}, + }, debug::{self, LogLevel}, device::{ self, @@ -59,16 +62,7 @@ use self::{ smp::CPU_COUNT, }; -use super::{CpuAccess, CpuMessage, Platform}; - -/// x86-64-specific interrupt number -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum IrqNumber { - /// Legacy (ISA) interrupt - Isa(u8), - /// Global System Interrupt - Gsi(u8), -} +use super::{CpuAccess, IpiMessage, Platform}; /// x86-64 architecture implementation pub struct X86_64 { @@ -99,7 +93,6 @@ pub static PLATFORM: X86_64 = X86_64 { impl Platform for X86_64 { const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; - type IrqNumber = IrqNumber; type L3 = kernel_arch_x86_64::mem::table::L3; unsafe fn start_application_processors(&self) { @@ -120,9 +113,7 @@ impl Platform for X86_64 { CPU_COUNT.load(Ordering::Acquire) } - fn external_interrupt_controller( - &'static self, - ) -> &'static dyn ExternalInterruptController { + fn external_interrupt_controller(&'static self) -> &'static dyn ExternalInterruptController { self.ioapic.get() } @@ -136,7 +127,7 @@ impl Platform for X86_64 { } impl X86_64 { - unsafe fn handle_ipi(&self, _msg: CpuMessage) { + unsafe fn handle_ipi(&self, _msg: IpiMessage) { warnln!("Received an IPI"); todo!(); } @@ -345,7 +336,11 @@ impl X86_64 { self.timer.init(I8253::new()); - let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)))); + let com1_3 = Box::leak(Box::new(ComPort::new( + 0x3F8, + 0x3E8, + Irq::External(ISA_IRQ_OFFSET + 4), + ))); debug::add_sink(com1_3.port_a(), LogLevel::Debug); self.init_framebuffer()?; @@ -359,8 +354,8 @@ impl X86_64 { ); let ps2 = Box::leak(Box::new(PS2Controller::new( - IrqNumber::Isa(1), - IrqNumber::Isa(12), + Irq::External(ISA_IRQ_OFFSET + 1), + Irq::External(ISA_IRQ_OFFSET + 12), 0x64, 0x60, ))); diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index dbe63084..f2bbb3e1 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -1,14 +1,18 @@ use core::time::Duration; use abi::error::Error; -use device_api::{interrupt::InterruptHandler, timer::MonotonicTimestampProviderDevice, Device}; +use device_api::{ + interrupt::{InterruptHandler, Irq}, + timer::MonotonicTimestampProviderDevice, + Device, +}; use libk::runtime; use libk_util::sync::IrqSafeSpinlock; use crate::arch::{ x86_64::{ + apic::ioapic::ISA_IRQ_OFFSET, intrinsics::{IoPort, IoPortAccess}, - IrqNumber, }, Platform, PLATFORM, }; @@ -71,7 +75,7 @@ impl Device for I8253 { inner.ch0_data.write(div as u8); inner.ch0_data.write((div >> 8) as u8); - let irq = IrqNumber::Isa(0); + let irq = Irq::External(ISA_IRQ_OFFSET); intc.register_irq(irq, Default::default(), self)?; intc.enable_irq(irq)?; diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index c78be67f..e00168b0 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -3,7 +3,10 @@ use abi::{ error::Error, io::{KeyboardKey, KeyboardKeyEvent}, }; -use device_api::{interrupt::InterruptHandler, Device}; +use device_api::{ + interrupt::{InterruptHandler, Irq}, + Device, +}; use libk_util::sync::IrqSafeSpinlock; use crate::{ @@ -11,7 +14,6 @@ use crate::{ x86_64::{ intrinsics::{IoPort, IoPortAccess}, peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, - IrqNumber, }, Platform, PLATFORM, }, @@ -27,9 +29,9 @@ struct Inner { /// PS/2 controller driver pub struct PS2Controller { - primary_irq: IrqNumber, + primary_irq: Irq, #[allow(unused)] - auxiliary_irq: IrqNumber, + auxiliary_irq: Irq, inner: IrqSafeSpinlock, } @@ -145,12 +147,7 @@ impl Device for PS2Controller { impl PS2Controller { /// Constructs a new instance of the device - pub const fn new( - primary_irq: IrqNumber, - auxiliary_irq: IrqNumber, - cmd_port: u16, - data_port: u16, - ) -> Self { + pub const fn new(primary_irq: Irq, auxiliary_irq: Irq, cmd_port: u16, data_port: u16) -> Self { let inner = Inner { command: IoPort::new(cmd_port), data: IoPort::new(data_port), diff --git a/src/arch/x86_64/peripherals/serial.rs b/src/arch/x86_64/peripherals/serial.rs index 8d6de2f3..9bd079ef 100644 --- a/src/arch/x86_64/peripherals/serial.rs +++ b/src/arch/x86_64/peripherals/serial.rs @@ -1,13 +1,10 @@ //! Driver for x86 COM ports use abi::error::Error; -use device_api::{serial::SerialDevice, Device}; +use device_api::{interrupt::Irq, serial::SerialDevice, Device}; use libk_util::sync::IrqSafeSpinlock; use crate::{ - arch::x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - IrqNumber, - }, + arch::x86_64::intrinsics::{IoPort, IoPortAccess}, debug::DebugSink, }; @@ -27,7 +24,7 @@ pub struct Port { pub struct ComPort { port_a: Port, port_b: Port, - irq: IrqNumber, + irq: Irq, } impl DebugSink for Port { @@ -78,7 +75,7 @@ impl Port { impl ComPort { /// Constructs a COM port pair - pub const fn new(port_a: u16, port_b: u16, irq: IrqNumber) -> Self { + pub const fn new(port_a: u16, port_b: u16, irq: Irq) -> Self { Self { port_a: Port::new(port_a), port_b: Port::new(port_b), diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index fb6e0133..23a351da 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -1,7 +1,11 @@ //! ARM PL011 driver use abi::{error::Error, io::DeviceRequest}; use alloc::boxed::Box; -use device_api::{interrupt::InterruptHandler, serial::SerialDevice, Device}; +use device_api::{ + interrupt::{InterruptHandler, Irq}, + serial::SerialDevice, + Device, +}; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use futures_util::task::{Context, Poll}; use kernel_fs::devfs::{self, CharDeviceType}; @@ -19,7 +23,7 @@ use tock_registers::{ use vfs::{CharDevice, FileReadiness}; use crate::{ - arch::{aarch64::IrqNumber, Platform, PLATFORM}, + arch::{Platform, PLATFORM}, debug::{self, DebugSink, LogLevel}, device::tty::{TtyContext, TtyDevice}, task::process::ProcessId, @@ -71,7 +75,7 @@ struct Pl011Inner { pub struct Pl011 { inner: OneTimeInit>, base: PhysicalAddress, - irq: IrqNumber, + irq: Irq, context: TtyContext, } @@ -240,7 +244,7 @@ device_tree_driver! { Some(Box::new(Pl011 { inner: OneTimeInit::new(), // TODO obtain IRQ from dt - irq: IrqNumber::Shared(1), + irq: Irq::External(1), context: TtyContext::new(), base: PhysicalAddress::from_raw(base) })) diff --git a/src/panic.rs b/src/panic.rs index 35a63aa2..796fc581 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -1,12 +1,12 @@ //! Kernel panic handler code use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use device_api::interrupt::IpiDeliveryTarget; +use device_api::interrupt::{IpiDeliveryTarget, IpiMessage}; use kernel_arch::{Architecture, ArchitectureImpl}; use libk_util::sync::{hack_locks, SpinFence}; use crate::{ - arch::{CpuAccess, CpuMessage, Platform, PlatformImpl, PLATFORM}, + arch::{CpuAccess, Platform, PlatformImpl, PLATFORM}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, task::Cpu, @@ -55,7 +55,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { // Let other CPUs know we're screwed unsafe { PLATFORM - .send_ipi(IpiDeliveryTarget::OtherCpus, CpuMessage::Panic) + .send_ipi(IpiDeliveryTarget::OtherCpus, IpiMessage::Panic) .ok(); } From 5e3be6d559dea685b3615d8b1815571fcd562a02 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 8 Feb 2024 13:11:29 +0200 Subject: [PATCH 185/211] libk: move interrupt controllers to libk --- driver/bus/pci/src/device.rs | 2 +- libk/src/api.rs | 8 --- libk/src/device.rs | 49 ++++++++++++++++ libk/src/lib.rs | 16 +----- src/arch/aarch64/exception.rs | 8 +-- src/arch/aarch64/gic/mod.rs | 14 +++-- src/arch/aarch64/mod.rs | 70 +++++----------------- src/arch/aarch64/timer.rs | 4 +- src/arch/mod.rs | 80 ++++++-------------------- src/arch/x86_64/acpi.rs | 3 +- src/arch/x86_64/apic/mod.rs | 16 ++---- src/arch/x86_64/mod.rs | 24 ++------ src/arch/x86_64/peripherals/i8253.rs | 13 ++--- src/arch/x86_64/peripherals/ps2/mod.rs | 12 ++-- src/device/serial/pl011.rs | 5 +- 15 files changed, 119 insertions(+), 205 deletions(-) create mode 100644 libk/src/device.rs diff --git a/driver/bus/pci/src/device.rs b/driver/bus/pci/src/device.rs index c2ddcda0..748da52c 100644 --- a/driver/bus/pci/src/device.rs +++ b/driver/bus/pci/src/device.rs @@ -5,7 +5,7 @@ use device_api::{ interrupt::{InterruptAffinity, InterruptHandler, IrqOptions, MsiInfo}, Device, }; -use libk::{message_interrupt_controller, register_global_interrupt}; +use libk::device::{message_interrupt_controller, register_global_interrupt}; use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; use yggdrasil_abi::error::Error; diff --git a/libk/src/api.rs b/libk/src/api.rs index 98286166..c3fc5add 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -1,7 +1,6 @@ use core::time::Duration; use alloc::{string::String, sync::Arc}; -use device_api::interrupt::{InterruptHandler, IrqOptions, MessageInterruptController}; use yggdrasil_abi::{ error::Error, process::{ExitCode, Signal}, @@ -28,12 +27,5 @@ extern "Rust" { pub fn __monotonic_timestamp() -> Result; - pub fn __message_interrupt_controller() -> &'static dyn MessageInterruptController; - pub fn __register_global_interrupt( - irq: u32, - options: IrqOptions, - handler: &'static dyn InterruptHandler, - ) -> Result<(), Error>; - pub fn __signal_process_group(group_id: u32, signal: Signal); } diff --git a/libk/src/device.rs b/libk/src/device.rs new file mode 100644 index 00000000..da83630c --- /dev/null +++ b/libk/src/device.rs @@ -0,0 +1,49 @@ +use device_api::interrupt::{ + ExternalInterruptController, InterruptHandler, IrqOptions, LocalInterruptController, + MessageInterruptController, +}; +use libk_util::OneTimeInit; +use yggdrasil_abi::error::Error; + +macro_rules! register_get { + ($register_name:ident, $get_name:ident, $global:ident, $ty:ty) => { + static $global: OneTimeInit<$ty> = OneTimeInit::new(); + + pub fn $register_name(intc: $ty) { + $global.init(intc); + } + + pub fn $get_name() -> $ty { + *$global.get() + } + }; +} + +register_get!( + register_external_interrupt_controller, + external_interrupt_controller, + EXTERNAL_INTC, + &'static dyn ExternalInterruptController +); +register_get!( + register_local_interrupt_controller, + local_interrupt_controller, + LOCAL_INTC, + &'static dyn LocalInterruptController +); +register_get!( + register_message_interrupt_controller, + message_interrupt_controller, + MESSAGE_INTC, + &'static dyn MessageInterruptController +); + +#[inline] +pub fn register_global_interrupt( + _irq: u32, + _options: IrqOptions, + _handler: &'static dyn InterruptHandler, +) -> Result<(), Error> { + // XXX + todo!() +} diff --git a/libk/src/lib.rs b/libk/src/lib.rs index b69ce580..ee9ebf5c 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -17,31 +17,17 @@ use core::time::Duration; -use device_api::interrupt::{InterruptHandler, IrqOptions, MessageInterruptController}; use yggdrasil_abi::{error::Error, process::Signal}; extern crate alloc; pub(crate) mod api; +pub mod device; pub mod runtime; pub mod sync; pub mod thread; -#[inline] -pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController { - unsafe { api::__message_interrupt_controller() } -} - -#[inline] -pub fn register_global_interrupt( - irq: u32, - options: IrqOptions, - handler: &'static dyn InterruptHandler, -) -> Result<(), Error> { - unsafe { api::__register_global_interrupt(irq, options, handler) } -} - #[inline] pub fn cpu_index() -> usize { unsafe { api::__cpu_index() } diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index e4ab320d..258e7dfa 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -14,17 +14,15 @@ use abi::{ syscall::SyscallFunction, }; use kernel_arch::{Architecture, ArchitectureImpl}; +use libk::device::external_interrupt_controller; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::Platform, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, thread::Thread}, }; -use super::PLATFORM; - /// Struct for register values saved when taking an exception #[repr(C)] pub struct ExceptionFrame { @@ -352,9 +350,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { } fn irq_common() { - PLATFORM - .external_interrupt_controller() - .handle_pending_irqs(); + external_interrupt_controller().handle_pending_irqs(); } unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) { diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index bd7c9a86..3a842836 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -14,17 +14,21 @@ use device_api::{ Device, }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; +use libk::device::{ + register_external_interrupt_controller, register_local_interrupt_controller, + register_message_interrupt_controller, +}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, RawDeviceMemoryMapping}, }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::arch::{CpuAccess, Platform}; +use crate::arch::CpuAccess; use self::{gicc::Gicc, gicd::Gicd}; -use super::{cpu::Cpu, smp::CPU_COUNT, PLATFORM}; +use super::{cpu::Cpu, smp::CPU_COUNT}; const MAX_IRQ: usize = 300; const IPI_VECTOR: u64 = 1; @@ -68,9 +72,9 @@ impl Device for Gic { self.gicd.init(gicd); self.gicc.init(gicc); - PLATFORM.register_external_interrupt_controller(self)?; - PLATFORM.register_local_interrupt_controller(self)?; - PLATFORM.register_message_interrupt_controller(self)?; + register_external_interrupt_controller(self); + register_local_interrupt_controller(self); + register_message_interrupt_controller(self); Ok(()) } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 97d67f28..5cd1b60a 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -5,10 +5,7 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use device_api::{ - interrupt::{ - ExternalInterruptController, IpiDeliveryTarget, IpiMessage, Irq, LocalInterruptController, - MessageInterruptController, - }, + interrupt::{IpiDeliveryTarget, IpiMessage, Irq}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; @@ -18,6 +15,7 @@ use kernel_arch_aarch64::mem::{ table::{L1, L2, L3}, EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, }; +use libk::device::{external_interrupt_controller, local_interrupt_controller}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, @@ -63,10 +61,6 @@ pub struct AArch64 { pub psci: OneTimeInit<&'static Psci>, reset: OneTimeInit<&'static dyn ResetDevice>, - lintc: OneTimeInit<&'static dyn LocalInterruptController>, - xintc: OneTimeInit<&'static dyn ExternalInterruptController>, - msi_intc: OneTimeInit<&'static dyn MessageInterruptController>, - mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, initrd: OneTimeInit>, @@ -88,42 +82,6 @@ impl Platform for AArch64 { smp::CPU_COUNT.load(Ordering::Acquire) } - fn local_interrupt_controller(&'static self) -> &'static dyn LocalInterruptController { - *self.lintc.get() - } - - fn external_interrupt_controller(&'static self) -> &'static dyn ExternalInterruptController { - *self.xintc.get() - } - - fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { - *self.msi_intc.get() - } - - fn register_local_interrupt_controller( - &self, - intc: &'static dyn LocalInterruptController, - ) -> Result<(), Error> { - self.lintc.init(intc); - Ok(()) - } - - fn register_external_interrupt_controller( - &self, - intc: &'static dyn ExternalInterruptController, - ) -> Result<(), Error> { - self.xintc.init(intc); - Ok(()) - } - - fn register_message_interrupt_controller( - &self, - intc: &'static dyn MessageInterruptController, - ) -> Result<(), Error> { - self.msi_intc.init(intc); - Ok(()) - } - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { *self.mtimer.get() } @@ -136,12 +94,14 @@ impl Platform for AArch64 { Ok(()) } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result<(), Error> { - if let Some(local_intc) = self.lintc.try_get() { - local_intc.send_ipi(target, msg) - } else { - Ok(()) - } + unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: IpiMessage) -> Result<(), Error> { + // XXX + todo!() + // if let Some(local_intc) = self.lintc.try_get() { + // local_intc.send_ipi(target, msg) + // } else { + // Ok(()) + // } } fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { @@ -377,7 +337,7 @@ impl AArch64 { } else { // BSP already initialized everything needed // Setup timer and local interrupt controller - let intc = self.lintc.get(); + let intc = local_interrupt_controller(); unsafe { intc.init_ap().unwrap(); @@ -386,7 +346,9 @@ impl AArch64 { // TODO device-tree initialization for this CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); CNTP_TVAL_EL0.set(10000000); - self.xintc.get().enable_irq(Irq::Private(14)).unwrap(); + external_interrupt_controller() + .enable_irq(Irq::Private(14)) + .unwrap(); } Ok(()) @@ -401,9 +363,5 @@ pub static PLATFORM: AArch64 = AArch64 { psci: OneTimeInit::new(), reset: OneTimeInit::new(), - lintc: OneTimeInit::new(), - xintc: OneTimeInit::new(), - msi_intc: OneTimeInit::new(), - mtimer: OneTimeInit::new(), }; diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index a22a10bd..0623164d 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -11,7 +11,7 @@ use device_api::{ Device, }; use device_tree::device_tree_driver; -use libk::runtime; +use libk::{device::external_interrupt_controller, runtime}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::arch::{CpuAccess, Platform, PLATFORM}; @@ -62,7 +62,7 @@ impl Device for ArmTimer { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.external_interrupt_controller(); + let intc = external_interrupt_controller(); intc.register_irq(self.irq, Default::default(), self)?; diff --git a/src/arch/mod.rs b/src/arch/mod.rs index e0678e34..cd7be972 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -6,10 +6,7 @@ use abi::error::Error; use cfg_if::cfg_if; use device_api::{ - interrupt::{ - ExternalInterruptController, InterruptHandler, IpiDeliveryTarget, IpiMessage, Irq, - IrqOptions, LocalInterruptController, MessageInterruptController, - }, + interrupt::{IpiDeliveryTarget, IpiMessage}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; @@ -59,30 +56,6 @@ pub trait Platform { /// Returns the count of present CPUs, including the BSP fn cpu_count() -> usize; - /// Adds an external interrupt controller to the system - fn register_external_interrupt_controller( - &self, - intc: &'static dyn ExternalInterruptController, - ) -> Result<(), Error> { - Err(Error::NotImplemented) - } - - /// Adds a local interrupt controller to the system - fn register_local_interrupt_controller( - &self, - intc: &'static dyn LocalInterruptController, - ) -> Result<(), Error> { - Err(Error::NotImplemented) - } - - /// Adds a message-signalled interrupt (MSI/MSI-X) controller to the system - fn register_message_interrupt_controller( - &self, - intc: &'static dyn MessageInterruptController, - ) -> Result<(), Error> { - Err(Error::NotImplemented) - } - /// Adds a monotonic timer to the system fn register_monotonic_timer( &self, @@ -96,22 +69,6 @@ pub trait Platform { Err(Error::NotImplemented) } - // TODO only supports 1 extintc per system - /// Returns the primary external interrupt controller - fn external_interrupt_controller(&'static self) -> &'static dyn ExternalInterruptController { - unimplemented!() - } - - /// Returns the local interrupt controller - fn local_interrupt_controller(&'static self) -> &'static dyn LocalInterruptController { - unimplemented!() - } - - /// Returns the MSI/MSI-X-capable interrupt controller - fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { - unimplemented!() - } - /// Returns the monotonic timer fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { unimplemented!() @@ -228,23 +185,18 @@ fn __monotonic_timestamp() -> Result { PLATFORM.monotonic_timer().monotonic_timestamp() } -#[no_mangle] -fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { - PLATFORM.message_interrupt_controller() -} - -#[no_mangle] -fn __register_global_interrupt( - irq: u32, - options: IrqOptions, - handler: &'static dyn InterruptHandler, -) -> Result<(), Error> { - let intc = PLATFORM.external_interrupt_controller(); - - let irq = Irq::External(irq); - - intc.register_irq(irq, options, handler)?; - intc.enable_irq(irq)?; - - Ok(()) -} +// #[no_mangle] +// fn __register_global_interrupt( +// irq: u32, +// options: IrqOptions, +// handler: &'static dyn InterruptHandler, +// ) -> Result<(), Error> { +// let intc = PLATFORM.external_interrupt_controller(); +// +// let irq = Irq::External(irq); +// +// intc.register_irq(irq, options, handler)?; +// intc.enable_irq(irq)?; +// +// Ok(()) +// } diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 885d3181..614d057e 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -15,6 +15,7 @@ use device_api::{ interrupt::{InterruptHandler, IpiDeliveryTarget, IpiMessage, Irq}, Device, }; +use libk::device::external_interrupt_controller; use libk_mm::{ address::{FromRaw, PhysicalAddress, Virtualize}, pointer::PhysicalRef, @@ -169,7 +170,7 @@ impl acpi_system::Handler for AcpiHandlerImpl { fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> { infoln!("Installing ACPI SCI handler at IRQ #{}", irq); - let intc = PLATFORM.external_interrupt_controller(); + let intc = external_interrupt_controller(); let handler = Box::leak(Box::new(SciHandler)); let irq = Irq::External(irq + ISA_IRQ_OFFSET); diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index d4a3132c..c9aa835a 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -2,17 +2,15 @@ use core::arch::global_asm; +use libk::device::{external_interrupt_controller, message_interrupt_controller}; use static_assertions::{const_assert, const_assert_eq}; use crate::{ - arch::{x86_64::cpu::Cpu, CpuAccess, Platform}, + arch::{x86_64::cpu::Cpu, CpuAccess}, task::thread::Thread, }; -use super::{ - exception::{self, IrqFrame}, - PLATFORM, -}; +use super::exception::{self, IrqFrame}; pub mod ioapic; pub mod local; @@ -67,9 +65,7 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { let cpu = Cpu::local(); let frame = &mut *frame; - PLATFORM - .external_interrupt_controller() - .handle_specific_irq(vector); + external_interrupt_controller().handle_specific_irq(vector); cpu.local_apic().clear_interrupt(); if let Some(thread) = Thread::get_current() { @@ -85,9 +81,7 @@ unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) { let cpu = Cpu::local(); let frame = &mut *frame; - PLATFORM - .message_interrupt_controller() - .handle_msi(vector); + message_interrupt_controller().handle_msi(vector); cpu.local_apic().clear_interrupt(); if let Some(thread) = Thread::get_current() { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 619f08af..252ca74e 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -4,11 +4,7 @@ use core::{mem::size_of, ops::DerefMut, sync::atomic::Ordering}; use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; -use device_api::{ - interrupt::{ExternalInterruptController, Irq, MessageInterruptController}, - timer::MonotonicTimestampProviderDevice, - Device, -}; +use device_api::{interrupt::Irq, timer::MonotonicTimestampProviderDevice, Device}; use git_version::git_version; use kernel_arch_x86_64::mem::{ init_fixed_tables, map_heap_block, @@ -16,6 +12,7 @@ use kernel_arch_x86_64::mem::{ EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1, }; use kernel_fs::devfs; +use libk::device::register_external_interrupt_controller; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, @@ -62,7 +59,7 @@ use self::{ smp::CPU_COUNT, }; -use super::{CpuAccess, IpiMessage, Platform}; +use super::{IpiMessage, Platform}; /// x86-64 architecture implementation pub struct X86_64 { @@ -73,7 +70,6 @@ pub struct X86_64 { framebuffer: OneTimeInit, fbconsole: OneTimeInit, - ioapic: OneTimeInit, timer: OneTimeInit, } @@ -87,7 +83,6 @@ pub static PLATFORM: X86_64 = X86_64 { framebuffer: OneTimeInit::new(), fbconsole: OneTimeInit::new(), - ioapic: OneTimeInit::new(), timer: OneTimeInit::new(), }; @@ -113,14 +108,6 @@ impl Platform for X86_64 { CPU_COUNT.load(Ordering::Acquire) } - fn external_interrupt_controller(&'static self) -> &'static dyn ExternalInterruptController { - self.ioapic.get() - } - - fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController { - Cpu::local().local_apic() - } - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { self.timer.get() } @@ -369,7 +356,6 @@ impl X86_64 { // ps2.connect(self.tty.get()); ps2.init_irq()?; - device::register_device(self.ioapic.get()); device::register_device(ps2); PciBusManager::setup_bus_devices()?; @@ -406,7 +392,9 @@ impl X86_64 { panic!("The processor does not support APIC"); }; - self.ioapic.init(IoApic::from_acpi(&apic_info)?); + let ioapic = IoApic::from_acpi(&apic_info)?; + let ioapic = Box::leak(Box::new(ioapic)); + register_external_interrupt_controller(ioapic); // acpi::init_acpi(acpi).unwrap(); diff --git a/src/arch/x86_64/peripherals/i8253.rs b/src/arch/x86_64/peripherals/i8253.rs index f2bbb3e1..0b3b24cf 100644 --- a/src/arch/x86_64/peripherals/i8253.rs +++ b/src/arch/x86_64/peripherals/i8253.rs @@ -6,15 +6,12 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, Device, }; -use libk::runtime; +use libk::{device::external_interrupt_controller, runtime}; use libk_util::sync::IrqSafeSpinlock; -use crate::arch::{ - x86_64::{ - apic::ioapic::ISA_IRQ_OFFSET, - intrinsics::{IoPort, IoPortAccess}, - }, - Platform, PLATFORM, +use crate::arch::x86_64::{ + apic::ioapic::ISA_IRQ_OFFSET, + intrinsics::{IoPort, IoPortAccess}, }; const FREQUENCY: u32 = 1193180; @@ -66,7 +63,7 @@ impl Device for I8253 { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.external_interrupt_controller(); + let intc = external_interrupt_controller(); let inner = self.inner.lock(); let div: u16 = (FREQUENCY / 1000).try_into().unwrap(); diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index e00168b0..b70005a9 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -7,15 +7,13 @@ use device_api::{ interrupt::{InterruptHandler, Irq}, Device, }; +use libk::device::external_interrupt_controller; use libk_util::sync::IrqSafeSpinlock; use crate::{ - arch::{ - x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, - }, - Platform, PLATFORM, + arch::x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, }, device::input, }; @@ -123,7 +121,7 @@ impl Device for PS2Controller { unsafe fn init_irq(&'static self) -> Result<(), Error> { let mut inner = self.inner.lock(); // let intc = PLATFORM.interrupt_controller(); - let intc = PLATFORM.external_interrupt_controller(); + let intc = external_interrupt_controller(); intc.register_irq(self.primary_irq, Default::default(), self)?; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 23a351da..061d77b3 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -9,7 +9,7 @@ use device_api::{ use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use futures_util::task::{Context, Poll}; use kernel_fs::devfs::{self, CharDeviceType}; -use libk::block; +use libk::{block, device::external_interrupt_controller}; use libk_mm::{ address::{FromRaw, PhysicalAddress}, device::DeviceMemoryIo, @@ -23,7 +23,6 @@ use tock_registers::{ use vfs::{CharDevice, FileReadiness}; use crate::{ - arch::{Platform, PLATFORM}, debug::{self, DebugSink, LogLevel}, device::tty::{TtyContext, TtyDevice}, task::process::ProcessId, @@ -225,7 +224,7 @@ impl Device for Pl011 { } unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.external_interrupt_controller(); + let intc = external_interrupt_controller(); intc.register_irq(self.irq, Default::default(), self)?; self.inner.get().lock().regs.IMSC.modify(IMSC::RXIM::SET); From c22e3699ca1946480dfb025b270fe41f7e137505 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 8 Feb 2024 15:50:25 +0200 Subject: [PATCH 186/211] arch: move Cpu structs to arch crates --- arch/aarch64/Cargo.toml | 1 + arch/aarch64/src/lib.rs | 47 +++- arch/interface/Cargo.toml | 1 + arch/interface/src/cpu.rs | 151 +++++++++++++ .../src/sync => arch/interface/src}/guard.rs | 16 +- arch/interface/src/lib.rs | 32 ++- arch/interface/src/sync.rs | 155 +++++++++++++ arch/interface/src/task.rs | 3 + arch/interface/src/util.rs | 125 +++++++++++ arch/src/lib.rs | 5 +- arch/x86_64/Cargo.toml | 1 + arch/x86_64/src/lib.rs | 76 ++++++- libk-util/src/lib.rs | 124 +---------- libk-util/src/sync/mod.rs | 9 +- libk-util/src/sync/spinlock.rs | 152 ------------- libk/Cargo.toml | 1 + libk/src/device.rs | 39 ++-- src/arch/aarch64/cpu.rs | 166 -------------- src/arch/aarch64/gic/mod.rs | 25 ++- src/arch/aarch64/mod.rs | 44 ++-- src/arch/aarch64/timer.rs | 9 +- src/arch/mod.rs | 94 +------- src/arch/x86_64/apic/local.rs | 125 +++++------ src/arch/x86_64/apic/mod.rs | 7 +- src/arch/x86_64/cpu.rs | 209 ------------------ src/arch/x86_64/exception.rs | 2 +- src/arch/x86_64/mod.rs | 41 +++- src/arch/x86_64/smp.rs | 5 +- src/debug.rs | 2 +- src/main.rs | 2 +- src/panic.rs | 2 +- src/task/context.rs | 4 +- src/task/mod.rs | 106 ++++++++- src/task/sched.rs | 10 +- src/task/thread.rs | 11 +- 35 files changed, 900 insertions(+), 902 deletions(-) create mode 100644 arch/interface/src/cpu.rs rename {libk-util/src/sync => arch/interface/src}/guard.rs (50%) create mode 100644 arch/interface/src/sync.rs create mode 100644 arch/interface/src/task.rs create mode 100644 arch/interface/src/util.rs delete mode 100644 src/arch/aarch64/cpu.rs delete mode 100644 src/arch/x86_64/cpu.rs diff --git a/arch/aarch64/Cargo.toml b/arch/aarch64/Cargo.toml index 6e5a7d95..882600f0 100644 --- a/arch/aarch64/Cargo.toml +++ b/arch/aarch64/Cargo.toml @@ -10,6 +10,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../interface" } libk-mm-interface = { path = "../../libk-mm/interface" } memtables = { path = "../../lib/memtables" } +device-api = { path = "../../lib/device-api", features = ["derive"] } bitflags = "2.3.3" static_assertions = "1.1.0" diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index 7ce6f156..76f14381 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -1,9 +1,13 @@ #![no_std] #![feature(effects, strict_provenance)] -use aarch64_cpu::registers::DAIF; -use kernel_arch_interface::Architecture; -use tock_registers::interfaces::{ReadWriteable, Readable}; +extern crate alloc; + +use aarch64_cpu::registers::{DAIF, TPIDR_EL1}; +use alloc::vec::Vec; +use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; +use kernel_arch_interface::{cpu::IpiQueue, util::OneTimeInit, Architecture}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; pub mod mem; @@ -11,7 +15,23 @@ pub use mem::{process::ProcessAddressSpaceImpl, KernelTableManagerImpl}; pub struct ArchitectureImpl; +pub trait GicInterface: LocalInterruptController {} + +pub struct PerCpuData { + pub gic: OneTimeInit<&'static dyn GicInterface>, +} + +static IPI_QUEUES: OneTimeInit>> = OneTimeInit::new(); + +impl ArchitectureImpl { + pub fn local_cpu_data() -> Option<&'static mut PerCpuData> { + unsafe { (Self::local_cpu() as *mut PerCpuData).as_mut() } + } +} + impl Architecture for ArchitectureImpl { + type PerCpuData = PerCpuData; + fn interrupt_mask() -> bool { DAIF.read(DAIF::I) != 0 } @@ -29,4 +49,25 @@ impl Architecture for ArchitectureImpl { fn wait_for_interrupt() { aarch64_cpu::asm::wfi(); } + + unsafe fn set_local_cpu(cpu: *mut ()) { + TPIDR_EL1.set(cpu as _); + } + + fn local_cpu() -> *mut () { + TPIDR_EL1.get() as _ + } + + unsafe fn init_ipi_queues(queues: Vec>) { + IPI_QUEUES.init(queues); + } + + fn local_interrupt_controller() -> &'static dyn LocalInterruptController { + let local = Self::local_cpu_data().unwrap(); + *local.gic.get() + } + + fn message_interrupt_controller() -> &'static dyn MessageInterruptController { + todo!() + } } diff --git a/arch/interface/Cargo.toml b/arch/interface/Cargo.toml index 7161facb..cdcc580b 100644 --- a/arch/interface/Cargo.toml +++ b/arch/interface/Cargo.toml @@ -7,3 +7,4 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../../lib/device-api", features = ["derive"] } diff --git a/arch/interface/src/cpu.rs b/arch/interface/src/cpu.rs new file mode 100644 index 00000000..61830a44 --- /dev/null +++ b/arch/interface/src/cpu.rs @@ -0,0 +1,151 @@ +use core::{ + marker::PhantomData, + ops::{Deref, DerefMut}, +}; + +use alloc::vec::Vec; +use device_api::interrupt::IpiMessage; + +use crate::{ + guard::IrqGuard, sync::IrqSafeSpinlock, task::Scheduler, util::OneTimeInit, Architecture, +}; + +#[repr(C, align(0x10))] +pub struct CpuImpl { + inner: A::PerCpuData, + scheduler: OneTimeInit<&'static S>, + + id: u32, + current_thread_id: Option, + + _pd: PhantomData, +} + +pub struct LocalCpuImpl<'a, A: Architecture, S: Scheduler + 'static> { + cpu: &'a mut CpuImpl, + guard: IrqGuard, +} + +pub struct IpiQueue { + data: IrqSafeSpinlock>, +} + +impl CpuImpl { + pub fn new(id: u32, inner: A::PerCpuData) -> Self { + Self { + inner, + scheduler: OneTimeInit::new(), + id, + current_thread_id: None, + _pd: PhantomData, + } + } + + pub fn init_ipi_queues(cpu_count: usize) { + let queues = Vec::from_iter((0..cpu_count).map(|_| IpiQueue::new())); + unsafe { A::init_ipi_queues(queues) } + } + + pub fn set_current_thread_id(&mut self, id: Option) { + self.current_thread_id = id; + } + + pub fn current_thread_id(&self) -> Option { + self.current_thread_id + } + + pub fn set_scheduler(&mut self, sched: &'static S) { + self.scheduler.init(sched); + } + + pub fn try_get_scheduler(&self) -> Option<&'static S> { + self.scheduler.try_get().copied() + } + + pub fn scheduler(&self) -> &'static S { + self.scheduler.get() + } + + pub unsafe fn set_local(&'static mut self) { + A::set_local_cpu(self as *mut _ as *mut _) + } + + pub fn try_local<'a>() -> Option> { + let guard = IrqGuard::acquire(); + let cpu = A::local_cpu() as *mut Self; + + unsafe { cpu.as_mut().map(|cpu| LocalCpuImpl { cpu, guard }) } + } + + pub fn local<'a>() -> LocalCpuImpl<'a, A, S> { + Self::try_local().expect("Local CPU not initialized") + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn push_ipi_queue(_cpu_id: u32, _msg: IpiMessage) { + // XXX + todo!() + } + + pub fn get_ipi(&self) -> Option { + // XXX + todo!() + } +} + +impl Deref for CpuImpl { + type Target = A::PerCpuData; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for CpuImpl { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl<'a, A: Architecture, S: Scheduler + 'static> LocalCpuImpl<'a, A, S> { + pub fn into_guard(self) -> IrqGuard { + self.guard + } +} + +impl<'a, A: Architecture, S: Scheduler> Deref for LocalCpuImpl<'a, A, S> { + type Target = CpuImpl; + + fn deref(&self) -> &Self::Target { + self.cpu + } +} + +impl<'a, A: Architecture, S: Scheduler> DerefMut for LocalCpuImpl<'a, A, S> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.cpu + } +} + +impl IpiQueue { + pub const fn new() -> Self { + Self { + data: IrqSafeSpinlock::::new(None), + } + } + + pub fn push(&self, msg: IpiMessage) { + let mut lock = self.data.lock(); + + assert!(lock.is_none()); + lock.replace(msg); + } + + pub fn pop(&self) -> Option { + let mut lock = self.data.lock(); + lock.take() + } +} diff --git a/libk-util/src/sync/guard.rs b/arch/interface/src/guard.rs similarity index 50% rename from libk-util/src/sync/guard.rs rename to arch/interface/src/guard.rs index 8b4625a2..ef20a092 100644 --- a/libk-util/src/sync/guard.rs +++ b/arch/interface/src/guard.rs @@ -1,22 +1,24 @@ -use kernel_arch::{Architecture, ArchitectureImpl}; +use core::marker::PhantomData; + +use crate::Architecture; /// Token type used to prevent IRQs from firing during some critical section. Normal IRQ operation /// (if enabled before) is resumed when [IrqGuard]'s lifetime is over. -pub struct IrqGuard(bool); +pub struct IrqGuard(bool, PhantomData); // IrqGuard impls -impl IrqGuard { +impl IrqGuard { /// Saves the current IRQ state and masks them pub fn acquire() -> Self { - let mask = unsafe { ArchitectureImpl::set_interrupt_mask(true) }; - Self(mask) + let mask = unsafe { A::set_interrupt_mask(true) }; + Self(mask, PhantomData) } } -impl Drop for IrqGuard { +impl Drop for IrqGuard { fn drop(&mut self) { unsafe { - ArchitectureImpl::set_interrupt_mask(self.0); + A::set_interrupt_mask(self.0); } } } diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs index 65c3e138..e210aacc 100644 --- a/arch/interface/src/lib.rs +++ b/arch/interface/src/lib.rs @@ -1,14 +1,40 @@ #![no_std] #![feature(step_trait, effects, const_trait_impl)] +use alloc::vec::Vec; +use cpu::IpiQueue; +use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; + +extern crate alloc; + +pub mod cpu; +pub mod guard; pub mod mem; +pub mod sync; +pub mod task; +pub mod util; pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; -pub trait Architecture { +pub trait Architecture: Sized { + type PerCpuData; + + // Cpu management + unsafe fn set_local_cpu(cpu: *mut ()); + fn local_cpu() -> *mut (); + unsafe fn init_ipi_queues(queues: Vec>); + + // Interrupt management fn interrupt_mask() -> bool; - unsafe fn set_interrupt_mask(mask: bool) -> bool; - fn wait_for_interrupt(); + + // Architectural devices + fn local_interrupt_controller() -> &'static dyn LocalInterruptController { + unimplemented!() + } + + fn message_interrupt_controller() -> &'static dyn MessageInterruptController { + unimplemented!() + } } diff --git a/arch/interface/src/sync.rs b/arch/interface/src/sync.rs new file mode 100644 index 00000000..940aa375 --- /dev/null +++ b/arch/interface/src/sync.rs @@ -0,0 +1,155 @@ +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem, + ops::{Deref, DerefMut}, + sync::atomic::{AtomicBool, Ordering}, +}; + +use crate::{guard::IrqGuard, Architecture}; + +struct SpinlockInner { + value: UnsafeCell, + state: AtomicBool, + _pd: PhantomData, +} + +struct SpinlockInnerGuard<'a, A: Architecture, T> { + lock: &'a SpinlockInner, +} + +/// Spinlock implementation which prevents interrupts to avoid deadlocks when an interrupt handler +/// tries to acquire a lock taken before the IRQ fired. +pub struct IrqSafeSpinlock { + inner: SpinlockInner, +} + +/// Token type allowing safe access to the underlying data of the [IrqSafeSpinlock]. Resumes normal +/// IRQ operation (if enabled before acquiring) when the lifetime is over. +pub struct IrqSafeSpinlockGuard<'a, A: Architecture, T> { + // Must come first to ensure the lock is dropped first and only then IRQs are re-enabled + inner: SpinlockInnerGuard<'a, A, T>, + _irq: IrqGuard, +} + +// Spinlock impls +impl SpinlockInner { + const fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + state: AtomicBool::new(false), + _pd: PhantomData, + } + } + + fn lock(&self) -> SpinlockInnerGuard { + // Loop until the lock can be acquired + // if LOCK_HACK.load(Ordering::Acquire) { + // return SpinlockInnerGuard { lock: self }; + // } + while self + .state + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + core::hint::spin_loop(); + } + + SpinlockInnerGuard { lock: self } + } +} + +impl<'a, A: Architecture, T> Deref for SpinlockInnerGuard<'a, A, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.lock.value.get() } + } +} + +impl<'a, A: Architecture, T> DerefMut for SpinlockInnerGuard<'a, A, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.lock.value.get() } + } +} + +impl<'a, A: Architecture, T> Drop for SpinlockInnerGuard<'a, A, T> { + fn drop(&mut self) { + // if !LOCK_HACK.load(Ordering::Acquire) { + self.lock + .state + .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) + .unwrap(); + // } + } +} + +unsafe impl Sync for SpinlockInner {} +unsafe impl Send for SpinlockInner {} + +// IrqSafeSpinlock impls +impl IrqSafeSpinlock { + /// Wraps the value in a spinlock primitive + pub const fn new(value: T) -> Self { + Self { + inner: SpinlockInner::new(value), + } + } + + #[inline] + pub fn replace(&self, value: T) -> T { + let mut lock = self.lock(); + mem::replace(&mut lock, value) + } + + /// Attempts to acquire a lock. IRQs will be disabled until the lock is released. + pub fn lock(&self) -> IrqSafeSpinlockGuard { + // Disable IRQs to avoid IRQ handler trying to acquire the same lock + let irq_guard = IrqGuard::acquire(); + + // Acquire the inner lock + let inner = self.inner.lock(); + + IrqSafeSpinlockGuard { + inner, + _irq: irq_guard, + } + } + + /// Returns an unsafe reference to the inner value. + /// + /// # Safety + /// + /// Unsafe: explicitly ignores proper access sharing. + #[allow(clippy::mut_from_ref)] + pub unsafe fn grab(&self) -> &mut T { + unsafe { &mut *self.inner.value.get() } + } +} + +impl IrqSafeSpinlock { + pub fn get_cloned(&self) -> T { + self.lock().clone() + } +} + +impl Clone for IrqSafeSpinlock { + fn clone(&self) -> Self { + let inner = self.lock(); + IrqSafeSpinlock::new(inner.clone()) + } +} + +impl<'a, A: Architecture, T> Deref for IrqSafeSpinlockGuard<'a, A, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.inner.deref() + } +} + +impl<'a, A: Architecture, T> DerefMut for IrqSafeSpinlockGuard<'a, A, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.deref_mut() + } +} diff --git a/arch/interface/src/task.rs b/arch/interface/src/task.rs new file mode 100644 index 00000000..230b2d25 --- /dev/null +++ b/arch/interface/src/task.rs @@ -0,0 +1,3 @@ +pub trait Scheduler { + type ThreadId: Copy; +} diff --git a/arch/interface/src/util.rs b/arch/interface/src/util.rs new file mode 100644 index 00000000..0778b012 --- /dev/null +++ b/arch/interface/src/util.rs @@ -0,0 +1,125 @@ +use core::{ + cell::UnsafeCell, + mem::MaybeUninit, + panic, + sync::atomic::{AtomicUsize, Ordering}, +}; + +/// Wrapper struct to ensure a value can only be initialized once and used only after that +#[repr(C)] +pub struct OneTimeInit { + value: UnsafeCell>, + state: AtomicUsize, +} + +unsafe impl Sync for OneTimeInit {} +unsafe impl Send for OneTimeInit {} + +impl OneTimeInit { + const STATE_UNINITIALIZED: usize = 0; + const STATE_INITIALIZING: usize = 1; + const STATE_INITIALIZED: usize = 2; + + /// Wraps the value in an [OneTimeInit] + pub const fn new() -> Self { + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + state: AtomicUsize::new(Self::STATE_UNINITIALIZED), + } + } + + /// Returns `true` if the value has already been initialized + #[inline] + pub fn is_initialized(&self) -> bool { + self.state.load(Ordering::Acquire) == Self::STATE_INITIALIZED + } + + pub fn try_init_with T>(&self, f: F) -> Option<&T> { + if self + .state + .compare_exchange( + Self::STATE_UNINITIALIZED, + Self::STATE_INITIALIZING, + Ordering::Release, + Ordering::Relaxed, + ) + .is_err() + { + // Already initialized + return None; + } + + let value = unsafe { (*self.value.get()).write(f()) }; + + self.state + .compare_exchange( + Self::STATE_INITIALIZING, + Self::STATE_INITIALIZED, + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); + + Some(value) + } + + /// Sets the underlying value of the [OneTimeInit]. If already initialized, panics. + #[track_caller] + pub fn init(&self, value: T) -> &T { + // Transition to "initializing" state + if self + .state + .compare_exchange( + Self::STATE_UNINITIALIZED, + Self::STATE_INITIALIZING, + Ordering::Release, + Ordering::Relaxed, + ) + .is_err() + { + panic!( + "{:?}: Double initialization of OneTimeInit", + panic::Location::caller() + ); + } + + let value = unsafe { (*self.value.get()).write(value) }; + + // Transition to "initialized" state. This must not fail + self.state + .compare_exchange( + Self::STATE_INITIALIZING, + Self::STATE_INITIALIZED, + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); + + value + } + + /// Returns an immutable reference to the underlying value and panics if it hasn't yet been + /// initialized + #[track_caller] + pub fn get(&self) -> &T { + // TODO check for INITIALIZING state and wait until it becomes INITIALIZED? + if !self.is_initialized() { + panic!( + "{:?}: Attempt to dereference an uninitialized value", + panic::Location::caller() + ); + } + + unsafe { (*self.value.get()).assume_init_ref() } + } + + /// Returns an immutable reference to the underlying value and [None] if the value hasn't yet + /// been initialized + pub fn try_get(&self) -> Option<&T> { + if self.is_initialized() { + Some(self.get()) + } else { + None + } + } +} diff --git a/arch/src/lib.rs b/arch/src/lib.rs index c733cde5..9955b9c2 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -31,4 +31,7 @@ cfg_if! { pub use imp::{ArchitectureImpl, KernelTableManagerImpl, ProcessAddressSpaceImpl}; -pub use kernel_arch_interface::{mem, Architecture}; +pub use kernel_arch_interface::{guard, mem, sync, task, util, Architecture}; + +pub type CpuImpl = kernel_arch_interface::cpu::CpuImpl; +pub type LocalCpuImpl<'a, S> = kernel_arch_interface::cpu::LocalCpuImpl<'a, ArchitectureImpl, S>; diff --git a/arch/x86_64/Cargo.toml b/arch/x86_64/Cargo.toml index a78d7d10..f513dc5d 100644 --- a/arch/x86_64/Cargo.toml +++ b/arch/x86_64/Cargo.toml @@ -10,6 +10,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../interface" } libk-mm-interface = { path = "../../libk-mm/interface" } memtables = { path = "../../lib/memtables" } +device-api = { path = "../../lib/device-api", features = ["derive"] } bitflags = "2.3.3" static_assertions = "1.1.0" diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index a0a98925..fcc8dee5 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -1,7 +1,14 @@ #![no_std] #![feature(effects, strict_provenance)] -use kernel_arch_interface::Architecture; +extern crate alloc; + +use alloc::vec::Vec; +use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; +use kernel_arch_interface::{cpu::IpiQueue, util::OneTimeInit, Architecture}; +use libk_mm_interface::address::PhysicalAddress; +use registers::MSR_IA32_KERNEL_GS_BASE; +use tock_registers::interfaces::Writeable; pub mod mem; pub mod registers; @@ -12,7 +19,64 @@ pub struct ArchitectureImpl; pub const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; +pub trait LocalApicInterface: LocalInterruptController + MessageInterruptController { + /// Performs an application processor startup sequence. + /// + /// # Safety + /// + /// Unsafe: only meant to be called by the BSP during SMP init. + unsafe fn wakeup_cpu(&self, apic_id: u32, bootstrap_code: PhysicalAddress); + + /// Signals local APIC that we've handled the IRQ + fn clear_interrupt(&self); +} + +#[repr(C, align(0x10))] +pub struct PerCpuData { + // 0x00 + pub this: *mut Self, + // 0x08, used in assembly + pub tss_address: usize, + // 0x10, used in assembly + pub tmp_address: usize, + + pub local_apic: &'static dyn LocalApicInterface, +} + +impl PerCpuData { + pub fn local_apic(&self) -> &'static dyn LocalApicInterface { + self.local_apic + } +} + +static IPI_QUEUES: OneTimeInit>> = OneTimeInit::new(); + +impl ArchitectureImpl { + fn local_cpu_data() -> Option<&'static mut PerCpuData> { + unsafe { (Self::local_cpu() as *mut PerCpuData).as_mut() } + } +} + impl Architecture for ArchitectureImpl { + type PerCpuData = PerCpuData; + + unsafe fn set_local_cpu(cpu: *mut ()) { + MSR_IA32_KERNEL_GS_BASE.set(cpu as u64); + core::arch::asm!("wbinvd; swapgs"); + } + + fn local_cpu() -> *mut () { + let mut addr: u64; + unsafe { + core::arch::asm!("movq %gs:(0), {0}", out(reg) addr, options(att_syntax)); + } + addr as _ + } + + unsafe fn init_ipi_queues(queues: Vec>) { + IPI_QUEUES.init(queues); + } + fn interrupt_mask() -> bool { let mut flags: u64; unsafe { @@ -38,4 +102,14 @@ impl Architecture for ArchitectureImpl { core::arch::asm!("hlt"); } } + + fn local_interrupt_controller() -> &'static dyn LocalInterruptController { + let local = Self::local_cpu_data().unwrap(); + local.local_apic + } + + fn message_interrupt_controller() -> &'static dyn MessageInterruptController { + let local = Self::local_cpu_data().unwrap(); + local.local_apic + } } diff --git a/libk-util/src/lib.rs b/libk-util/src/lib.rs index 1e7aec4b..f7d5495d 100644 --- a/libk-util/src/lib.rs +++ b/libk-util/src/lib.rs @@ -4,11 +4,9 @@ extern crate alloc; use core::{ - cell::UnsafeCell, mem::MaybeUninit, ops::{Deref, DerefMut}, panic, - sync::atomic::{AtomicUsize, Ordering}, }; pub mod event; @@ -17,6 +15,9 @@ pub mod ring; pub mod sync; pub mod waker; +// TODO make a crate for such utils? +pub use kernel_arch::util::OneTimeInit; + pub enum ConstAssert {} pub trait IsTrue {} @@ -28,125 +29,6 @@ pub struct StaticVector { len: usize, } -/// Wrapper struct to ensure a value can only be initialized once and used only after that -#[repr(C)] -pub struct OneTimeInit { - value: UnsafeCell>, - state: AtomicUsize, -} - -unsafe impl Sync for OneTimeInit {} -unsafe impl Send for OneTimeInit {} - -impl OneTimeInit { - const STATE_UNINITIALIZED: usize = 0; - const STATE_INITIALIZING: usize = 1; - const STATE_INITIALIZED: usize = 2; - - /// Wraps the value in an [OneTimeInit] - pub const fn new() -> Self { - Self { - value: UnsafeCell::new(MaybeUninit::uninit()), - state: AtomicUsize::new(Self::STATE_UNINITIALIZED), - } - } - - /// Returns `true` if the value has already been initialized - #[inline] - pub fn is_initialized(&self) -> bool { - self.state.load(Ordering::Acquire) == Self::STATE_INITIALIZED - } - - pub fn try_init_with T>(&self, f: F) -> Option<&T> { - if self - .state - .compare_exchange( - Self::STATE_UNINITIALIZED, - Self::STATE_INITIALIZING, - Ordering::Release, - Ordering::Relaxed, - ) - .is_err() - { - // Already initialized - return None; - } - - let value = unsafe { (*self.value.get()).write(f()) }; - - self.state - .compare_exchange( - Self::STATE_INITIALIZING, - Self::STATE_INITIALIZED, - Ordering::Release, - Ordering::Relaxed, - ) - .unwrap(); - - Some(value) - } - - /// Sets the underlying value of the [OneTimeInit]. If already initialized, panics. - #[track_caller] - pub fn init(&self, value: T) -> &T { - // Transition to "initializing" state - if self - .state - .compare_exchange( - Self::STATE_UNINITIALIZED, - Self::STATE_INITIALIZING, - Ordering::Release, - Ordering::Relaxed, - ) - .is_err() - { - panic!( - "{:?}: Double initialization of OneTimeInit", - panic::Location::caller() - ); - } - - let value = unsafe { (*self.value.get()).write(value) }; - - // Transition to "initialized" state. This must not fail - self.state - .compare_exchange( - Self::STATE_INITIALIZING, - Self::STATE_INITIALIZED, - Ordering::Release, - Ordering::Relaxed, - ) - .unwrap(); - - value - } - - /// Returns an immutable reference to the underlying value and panics if it hasn't yet been - /// initialized - #[track_caller] - pub fn get(&self) -> &T { - // TODO check for INITIALIZING state and wait until it becomes INITIALIZED? - if !self.is_initialized() { - panic!( - "{:?}: Attempt to dereference an uninitialized value", - panic::Location::caller() - ); - } - - unsafe { (*self.value.get()).assume_init_ref() } - } - - /// Returns an immutable reference to the underlying value and [None] if the value hasn't yet - /// been initialized - pub fn try_get(&self) -> Option<&T> { - if self.is_initialized() { - Some(self.get()) - } else { - None - } - } -} - impl StaticVector { /// Constructs an empty instance of [StaticVector] pub const fn new() -> Self diff --git a/libk-util/src/sync/mod.rs b/libk-util/src/sync/mod.rs index 13966f17..830dbe62 100644 --- a/libk-util/src/sync/mod.rs +++ b/libk-util/src/sync/mod.rs @@ -1,15 +1,16 @@ use core::sync::atomic::{AtomicBool, Ordering}; +use kernel_arch::ArchitectureImpl; use yggdrasil_abi::error::Error; pub mod fence; -pub mod guard; pub mod spin_rwlock; -pub mod spinlock; pub use fence::SpinFence; -pub use guard::IrqGuard; -pub use spinlock::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; +pub type IrqSafeSpinlock = kernel_arch::sync::IrqSafeSpinlock; +pub type IrqSafeSpinlockGuard<'a, T> = + kernel_arch::sync::IrqSafeSpinlockGuard<'a, ArchitectureImpl, T>; +pub type IrqGuard = kernel_arch::guard::IrqGuard; static LOCK_HACK: AtomicBool = AtomicBool::new(false); diff --git a/libk-util/src/sync/spinlock.rs b/libk-util/src/sync/spinlock.rs index ad1bcd12..e69de29b 100644 --- a/libk-util/src/sync/spinlock.rs +++ b/libk-util/src/sync/spinlock.rs @@ -1,152 +0,0 @@ -use core::{ - cell::UnsafeCell, - mem, - ops::{Deref, DerefMut}, - sync::atomic::{AtomicBool, Ordering}, -}; - -use super::{IrqGuard, LOCK_HACK}; - -struct SpinlockInner { - value: UnsafeCell, - state: AtomicBool, -} - -struct SpinlockInnerGuard<'a, T> { - lock: &'a SpinlockInner, -} - -/// Spinlock implementation which prevents interrupts to avoid deadlocks when an interrupt handler -/// tries to acquire a lock taken before the IRQ fired. -pub struct IrqSafeSpinlock { - inner: SpinlockInner, -} - -/// Token type allowing safe access to the underlying data of the [IrqSafeSpinlock]. Resumes normal -/// IRQ operation (if enabled before acquiring) when the lifetime is over. -pub struct IrqSafeSpinlockGuard<'a, T> { - // Must come first to ensure the lock is dropped first and only then IRQs are re-enabled - inner: SpinlockInnerGuard<'a, T>, - _irq: IrqGuard, -} - -// Spinlock impls -impl SpinlockInner { - const fn new(value: T) -> Self { - Self { - value: UnsafeCell::new(value), - state: AtomicBool::new(false), - } - } - - fn lock(&self) -> SpinlockInnerGuard { - // Loop until the lock can be acquired - if LOCK_HACK.load(Ordering::Acquire) { - return SpinlockInnerGuard { lock: self }; - } - while self - .state - .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_err() - { - core::hint::spin_loop(); - } - - SpinlockInnerGuard { lock: self } - } -} - -impl<'a, T> Deref for SpinlockInnerGuard<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - unsafe { &*self.lock.value.get() } - } -} - -impl<'a, T> DerefMut for SpinlockInnerGuard<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - unsafe { &mut *self.lock.value.get() } - } -} - -impl<'a, T> Drop for SpinlockInnerGuard<'a, T> { - fn drop(&mut self) { - if !LOCK_HACK.load(Ordering::Acquire) { - self.lock - .state - .compare_exchange(true, false, Ordering::Release, Ordering::Relaxed) - .unwrap(); - } - } -} - -unsafe impl Sync for SpinlockInner {} -unsafe impl Send for SpinlockInner {} - -// IrqSafeSpinlock impls -impl IrqSafeSpinlock { - /// Wraps the value in a spinlock primitive - pub const fn new(value: T) -> Self { - Self { - inner: SpinlockInner::new(value), - } - } - - #[inline] - pub fn replace(&self, value: T) -> T { - let mut lock = self.lock(); - mem::replace(&mut lock, value) - } - - /// Attempts to acquire a lock. IRQs will be disabled until the lock is released. - pub fn lock(&self) -> IrqSafeSpinlockGuard { - // Disable IRQs to avoid IRQ handler trying to acquire the same lock - let irq_guard = IrqGuard::acquire(); - - // Acquire the inner lock - let inner = self.inner.lock(); - - IrqSafeSpinlockGuard { - inner, - _irq: irq_guard, - } - } - - /// Returns an unsafe reference to the inner value. - /// - /// # Safety - /// - /// Unsafe: explicitly ignores proper access sharing. - #[allow(clippy::mut_from_ref)] - pub unsafe fn grab(&self) -> &mut T { - unsafe { &mut *self.inner.value.get() } - } -} - -impl IrqSafeSpinlock { - pub fn get_cloned(&self) -> T { - self.lock().clone() - } -} - -impl Clone for IrqSafeSpinlock { - fn clone(&self) -> Self { - let inner = self.lock(); - IrqSafeSpinlock::new(inner.clone()) - } -} - -impl<'a, T> Deref for IrqSafeSpinlockGuard<'a, T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.inner.deref() - } -} - -impl<'a, T> DerefMut for IrqSafeSpinlockGuard<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.inner.deref_mut() - } -} diff --git a/libk/Cargo.toml b/libk/Cargo.toml index a4466c62..414a7248 100644 --- a/libk/Cargo.toml +++ b/libk/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] libk-mm = { path = "../libk-mm" } libk-util = { path = "../libk-util" } +kernel-arch = { path = "../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../lib/device-api", features = ["derive"] } diff --git a/libk/src/device.rs b/libk/src/device.rs index da83630c..eeef2e45 100644 --- a/libk/src/device.rs +++ b/libk/src/device.rs @@ -1,7 +1,8 @@ use device_api::interrupt::{ - ExternalInterruptController, InterruptHandler, IrqOptions, LocalInterruptController, + ExternalInterruptController, InterruptHandler, Irq, IrqOptions, LocalInterruptController, MessageInterruptController, }; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk_util::OneTimeInit; use yggdrasil_abi::error::Error; @@ -25,25 +26,27 @@ register_get!( EXTERNAL_INTC, &'static dyn ExternalInterruptController ); -register_get!( - register_local_interrupt_controller, - local_interrupt_controller, - LOCAL_INTC, - &'static dyn LocalInterruptController -); -register_get!( - register_message_interrupt_controller, - message_interrupt_controller, - MESSAGE_INTC, - &'static dyn MessageInterruptController -); + +pub fn local_interrupt_controller() -> &'static dyn LocalInterruptController { + ArchitectureImpl::local_interrupt_controller() +} + +pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController { + ArchitectureImpl::message_interrupt_controller() +} #[inline] pub fn register_global_interrupt( - _irq: u32, - _options: IrqOptions, - _handler: &'static dyn InterruptHandler, + irq: u32, + options: IrqOptions, + handler: &'static dyn InterruptHandler, ) -> Result<(), Error> { - // XXX - todo!() + let intc = external_interrupt_controller(); + + let irq = Irq::External(irq); + + intc.register_irq(irq, options, handler)?; + intc.enable_irq(irq)?; + + Ok(()) } diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs deleted file mode 100644 index dab57df4..00000000 --- a/src/arch/aarch64/cpu.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Per-CPU data structures -use core::{ - ops::{Deref, DerefMut}, - sync::atomic::Ordering, -}; - -use aarch64_cpu::registers::{MPIDR_EL1, TPIDR_EL1}; -use alloc::{boxed::Box, vec::Vec}; -use device_api::interrupt::IpiMessage; -use libk_util::{ - sync::{IrqGuard, IrqSafeSpinlock}, - OneTimeInit, -}; -use tock_registers::interfaces::{Readable, Writeable}; - -use crate::{ - arch::{CpuAccess, LocalCpuAccess}, - panic, - task::{sched::CpuQueue, thread::ThreadId}, -}; - -use super::{gic::GicPerCpu, smp::CPU_COUNT}; - -// use super::smp::CPU_COUNT; - -/// Per-CPU private data structure -#[repr(C, align(0x10))] -pub struct Cpu { - id: u32, - - queue: OneTimeInit<&'static CpuQueue>, - thread_id: Option, - - pub(super) gic_per_cpu: GicPerCpu, -} - -/// Handle allowing safe access to the local CPU. -/// -/// # Note -/// -/// Obtaining this handle prevents IRQs from being delivered to ensure the context having this -/// struct cannot be interrupted by another (which could obtain it as well). -pub struct LocalCpu(&'static mut Cpu, IrqGuard); - -struct IpiQueue { - data: IrqSafeSpinlock>, -} - -static IPI_QUEUES: OneTimeInit> = OneTimeInit::new(); - -impl IpiQueue { - pub const fn new() -> Self { - Self { - data: IrqSafeSpinlock::new(None), - } - } - - pub fn push(&self, msg: IpiMessage) { - let mut lock = self.data.lock(); - - assert!(lock.is_none()); - lock.replace(msg); - } - - pub fn pop(&self) -> Option { - let mut lock = self.data.lock(); - lock.take() - } -} - -impl Cpu { - /// Sets up global list of interprocessor message queues - pub fn init_ipi_queues() { - IPI_QUEUES.init(Vec::from_iter( - (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), - )); - } - - /// Sets up the local CPU's private data structure. - /// - /// # Safety - /// - /// The function is only meant to be called once during the early init process. - pub unsafe fn init_local() { - let this = Box::new(Cpu { - id: Self::local_id(), - queue: OneTimeInit::new(), - thread_id: None, - gic_per_cpu: GicPerCpu::new(), - }); - TPIDR_EL1.set(Box::into_raw(this) as _); - } -} - -impl CpuAccess for Cpu { - type Local = LocalCpu; - - fn id(&self) -> u32 { - self.id - } - - fn local_id() -> u32 { - (MPIDR_EL1.get() & 0xFF) as _ - } - - fn try_local() -> Option { - let guard = IrqGuard::acquire(); - let tpidr = TPIDR_EL1.get() as *mut Cpu; - unsafe { tpidr.as_mut() }.map(|cpu| LocalCpu(cpu, guard)) - } - - fn get_queue(&self) -> Option<&'static CpuQueue> { - self.queue.try_get().copied() - } - - fn init_queue(&mut self, queue: &'static CpuQueue) { - self.queue.init(queue); - } - - unsafe fn set_current_thread_id(&mut self, id: Option) { - self.thread_id = id; - } - - fn current_thread_id(&self) -> Option { - self.thread_id - } - - fn push_ipi(id: u32, msg: IpiMessage) { - if let Some(q) = IPI_QUEUES.try_get().and_then(|q| q.get(id as usize)) { - q.push(msg); - } - } - - fn handle_ipi(&mut self) { - if let Some(ipi) = IPI_QUEUES - .try_get() - .and_then(|q| q.get(self.id() as usize)) - .and_then(|q| q.pop()) - { - match ipi { - IpiMessage::Panic => panic::panic_secondary(), - IpiMessage::Shutdown => todo!(), - } - } - } -} - -impl LocalCpuAccess for LocalCpu { - fn into_guard(self) -> IrqGuard { - self.1 - } -} - -impl Deref for LocalCpu { - type Target = Cpu; - - fn deref(&self) -> &Self::Target { - self.0 - } -} - -impl DerefMut for LocalCpu { - fn deref_mut(&mut self) -> &mut Self::Target { - self.0 - } -} diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 3a842836..8e497b3a 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -14,21 +14,19 @@ use device_api::{ Device, }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; -use libk::device::{ - register_external_interrupt_controller, register_local_interrupt_controller, - register_message_interrupt_controller, -}; +use kernel_arch_aarch64::GicInterface; +use libk::device::register_external_interrupt_controller; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, RawDeviceMemoryMapping}, }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::arch::CpuAccess; +use crate::{arch::cpu_index, task::Cpu}; use self::{gicc::Gicc, gicd::Gicd}; -use super::{cpu::Cpu, smp::CPU_COUNT}; +use super::{smp::CPU_COUNT, AArch64}; const MAX_IRQ: usize = 300; const IPI_VECTOR: u64 = 1; @@ -48,6 +46,8 @@ pub struct Gic { /// Per-CPU GIC information pub struct GicPerCpu {} +impl GicInterface for Gic {} + impl Device for Gic { fn display_name(&self) -> &'static str { "ARM Generic Interrupt Controller v2" @@ -73,8 +73,7 @@ impl Device for Gic { self.gicc.init(gicc); register_external_interrupt_controller(self); - register_local_interrupt_controller(self); - register_message_interrupt_controller(self); + AArch64::set_gic(self); Ok(()) } @@ -129,8 +128,10 @@ impl ExternalInterruptController for Gic { if irq_number == IPI_VECTOR as usize { gicc.clear_irq(irq_number); - Cpu::local().handle_ipi(); - return; + // XXX + todo!(); + // Cpu::local().handle_ipi(); + // return; } gicc.clear_irq(irq_number); @@ -167,10 +168,10 @@ impl LocalInterruptController for Gic { // TODO message queue insertion should be moved match target { IpiDeliveryTarget::OtherCpus => { - let local = Cpu::local_id(); + let local = cpu_index(); for i in 0..CPU_COUNT.load(Ordering::Acquire) { if i != local as usize { - Cpu::push_ipi(i as u32, msg); + Cpu::push_ipi_queue(i as u32, msg); } } } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 5cd1b60a..73099c97 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -5,17 +5,20 @@ use core::sync::atomic::Ordering; use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use device_api::{ - interrupt::{IpiDeliveryTarget, IpiMessage, Irq}, + interrupt::{IpiDeliveryTarget, IpiMessage, Irq, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; use git_version::git_version; -use kernel_arch_aarch64::mem::{ - table::{L1, L2, L3}, - EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, +use kernel_arch_aarch64::{ + mem::{ + table::{L1, L2, L3}, + EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1_COUNT, + }, + ArchitectureImpl, PerCpuData, }; -use libk::device::{external_interrupt_controller, local_interrupt_controller}; +use libk::device::external_interrupt_controller; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, @@ -28,18 +31,19 @@ use tock_registers::interfaces::Writeable; use ygg_driver_pci::PciBusManager; use crate::{ - arch::aarch64::cpu::Cpu, debug, device::{self, power::arm_psci::Psci}, fs::{Initrd, INITRD_DATA}, mem::heap, + task::Cpu, }; +use self::gic::Gic; + use super::Platform; pub mod boot; pub mod context; -pub mod cpu; pub mod exception; pub mod gic; pub mod smp; @@ -95,8 +99,8 @@ impl Platform for AArch64 { } unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: IpiMessage) -> Result<(), Error> { + Ok(()) // XXX - todo!() // if let Some(local_intc) = self.lintc.try_get() { // local_intc.send_ipi(target, msg) // } else { @@ -119,7 +123,13 @@ impl Platform for AArch64 { } } +static GIC: OneTimeInit<&'static Gic> = OneTimeInit::new(); + impl AArch64 { + fn set_gic(gic: &'static Gic) { + GIC.init(gic); + } + fn extract_initrd_from_dt( &self, dt: &DeviceTree, @@ -243,7 +253,10 @@ impl AArch64 { } unsafe fn init_platform(&self, is_bsp: bool) -> Result<(), Error> { - Cpu::init_local(); + let per_cpu = PerCpuData { + gic: OneTimeInit::new(), + }; + Cpu::init_local(per_cpu); if is_bsp { ygg_driver_pci::register_vendor_driver( @@ -337,10 +350,10 @@ impl AArch64 { } else { // BSP already initialized everything needed // Setup timer and local interrupt controller - let intc = local_interrupt_controller(); - - unsafe { - intc.init_ap().unwrap(); + if let Some(gic) = GIC.try_get() { + unsafe { + gic.init_ap().unwrap(); + } } // TODO device-tree initialization for this @@ -351,6 +364,11 @@ impl AArch64 { .unwrap(); } + if let Some(gic) = GIC.try_get() { + let cpu_data = ArchitectureImpl::local_cpu_data().unwrap(); + cpu_data.gic.init(*gic); + } + Ok(()) } } diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 0623164d..932341f4 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -14,9 +14,10 @@ use device_tree::device_tree_driver; use libk::{device::external_interrupt_controller, runtime}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use crate::arch::{CpuAccess, Platform, PLATFORM}; - -use super::cpu::Cpu; +use crate::{ + arch::{Platform, PLATFORM}, + task::Cpu, +}; /// ARM Generic Timer driver pub struct ArmTimer { @@ -34,7 +35,7 @@ impl InterruptHandler for ArmTimer { runtime::tick(now); unsafe { - Cpu::local().queue().yield_cpu(); + Cpu::local().scheduler().yield_cpu(); } true diff --git a/src/arch/mod.rs b/src/arch/mod.rs index cd7be972..162dcf0d 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,6 +1,6 @@ //! Provides architecture/platform-specific implementation details -use core::{ops::DerefMut, time::Duration}; +use core::time::Duration; use abi::error::Error; @@ -10,11 +10,10 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use kernel_arch::{Architecture, ArchitectureImpl}; +use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl}; use libk_mm::table::EntryLevel; -use libk_util::sync::IrqGuard; -use crate::task::{sched::CpuQueue, thread::ThreadId, Cpu}; +use crate::task::Cpu; cfg_if! { if #[cfg(target_arch = "aarch64")] { @@ -102,74 +101,13 @@ pub trait Platform { } } -/// Interface for accessing local CPU (the CPU of the current context) -pub trait LocalCpuAccess: DerefMut { - /// Consumes the struct, returning the inner IrqGuard for further use - fn into_guard(self) -> IrqGuard; -} - -/// Interface for accessing CPU-local data -pub trait CpuAccess: Sized { - /// Safe wrapper for accessing local CPU - type Local: LocalCpuAccess; - - /// Returns the local CPU wrapper or None, if not yet initialized: - fn try_local() -> Option; - - /// Returns the local CPU wrapper - fn local() -> Self::Local { - Self::try_local().expect("Local CPU has not yet been initialized") - } - - /// Returns the ID of the local processor or 0 if the processor has not yet been initialized - fn local_id() -> u32 { - Self::try_local().map(|cpu| cpu.id()).unwrap_or(0) - } - - /// Initializes the CPU's scheduling queue. - /// - /// # Panics - /// - /// Will panic, if the initialization has already been performed. - fn init_queue(&mut self, queue: &'static CpuQueue); - - /// Returns the CPU's scheduling queue or None if it has not yet been initialized - fn get_queue(&self) -> Option<&'static CpuQueue>; - - /// Returns the CPU's scheduling queue or panics if it has not yet been initialized - fn queue(&self) -> &'static CpuQueue { - self.get_queue() - .expect("CPU's queue has not yet been initialized") - } - - /// Returns the CPU index - fn id(&self) -> u32; - - /// Returns current thread ID or none if idle - fn current_thread_id(&self) -> Option; - - /// Update the current thread ID. - /// - /// # Safety - /// - /// Only meant to be called from within the scheduler. - unsafe fn set_current_thread_id(&mut self, id: Option); - - /// Push an IPI message to the CPU's queue - fn push_ipi(id: u32, msg: IpiMessage) { - let _ = id; - let _ = msg; - // Not implemented by default - } - - /// Pops an IPI message (if any is present) from the CPU's queue and handles it - fn handle_ipi(&mut self) { - // Not implemented by default - } -} - // External API for architecture specifics +/// Returns local CPU index +pub fn cpu_index() -> u32 { + Cpu::try_local().as_deref().map(CpuImpl::id).unwrap_or(0) +} + #[no_mangle] fn __cpu_index() -> usize { Cpu::local().id() as _ @@ -184,19 +122,3 @@ fn __cpu_count() -> usize { fn __monotonic_timestamp() -> Result { PLATFORM.monotonic_timer().monotonic_timestamp() } - -// #[no_mangle] -// fn __register_global_interrupt( -// irq: u32, -// options: IrqOptions, -// handler: &'static dyn InterruptHandler, -// ) -> Result<(), Error> { -// let intc = PLATFORM.external_interrupt_controller(); -// -// let irq = Irq::External(irq); -// -// intc.register_irq(irq, options, handler)?; -// intc.enable_irq(irq)?; -// -// Ok(()) -// } diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 3909ec71..fcc79f92 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -10,7 +10,7 @@ use device_api::{ }, Device, }; -use kernel_arch_x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE}; +use kernel_arch_x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, LocalApicInterface}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, @@ -27,10 +27,7 @@ use tock_registers::{ }; use crate::{ - arch::{ - x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}, - CpuAccess, - }, + arch::x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}, task::Cpu, }; @@ -158,6 +155,63 @@ impl Device for LocalApic { } } +impl LocalApicInterface for LocalApic { + fn clear_interrupt(&self) { + self.regs.EndOfInterrupt.set(0); + } + + unsafe fn wakeup_cpu(&self, apic_id: u32, entry_vector: PhysicalAddress) { + infoln!("Waking up apic{}, entry = {:#x}", apic_id, entry_vector); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + let entry_vector = entry_vector.page_index::(); + + // INIT assert + self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); + + self.regs.ICR0.write( + ICR0::Destination::INIT + + ICR0::DestinationType::Physical + + ICR0::INIT0::Assert + + ICR0::INIT1::Assert, + ); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + // INIT deassert + self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); + + self.regs.ICR0.write( + ICR0::Destination::INIT + + ICR0::DestinationType::Physical + + ICR0::INIT0::Deassert + + ICR0::INIT1::Deassert, + ); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + + // Send another SIPI type IPI because the spec says so + self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); + + self.regs.ICR0.write( + ICR0::Vector.val(entry_vector as u32) + + ICR0::Destination::SIPI + + ICR0::DestinationType::Physical, + ); + + while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { + core::hint::spin_loop(); + } + } +} + impl MessageInterruptController for LocalApic { fn handle_msi(&self, vector: usize) { // TODO this is ugly @@ -309,67 +363,6 @@ impl LocalApic { } } - /// Signals local APIC that we've handled the IRQ - pub fn clear_interrupt(&self) { - self.regs.EndOfInterrupt.set(0); - } - - /// Performs an application processor startup sequence. - /// - /// # Safety - /// - /// Unsafe: only meant to be called by the BSP during SMP init. - pub unsafe fn wakeup_cpu(&self, apic_id: u32, entry_vector: PhysicalAddress) { - infoln!("Waking up apic{}, entry = {:#x}", apic_id, entry_vector); - - while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { - core::hint::spin_loop(); - } - - let entry_vector = entry_vector.page_index::(); - - // INIT assert - self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); - - self.regs.ICR0.write( - ICR0::Destination::INIT - + ICR0::DestinationType::Physical - + ICR0::INIT0::Assert - + ICR0::INIT1::Assert, - ); - - while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { - core::hint::spin_loop(); - } - - // INIT deassert - self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); - - self.regs.ICR0.write( - ICR0::Destination::INIT - + ICR0::DestinationType::Physical - + ICR0::INIT0::Deassert - + ICR0::INIT1::Deassert, - ); - - while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { - core::hint::spin_loop(); - } - - // Send another SIPI type IPI because the spec says so - self.regs.ICR1.write(ICR1::PhysicalDestination.val(apic_id)); - - self.regs.ICR0.write( - ICR0::Vector.val(entry_vector as u32) - + ICR0::Destination::SIPI - + ICR0::DestinationType::Physical, - ); - - while self.regs.ICR0.matches_all(ICR0::DeliveryStatus::SET) { - core::hint::spin_loop(); - } - } - #[inline] fn base() -> PhysicalAddress { PhysicalAddress::from_raw(MSR_IA32_APIC_BASE.read_base()) diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index c9aa835a..60488011 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -5,10 +5,7 @@ use core::arch::global_asm; use libk::device::{external_interrupt_controller, message_interrupt_controller}; use static_assertions::{const_assert, const_assert_eq}; -use crate::{ - arch::{x86_64::cpu::Cpu, CpuAccess}, - task::thread::Thread, -}; +use crate::task::{thread::Thread, Cpu}; use super::exception::{self, IrqFrame}; @@ -95,7 +92,7 @@ unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { // Clear interrupt before switching, because otherwise we won't receive the next one cpu.local_apic().clear_interrupt(); - if let Some(queue) = cpu.get_queue() { + if let Some(queue) = cpu.try_get_scheduler() { queue.yield_cpu(); } diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs deleted file mode 100644 index 8419ca23..00000000 --- a/src/arch/x86_64/cpu.rs +++ /dev/null @@ -1,209 +0,0 @@ -//! Per-CPU information and data structures -use core::{ - ops::{Deref, DerefMut}, - ptr::null_mut, - sync::atomic::Ordering, -}; - -use alloc::{boxed::Box, vec::Vec}; -use device_api::interrupt::IpiMessage; -use kernel_arch_x86_64::registers::MSR_IA32_KERNEL_GS_BASE; -use libk_util::{ - sync::{IrqGuard, IrqSafeSpinlock}, - OneTimeInit, -}; -use tock_registers::interfaces::Writeable; - -use crate::{ - arch::{ - x86_64::{cpuid, gdt, syscall}, - CpuAccess, LocalCpuAccess, - }, - task::{sched::CpuQueue, thread::ThreadId}, -}; - -use super::{apic::local::LocalApic, smp::CPU_COUNT}; - -/// Per-CPU data structure, only visible to the executing CPU -#[repr(C, align(0x10))] -pub struct Cpu { - // 0x00 - this: *mut Cpu, - // 0x08, used in assembly - tss_address: usize, - // 0x10, used in assembly for temporary RSP storage - tmp_address: usize, - // 0x08 - id: u32, - - local_apic: LocalApic, - current_thread_id: Option, - queue: OneTimeInit<&'static CpuQueue>, -} - -/// Handle allowing safe access to the local CPU. -/// -/// # Note -/// -/// Obtaining this handle prevents IRQs from being delivered to ensure the context having this -/// struct cannot be interrupted by another (which could obtain it as well). -pub struct LocalCpu(&'static mut Cpu, IrqGuard); - -struct IpiQueue { - data: IrqSafeSpinlock>, -} - -static IPI_QUEUES: OneTimeInit> = OneTimeInit::new(); - -impl IpiQueue { - pub const fn new() -> Self { - Self { - data: IrqSafeSpinlock::new(None), - } - } - - pub fn push(&self, msg: IpiMessage) { - let mut lock = self.data.lock(); - - assert!(lock.is_none()); - lock.replace(msg); - } - - pub fn pop(&self) -> Option { - let mut lock = self.data.lock(); - lock.take() - } -} - -impl Cpu { - /// Initializes the per-CPU data structure. - /// - /// # Safety - /// - /// Only meant to be called once per each CPU during their init. - pub(super) unsafe fn init_local(local_apic: LocalApic, id: u32) { - // Initialize CPU features - cpuid::enable_features(); - - let tss_address = gdt::init(); - - let this = Box::new(Cpu { - this: null_mut(), - tss_address, - tmp_address: 0, - id, - - local_apic, - current_thread_id: None, - queue: OneTimeInit::new(), - }); - let this = Box::into_raw(this); - (*this).this = this; - - MSR_IA32_KERNEL_GS_BASE.set(this as u64); - core::arch::asm!("wbinvd; swapgs"); - - infoln!("cpu{} initialized", id); - - syscall::init_syscall(); - } - - /// Sets up global list of interprocessor message queues - pub fn init_ipi_queues() { - IPI_QUEUES.init(Vec::from_iter( - (0..CPU_COUNT.load(Ordering::Acquire)).map(|_| IpiQueue::new()), - )); - } - - /// Inserts an IPI message to the back of the target CPU's message queue - pub fn push_ipi_queue(cpu_id: u32, msg: IpiMessage) { - let ipi_queue = &IPI_QUEUES.get()[cpu_id as usize]; - ipi_queue.push(msg); - } - - /// Pops the first IPI message received for this CPU. - /// - /// # Note - /// - /// Currently the queue consists of only one entry, so the CPU will only receive the last one. - pub fn get_ipi(&self) -> Option { - let ipi_queue = &IPI_QUEUES.get()[self.id as usize]; - ipi_queue.pop() - } -} - -impl LocalCpu { - /// Returns the local APIC associated with this CPU - pub fn local_apic(&self) -> &'static LocalApic { - // TODO find a way to safely represent this voodoo magic - let r = (&self.0.local_apic) as *const LocalApic; - unsafe { &*r } - } -} - -impl CpuAccess for Cpu { - type Local = LocalCpu; - - #[inline] - fn try_local() -> Option { - let mut addr: usize; - let guard = IrqGuard::acquire(); - unsafe { - core::arch::asm!("movq %gs:(0), {0}", out(reg) addr, options(att_syntax)); - - (addr as *mut Cpu).as_mut().map(|cpu| LocalCpu(cpu, guard)) - } - } - - #[inline] - fn id(&self) -> u32 { - self.id - } - - #[inline] - fn current_thread_id(&self) -> Option { - self.current_thread_id - } - - #[inline] - unsafe fn set_current_thread_id(&mut self, id: Option) { - self.current_thread_id = id; - } - - #[inline] - fn local_id() -> u32 { - if let Some(local) = Self::try_local() { - local.id() - } else { - 0 - } - } - - fn init_queue(&mut self, queue: &'static CpuQueue) { - self.queue.init(queue); - } - - fn get_queue(&self) -> Option<&'static CpuQueue> { - self.queue.try_get().copied() - } -} - -impl LocalCpuAccess for LocalCpu { - fn into_guard(self) -> IrqGuard { - self.1 - } -} - -impl Deref for LocalCpu { - type Target = Cpu; - - fn deref(&self) -> &Self::Target { - self.0 - } -} - -impl DerefMut for LocalCpu { - fn deref_mut(&mut self) -> &mut Self::Target { - self.0 - } -} diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 838604a7..5c5d2ea7 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -6,7 +6,7 @@ use kernel_arch_x86_64::registers::CR3; use tock_registers::interfaces::Readable; use crate::{ - arch::{x86_64::apic, CpuAccess}, + arch::x86_64::apic, debug, task::{context::TaskFrame, thread::Thread, Cpu}, }; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 252ca74e..c2664785 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,15 +1,22 @@ //! x86-64 architecture implementation -use core::{mem::size_of, ops::DerefMut, sync::atomic::Ordering}; +use core::{mem::size_of, ops::DerefMut, ptr::null_mut, sync::atomic::Ordering}; use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; -use device_api::{interrupt::Irq, timer::MonotonicTimestampProviderDevice, Device}; +use device_api::{ + interrupt::{Irq, MessageInterruptController}, + timer::MonotonicTimestampProviderDevice, + Device, +}; use git_version::git_version; -use kernel_arch_x86_64::mem::{ - init_fixed_tables, map_heap_block, - table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}, - EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1, +use kernel_arch_x86_64::{ + mem::{ + init_fixed_tables, map_heap_block, + table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}, + EarlyMapping, HEAP_MAPPING_OFFSET, MEMORY_LIMIT, RAM_MAPPING_L1, + }, + PerCpuData, }; use kernel_fs::devfs; use libk::device::register_external_interrupt_controller; @@ -26,7 +33,7 @@ mod acpi; mod apic; mod boot; pub mod context; -pub mod cpu; +// pub mod cpu; mod cpuid; mod exception; mod gdt; @@ -47,13 +54,13 @@ use crate::{ }, fs::{Initrd, INITRD_DATA}, mem::heap, + task::Cpu, }; use self::{ acpi::{AcpiAllocator, AcpiHandlerImpl}, apic::{ioapic::IoApic, local::LocalApic}, boot::BootData, - cpu::Cpu, cpuid::{ProcessorFeatures, PROCESSOR_FEATURES}, peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort}, smp::CPU_COUNT, @@ -284,7 +291,18 @@ impl X86_64 { } unsafe fn init_platform(&'static self, cpu_id: usize) -> Result<(), Error> { - Cpu::init_local(LocalApic::new(), cpu_id as _); + let local_apic = Box::leak(Box::new(LocalApic::new())); + let tss_address = gdt::init(); + + let cpu_data = PerCpuData { + this: null_mut(), + tss_address, + tmp_address: 0, + local_apic, + }; + cpuid::enable_features(); + Cpu::init_local(cpu_id as _, cpu_data); + syscall::init_syscall(); if cpu_id == 0 { // Register the PCI drivers @@ -484,3 +502,8 @@ impl X86_64 { pic_slave_cmd.write(0x20); } } + +#[no_mangle] +fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { + Cpu::local().local_apic +} diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index c91fdf6c..62a01d78 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -15,10 +15,7 @@ use libk_mm::{ TableAllocatorImpl, }; -use crate::{ - arch::{x86_64::boot::__x86_64_ap_entry, CpuAccess}, - task::Cpu, -}; +use crate::{arch::x86_64::boot::__x86_64_ap_entry, task::Cpu}; use super::acpi::AcpiAllocator; diff --git a/src/debug.rs b/src/debug.rs index dd97a7e9..4f020e9a 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -64,7 +64,7 @@ macro_rules! log_print_raw { macro_rules! log_print { ($level:expr, $($args:tt)+) => { - log_print_raw!($level, "cpu{}:{}:{}: {}", <$crate::task::Cpu as $crate::arch::CpuAccess>::local_id(), file!(), line!(), format_args!($($args)+)) + log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::arch::cpu_index(), file!(), line!(), format_args!($($args)+)) }; } diff --git a/src/main.rs b/src/main.rs index 0631189f..3cb0c578 100644 --- a/src/main.rs +++ b/src/main.rs @@ -103,7 +103,7 @@ pub fn kernel_main() -> ! { PLATFORM.start_application_processors(); } - Cpu::init_ipi_queues(); + Cpu::init_ipi_queues(PlatformImpl::cpu_count()); // Wait until all APs initialize CPU_INIT_FENCE.signal(); diff --git a/src/panic.rs b/src/panic.rs index 796fc581..c196f8d3 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -6,7 +6,7 @@ use kernel_arch::{Architecture, ArchitectureImpl}; use libk_util::sync::{hack_locks, SpinFence}; use crate::{ - arch::{CpuAccess, Platform, PlatformImpl, PLATFORM}, + arch::{Platform, PlatformImpl, PLATFORM}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, task::Cpu, diff --git a/src/task/context.rs b/src/task/context.rs index d78d486f..478a5f94 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -9,9 +9,9 @@ use crate::task::thread::Thread; cfg_if! { if #[cfg(target_arch = "aarch64")] { - pub use crate::arch::aarch64::{context::TaskContext, cpu::Cpu}; + pub use crate::arch::aarch64::context::TaskContext; } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::{context::TaskContext, cpu::Cpu}; + pub use crate::arch::x86_64::context::TaskContext; } } diff --git a/src/task/mod.rs b/src/task/mod.rs index 2df94a87..e15eafb2 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -2,13 +2,17 @@ #![allow(dead_code)] +use core::ops::{Deref, DerefMut}; + use abi::error::Error; use alloc::{string::String, vec::Vec}; +use device_api::interrupt::IpiMessage; +use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl, LocalCpuImpl}; use libk::{runtime, thread::Termination}; -use libk_util::sync::SpinFence; +use libk_util::sync::{IrqGuard, SpinFence}; use crate::{ - arch::{PlatformImpl, CpuAccess, Platform}, + arch::{Platform, PlatformImpl}, task::{sched::CpuQueue, thread::Thread}, }; @@ -20,7 +24,101 @@ pub mod sched; pub mod sync; pub mod thread; -pub use context::{Cpu, TaskContext}; +pub use context::TaskContext; + +/// Kernel wrapper for local CPU info structure. See [kernel_arch::LocalCpuImpl]. +pub struct LocalCpu<'a>(LocalCpuImpl<'a, CpuQueue>); +/// Kernel wrapper for per-CPU info structure. See [kernel_arch::LocalCpuImpl]. +pub struct Cpu(CpuImpl); + +impl Cpu { + /// Initializes the local CPU + #[cfg(target_arch = "x86_64")] + pub unsafe fn init_local(id: u32, inner: ::PerCpuData) { + use alloc::boxed::Box; + + let cpu = Box::leak(Box::new(CpuImpl::::new(id, inner))); + cpu.this = cpu.deref_mut(); + + cpu.set_local(); + } + + /// Initializes the local CPU + #[cfg(target_arch = "aarch64")] + pub unsafe fn init_local(inner: ::PerCpuData) { + use aarch64_cpu::registers::MPIDR_EL1; + use alloc::boxed::Box; + use tock_registers::interfaces::Readable; + + let id = (MPIDR_EL1.get() & 0xFF) as u32; + let cpu = Box::leak(Box::new(CpuImpl::::new(id, inner))); + + cpu.set_local(); + } + + /// Returns local CPU reference + #[inline] + pub fn local<'a>() -> LocalCpu<'a> { + LocalCpu(CpuImpl::local()) + } + + /// Returns local CPU reference or None if it hasn't yet been initialized + #[inline] + pub fn try_local<'a>() -> Option> { + CpuImpl::try_local().map(LocalCpu) + } + + /// Pushes a message to the IPI queue + #[inline] + pub fn push_ipi_queue(cpu_id: u32, msg: IpiMessage) { + CpuImpl::::push_ipi_queue(cpu_id, msg) + } + + /// Initialize the IPI queues for all present CPUs + #[inline] + pub fn init_ipi_queues(cpu_count: usize) { + CpuImpl::::init_ipi_queues(cpu_count) + } +} + +impl Deref for Cpu { + type Target = CpuImpl; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Cpu { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl<'a> LocalCpu<'a> { + /// Converts the local CPU handle into its IRQ guard + pub fn into_guard(self) -> IrqGuard { + self.0.into_guard() + } +} + +impl<'a> Deref for LocalCpu<'a> { + type Target = CpuImpl; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a> DerefMut for LocalCpu<'a> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.deref_mut() + } +} /// Creates a new kernel-space process to execute a closure and queues it to some CPU pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send + 'static>( @@ -70,7 +168,7 @@ pub unsafe fn enter() -> ! { } let queue = CpuQueue::for_cpu(cpu.id() as usize); - cpu.init_queue(queue); + cpu.set_scheduler(queue); queue.enter() } diff --git a/src/task/sched.rs b/src/task/sched.rs index 7ba464e1..04940552 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -5,10 +5,10 @@ use core::cell::Cell; use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; -use kernel_arch::{Architecture, ArchitectureImpl}; +use kernel_arch::{task::Scheduler, Architecture, ArchitectureImpl}; use libk_util::{sync::IrqGuard, OneTimeInit}; -use crate::{arch::CpuAccess, task::thread::ThreadState}; +use crate::task::thread::ThreadState; use super::{ context::TaskContextImpl, @@ -23,6 +23,10 @@ pub struct CpuQueue { idle: Cell, } +impl Scheduler for CpuQueue { + type ThreadId = ThreadId; +} + impl CpuQueue { /// Creates a new [CpuQueue] for CPU with given `index` pub fn new(index: usize) -> Self { @@ -181,7 +185,7 @@ impl CpuQueue { /// Returns a queue for the local CPU pub fn local() -> &'static CpuQueue { - Cpu::local().queue() + Cpu::local().scheduler() } /// Returns the preferred queue for a thread to be scheduled into. Takes affinity mask into diff --git a/src/task/thread.rs b/src/task/thread.rs index c55563cd..a7c1bfe8 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -22,12 +22,11 @@ use libk_util::{ }; use crate::{ - arch::{CpuAccess, LocalCpuAccess}, mem::{process::ProcessAddressSpace, ForeignPointer}, - task::{context::TaskContextImpl, Cpu}, + task::context::TaskContextImpl, }; -use super::{context::TaskFrame, process::Process, sched::CpuQueue, TaskContext}; +use super::{context::TaskFrame, process::Process, sched::CpuQueue, Cpu, TaskContext}; /// Represents the states a thread can be at some point in time #[atomic_enum] @@ -568,6 +567,8 @@ fn __exit_current(t: &CurrentThread, code: ExitCode) -> ! { #[no_mangle] unsafe fn __yield() { - let _irq = IrqGuard::acquire(); - Cpu::local().queue().yield_cpu(); + // XXX + todo!(); + // let _irq = IrqGuard::acquire(); + // Cpu::local().queue().yield_cpu(); } From f424c385b29a604b5cdb95ce63e9208b8af1303b Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 9 Feb 2024 14:20:13 +0200 Subject: [PATCH 187/211] arch: move contexts/frames to arch-specific crates --- .../aarch64 => arch/aarch64/src}/context.S | 6 +- arch/aarch64/src/context.rs | 242 ++++++++ arch/aarch64/src/lib.rs | 4 +- arch/interface/src/lib.rs | 2 +- arch/interface/src/mem/mod.rs | 9 + arch/interface/src/task.rs | 166 ++++++ arch/src/lib.rs | 2 +- .../arch/x86_64 => arch/x86_64/src}/context.S | 3 +- arch/x86_64/src/context.rs | 525 ++++++++++++++++++ arch/x86_64/src/lib.rs | 4 +- libk-mm/src/phys/mod.rs | 20 +- libk/src/lib.rs | 3 +- libk/src/thread.rs | 40 +- src/arch/aarch64/context.rs | 181 ------ src/arch/aarch64/exception.rs | 143 +---- src/arch/aarch64/mod.rs | 1 - src/arch/x86_64/apic/mod.rs | 3 +- src/arch/x86_64/context.rs | 264 --------- src/arch/x86_64/exception.rs | 278 ++-------- src/arch/x86_64/mod.rs | 2 - src/arch/x86_64/syscall.rs | 124 +---- src/proc/exec.rs | 13 +- src/task/context.rs | 118 ---- src/task/mod.rs | 19 +- src/task/process.rs | 17 +- src/task/sched.rs | 15 +- src/task/thread.rs | 40 +- 27 files changed, 1112 insertions(+), 1132 deletions(-) rename {src/arch/aarch64 => arch/aarch64/src}/context.S (93%) create mode 100644 arch/aarch64/src/context.rs rename {src/arch/x86_64 => arch/x86_64/src}/context.S (97%) create mode 100644 arch/x86_64/src/context.rs delete mode 100644 src/arch/aarch64/context.rs delete mode 100644 src/arch/x86_64/context.rs diff --git a/src/arch/aarch64/context.S b/arch/aarch64/src/context.S similarity index 93% rename from src/arch/aarch64/context.S rename to arch/aarch64/src/context.S index 04bc0eaf..581cf4cf 100644 --- a/src/arch/aarch64/context.S +++ b/arch/aarch64/src/context.S @@ -1,5 +1,9 @@ .global __aarch64_enter_task .global __aarch64_switch_task +.global __aarch64_switch_task_and_drop + +.global __aarch64_task_enter_kernel +.global __aarch64_task_enter_user .section .text @@ -105,7 +109,7 @@ __aarch64_switch_task_and_drop: mov sp, x0 mov x0, x1 - bl __aarch64_drop_context + bl __arch_drop_thread LOAD_TASK_STATE diff --git a/arch/aarch64/src/context.rs b/arch/aarch64/src/context.rs new file mode 100644 index 00000000..503c10d6 --- /dev/null +++ b/arch/aarch64/src/context.rs @@ -0,0 +1,242 @@ +//! AArch64-specific task context implementation +use core::{arch::global_asm, cell::UnsafeCell, fmt, marker::PhantomData}; + +use kernel_arch_interface::{ + mem::{KernelTableManager, PhysicalMemoryAllocator}, + task::{StackBuilder, TaskContext, TaskFrame}, +}; +use libk_mm_interface::address::PhysicalAddress; +use yggdrasil_abi::{arch::SavedFrame, error::Error}; + +/// Struct for register values saved when taking an exception +#[repr(C)] +pub struct ExceptionFrame { + /// General-purpose registers + pub r: [u64; 32], + /// SPSR_EL1, userspace flags register + pub spsr_el1: u64, + /// ELR_EL1, userspace program counter + pub elr_el1: u64, + /// SP_EL0, userspace stack pointer + pub sp_el0: u64, + _x: u64, + // ... +} + +#[repr(C, align(0x10))] +struct TaskContextInner { + // 0x00 + sp: usize, +} + +/// AArch64 implementation of a task context +#[allow(unused)] +pub struct TaskContextImpl< + K: KernelTableManager, + PA: PhysicalMemoryAllocator
, +> { + inner: UnsafeCell, + stack_base_phys: PhysicalAddress, + stack_size: usize, + + _alloc: PhantomData, + _table_manager: PhantomData, +} + +const COMMON_CONTEXT_SIZE: usize = 8 * 14; + +impl TaskFrame for ExceptionFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + gp_regs: self.r, + spsr_el1: self.spsr_el1, + elr_el1: self.elr_el1, + sp_el0: self.sp_el0, + } + } + + fn restore(&mut self, saved: &SavedFrame) { + self.r = saved.gp_regs; + self.spsr_el1 = saved.spsr_el1; + self.elr_el1 = saved.elr_el1; + self.sp_el0 = saved.sp_el0; + } + + fn argument(&self) -> u64 { + self.r[0] + } + + fn user_ip(&self) -> usize { + self.elr_el1 as _ + } + + fn user_sp(&self) -> usize { + self.sp_el0 as _ + } + + fn set_argument(&mut self, value: u64) { + self.r[0] = value; + } + + fn set_return_value(&mut self, value: u64) { + self.r[0] = value; + } + + fn set_user_ip(&mut self, value: usize) { + self.elr_el1 = value as _; + } + + fn set_user_sp(&mut self, value: usize) { + self.sp_el0 = value as _; + } +} + +impl fmt::Debug for ExceptionFrame { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for i in (0..32).step_by(2) { + write!( + f, + "x{:<2} = {:#020x}\tx{:<2} = {:#020x}", + i, + self.r[i], + i + 1, + self.r[i + 1] + )?; + if i != 30 { + f.write_str("\n")?; + } + } + + Ok(()) + } +} + +unsafe impl> Sync + for TaskContextImpl +{ +} + +impl> + TaskContext for TaskContextImpl +{ + const USER_STACK_EXTRA_ALIGN: usize = 0; + const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; + + fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { + const KERNEL_TASK_PAGES: usize = 8; + let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?; + let stack_base = stack_base_phys.raw_virtualize::(); + + let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); + + // Entry and argument + stack.push(entry as _); + stack.push(arg); + + setup_common_context(&mut stack, __aarch64_task_enter_kernel as _, 0, 0); + + let sp = stack.build(); + + // TODO stack is leaked + + Ok(Self { + inner: UnsafeCell::new(TaskContextInner { sp }), + stack_base_phys, + stack_size: KERNEL_TASK_PAGES * 0x1000, + + _alloc: PhantomData, + _table_manager: PhantomData, + }) + } + + fn user( + entry: usize, + arg: usize, + ttbr0: u64, + user_stack_sp: usize, + tpidr_el0: usize, + ) -> Result { + const USER_TASK_PAGES: usize = 16; + let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.raw_virtualize::(); + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + stack.push(entry as _); + stack.push(arg); + stack.push(0); + stack.push(user_stack_sp); + + setup_common_context( + &mut stack, + __aarch64_task_enter_user as _, + ttbr0, + tpidr_el0 as _, + ); + + let sp = stack.build(); + + Ok(Self { + inner: UnsafeCell::new(TaskContextInner { sp }), + stack_base_phys, + stack_size: USER_TASK_PAGES * 0x1000, + + _alloc: PhantomData, + _table_manager: PhantomData, + }) + } + + unsafe fn enter(&self) -> ! { + __aarch64_enter_task(self.inner.get()) + } + + unsafe fn switch(&self, from: &Self) { + __aarch64_switch_task(self.inner.get(), from.inner.get()) + } + + unsafe fn switch_and_drop(&self, thread: *const ()) { + __aarch64_switch_task_and_drop(self.inner.get(), thread); + } +} + +impl> Drop + for TaskContextImpl +{ + fn drop(&mut self) { + assert_eq!(self.stack_size % 0x1000, 0); + + for offset in (0..self.stack_size).step_by(0x1000) { + unsafe { + PA::free_page(self.stack_base_phys.add(offset)); + } + } + } +} + +fn setup_common_context(builder: &mut StackBuilder, entry: usize, ttbr0: u64, tpidr_el0: u64) { + builder.push(ttbr0 as _); // ttbr0_el1 + builder.push(tpidr_el0 as _); // tpidr_el0 + + builder.push(entry); // x30/lr + builder.push(0); // x29 + builder.push(0); // x28 + builder.push(0); // x27 + builder.push(0); // x26 + builder.push(0); // x25 + builder.push(0); // x24 + builder.push(0); // x23 + builder.push(0); // x22 + builder.push(0); // x21 + builder.push(0); // x20 + builder.push(0); // x19 +} + +extern "C" { + fn __aarch64_enter_task(to: *mut TaskContextInner) -> !; + fn __aarch64_switch_task(to: *mut TaskContextInner, from: *mut TaskContextInner); + fn __aarch64_switch_task_and_drop(to: *mut TaskContextInner, thread: *const ()) -> !; + fn __aarch64_task_enter_kernel(); + fn __aarch64_task_enter_user(); +} + +global_asm!(include_str!("context.S"), context_size = const COMMON_CONTEXT_SIZE); diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index 76f14381..0311809b 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -1,5 +1,5 @@ #![no_std] -#![feature(effects, strict_provenance)] +#![feature(effects, strict_provenance, asm_const)] extern crate alloc; @@ -9,8 +9,10 @@ use device_api::interrupt::{LocalInterruptController, MessageInterruptController use kernel_arch_interface::{cpu::IpiQueue, util::OneTimeInit, Architecture}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +pub mod context; pub mod mem; +pub use context::TaskContextImpl; pub use mem::{process::ProcessAddressSpaceImpl, KernelTableManagerImpl}; pub struct ArchitectureImpl; diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs index e210aacc..4c92bab6 100644 --- a/arch/interface/src/lib.rs +++ b/arch/interface/src/lib.rs @@ -1,5 +1,5 @@ #![no_std] -#![feature(step_trait, effects, const_trait_impl)] +#![feature(step_trait, effects, const_trait_impl, never_type)] use alloc::vec::Vec; use cpu::IpiQueue; diff --git a/arch/interface/src/mem/mod.rs b/arch/interface/src/mem/mod.rs index 5dcec697..73cdf364 100644 --- a/arch/interface/src/mem/mod.rs +++ b/arch/interface/src/mem/mod.rs @@ -5,6 +5,15 @@ use yggdrasil_abi::error::Error; pub mod address; pub mod table; +pub trait PhysicalMemoryAllocator { + type Address; + + fn allocate_page() -> Result; + fn allocate_contiguous_pages(count: usize) -> Result; + + unsafe fn free_page(page: Self::Address); +} + #[derive(Debug, Default, Clone, Copy)] pub enum DeviceMemoryCaching { #[default] diff --git a/arch/interface/src/task.rs b/arch/interface/src/task.rs index 230b2d25..d011e71a 100644 --- a/arch/interface/src/task.rs +++ b/arch/interface/src/task.rs @@ -1,3 +1,169 @@ +use core::fmt; + +use alloc::boxed::Box; +use yggdrasil_abi::{arch::SavedFrame, error::Error, process::ExitCode}; + +use crate::mem::{KernelTableManager, PhysicalMemoryAllocator}; + pub trait Scheduler { type ThreadId: Copy; } + +/// Conversion trait to allow multiple kernel closure return types +pub trait Termination { + /// Converts the closure return type into [ExitCode] + fn into_exit_code(self) -> ExitCode; +} + +/// Interface for task state save/restore mechanisms +pub trait TaskFrame { + /// Creates a "snapshot" of a exception/syscall frame + fn store(&self) -> SavedFrame; + + /// Restores the exception/syscall frame from its saved state + fn restore(&mut self, saved: &SavedFrame); + + /// Replaces the return value in the frame (or does nothing, if the frame is not a part of a + /// syscall signal handler) + fn set_return_value(&mut self, value: u64); + + /// Replaces the userspace stack pointer in the frame + fn set_user_sp(&mut self, value: usize); + + /// Replaces the userspace instruction pointer in the frame + fn set_user_ip(&mut self, value: usize); + + /// Replaces the argument in the frame + fn set_argument(&mut self, value: u64); + + /// Returns the argument (if any) of the frame being processed + fn argument(&self) -> u64; + + /// Returns the userspace stack pointer + fn user_sp(&self) -> usize; + /// Returns the userspace instruction pointer + fn user_ip(&self) -> usize; +} + +/// Interface for performing context fork operations +pub trait ForkFrame: Sized { + type Context: TaskContext; + + /// Constructs a "forked" task context by copying the registers from this one and supplying a + /// new address space to it. + /// + /// # Safety + /// + /// Unsafe: accepts raw frames and address space address. + unsafe fn fork(&self, address_space: u64) -> Result; + + /// Replaces the return value inside the frame with a new one + fn set_return_value(&mut self, value: u64); +} + +/// Platform-specific task context implementation +pub trait TaskContext: Sized { + /// Number of bytes to offset the signal stack pointer by + const SIGNAL_STACK_EXTRA_ALIGN: usize; + /// Number of bytes to offset the user stack pointer by + const USER_STACK_EXTRA_ALIGN: usize; + + /// Constructs a kernel-space task context + fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result; + + /// Constructs a user thread context. The caller is responsible for allocating the userspace + /// stack and setting up a valid address space for the context. + fn user( + entry: usize, + arg: usize, + cr3: u64, + user_stack_sp: usize, + tls_address: usize, + ) -> Result; + + /// Performs an entry into a context. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + unsafe fn enter(&self) -> !; + + /// Performs a context switch between two contexts. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code. + unsafe fn switch(&self, from: &Self); + + /// Performs a context switch and drops the source thread. + /// + /// # Safety + /// + /// Only meant to be called from the scheduler code after the `thread` has terminated. + unsafe fn switch_and_drop(&self, thread: *const ()); + + // XXX + /// Constructs a safe wrapper process to execute a kernel-space closure + fn kernel_closure ! + Send + 'static>(f: F) -> Result { + extern "C" fn closure_wrapper ! + Send + 'static>(closure_addr: usize) -> ! { + let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; + closure() + } + + let closure = Box::new(f); + Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) + } +} + +pub struct StackBuilder { + base: usize, + sp: usize, +} + +impl StackBuilder { + pub fn new(base: usize, size: usize) -> Self { + Self { + base, + sp: base + size, + } + } + + pub fn push(&mut self, value: usize) { + if self.sp == self.base { + panic!(); + } + self.sp -= 8; + unsafe { + (self.sp as *mut usize).write_volatile(value); + } + } + + pub fn build(self) -> usize { + self.sp + } +} + +impl Termination for Result { + fn into_exit_code(self) -> ExitCode { + match self { + Ok(_) => ExitCode::SUCCESS, + Err(_err) => { + // XXX + // log::warn!("Kernel thread failed: {:?}", err); + ExitCode::Exited(1) + } + } + } +} + +impl Termination for ExitCode { + fn into_exit_code(self) -> ExitCode { + self + } +} + +impl Termination for () { + fn into_exit_code(self) -> ExitCode { + ExitCode::SUCCESS + } +} diff --git a/arch/src/lib.rs b/arch/src/lib.rs index 9955b9c2..a1f0bffd 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -29,7 +29,7 @@ cfg_if! { } } -pub use imp::{ArchitectureImpl, KernelTableManagerImpl, ProcessAddressSpaceImpl}; +pub use imp::{ArchitectureImpl, KernelTableManagerImpl, ProcessAddressSpaceImpl, TaskContextImpl}; pub use kernel_arch_interface::{guard, mem, sync, task, util, Architecture}; diff --git a/src/arch/x86_64/context.S b/arch/x86_64/src/context.S similarity index 97% rename from src/arch/x86_64/context.S rename to arch/x86_64/src/context.S index 27d7e389..147c8ce3 100644 --- a/src/arch/x86_64/context.S +++ b/arch/x86_64/src/context.S @@ -57,6 +57,7 @@ .global __x86_64_task_enter_from_fork .global __x86_64_enter_task .global __x86_64_switch_task +.global __x86_64_switch_and_drop .section .text @@ -162,7 +163,7 @@ __x86_64_switch_and_drop: mov %rax, 4(%rdi) mov %rsi, %rdi - call __x86_64_drop_thread + call __arch_drop_thread LOAD_TASK_STATE diff --git a/arch/x86_64/src/context.rs b/arch/x86_64/src/context.rs new file mode 100644 index 00000000..bef56a63 --- /dev/null +++ b/arch/x86_64/src/context.rs @@ -0,0 +1,525 @@ +use core::{arch::global_asm, cell::UnsafeCell, marker::PhantomData}; + +use kernel_arch_interface::{ + mem::{KernelTableManager, PhysicalMemoryAllocator}, + task::{ForkFrame, StackBuilder, TaskContext, TaskFrame}, +}; +use libk_mm_interface::address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}; +use yggdrasil_abi::{arch::SavedFrame, error::Error}; + +use crate::{mem::KERNEL_TABLES, registers::FpuContext}; + +/// Frame saved onto the stack when taking an IRQ +#[derive(Debug)] +#[repr(C)] +pub struct IrqFrame { + pub rax: u64, + pub rcx: u64, + pub rdx: u64, + pub rbx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, + + pub rip: u64, + pub cs: u64, + pub rflags: u64, + pub rsp: u64, + pub ss: u64, +} + +/// Set of registers saved when taking an exception/interrupt +#[derive(Debug)] +#[repr(C)] +pub struct ExceptionFrame { + pub rax: u64, + pub rcx: u64, + pub rdx: u64, + pub rbx: u64, + pub rsi: u64, + pub rdi: u64, + pub rbp: u64, + pub r8: u64, + pub r9: u64, + pub r10: u64, + pub r11: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, + + pub exc_number: u64, + pub exc_code: u64, + + pub rip: u64, + pub cs: u64, + pub rflags: u64, + pub rsp: u64, + pub ss: u64, +} + +/// Set of registers saved when taking a syscall instruction +#[derive(Debug)] +#[repr(C)] +pub struct SyscallFrame { + pub rax: u64, + pub args: [u64; 6], + + pub rcx: u64, + pub r11: u64, + + pub user_ip: u64, + pub user_sp: u64, + pub user_flags: u64, + + pub rbx: u64, + pub rbp: u64, + pub r12: u64, + pub r13: u64, + pub r14: u64, + pub r15: u64, +} + +#[repr(C, align(0x10))] +struct Inner { + // 0x00 + sp: usize, + // 0x08 + tss_rsp0: usize, +} + +/// x86-64 implementation of a task context +#[allow(dead_code)] +pub struct TaskContextImpl< + K: KernelTableManager, + PA: PhysicalMemoryAllocator
, +> { + inner: UnsafeCell, + fpu_context: UnsafeCell, + stack_base_phys: PhysicalAddress, + stack_size: usize, + + _alloc: PhantomData, + _table_manager: PhantomData, +} + +// 8 registers + return address (which is not included) +const COMMON_CONTEXT_SIZE: usize = 8 * 8; + +impl TaskFrame for IrqFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.rdx, + rbx: self.rbx, + rsi: self.rsi, + rdi: self.rdi, + rbp: self.rbp, + r8: self.r8, + r9: self.r9, + r10: self.r10, + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.rip, + user_sp: self.rsp, + rflags: self.rflags, + } + } + + fn restore(&mut self, _saved: &SavedFrame) { + todo!() + } + + fn argument(&self) -> u64 { + self.rdi as _ + } + + fn user_ip(&self) -> usize { + self.rip as _ + } + + fn user_sp(&self) -> usize { + self.rsp as _ + } + + fn set_argument(&mut self, value: u64) { + self.rdi = value; + } + + fn set_return_value(&mut self, value: u64) { + self.rax = value; + } + + fn set_user_ip(&mut self, value: usize) { + self.rip = value as _; + } + + fn set_user_sp(&mut self, value: usize) { + self.rsp = value as _; + } +} + +impl TaskFrame for ExceptionFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.rdx, + rbx: self.rbx, + rsi: self.rsi, + rdi: self.rdi, + rbp: self.rbp, + r8: self.r8, + r9: self.r9, + r10: self.r10, + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.rip, + user_sp: self.rsp, + rflags: self.rflags, + } + } + + fn restore(&mut self, _saved: &SavedFrame) { + todo!() + } + + fn argument(&self) -> u64 { + 0 + } + + fn user_sp(&self) -> usize { + self.rsp as _ + } + + fn user_ip(&self) -> usize { + self.rip as _ + } + + fn set_user_sp(&mut self, value: usize) { + self.rsp = value as _; + } + + fn set_user_ip(&mut self, value: usize) { + self.rip = value as _; + } + + fn set_return_value(&mut self, _value: u64) { + // Not in syscall, do not overwrite + } + + fn set_argument(&mut self, value: u64) { + self.rdi = value; + } +} + +impl> ForkFrame + for SyscallFrame +{ + type Context = TaskContextImpl; + + unsafe fn fork(&self, address_space: u64) -> Result, Error> { + TaskContextImpl::from_syscall_frame(self, address_space) + } + + fn set_return_value(&mut self, value: u64) { + self.rax = value; + } +} + +impl TaskFrame for SyscallFrame { + fn store(&self) -> SavedFrame { + SavedFrame { + rax: self.rax, + rcx: self.rcx, + rdx: self.args[2], + rbx: self.rbx, + rsi: self.args[1], + rdi: self.args[0], + rbp: self.rbp, + r8: self.args[4], + r9: self.args[5], + r10: self.args[3], + r11: self.r11, + r12: self.r12, + r13: self.r13, + r14: self.r14, + r15: self.r15, + user_ip: self.user_ip, + user_sp: self.user_sp, + rflags: self.user_flags, + } + } + + fn restore(&mut self, saved: &SavedFrame) { + self.rax = saved.rax; + self.args[0] = saved.rdi; + self.args[1] = saved.rsi; + self.args[2] = saved.rdx; + self.args[3] = saved.r10; + self.args[4] = saved.r8; + self.args[5] = saved.r9; + + self.rcx = saved.rcx; + self.r11 = saved.r11; + + self.user_ip = saved.user_ip; + self.user_sp = saved.user_sp; + self.user_flags = saved.rflags; + + self.rbx = saved.rbx; + self.rbp = saved.rbp; + self.r12 = saved.r12; + self.r13 = saved.r13; + self.r14 = saved.r14; + self.r15 = saved.r15; + } + + fn argument(&self) -> u64 { + self.args[0] + } + + fn user_sp(&self) -> usize { + self.user_sp as _ + } + + fn user_ip(&self) -> usize { + self.user_ip as _ + } + + fn set_user_sp(&mut self, value: usize) { + self.user_sp = value as _; + } + + fn set_user_ip(&mut self, value: usize) { + self.user_ip = value as _; + } + + fn set_return_value(&mut self, value: u64) { + self.rax = value; + } + + fn set_argument(&mut self, value: u64) { + self.args[0] = value; + } +} + +impl> + TaskContextImpl +{ + /// Constructs a new task context from a "forked" syscall frame + pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result { + const USER_TASK_PAGES: usize = 8; + + let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.raw_virtualize::(); + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + // iretq frame + stack.push(0x1B); + stack.push(frame.user_sp as _); + stack.push(0x200); + stack.push(0x23); + stack.push(frame.user_ip as _); + + stack.push(frame.args[5] as _); // r9 + stack.push(frame.args[4] as _); // r8 + stack.push(frame.args[3] as _); // r10 + stack.push(frame.args[2] as _); // rdx + stack.push(frame.args[1] as _); // rsi + stack.push(frame.args[0] as _); // rdi + + // callee-saved registers + stack.push(__x86_64_task_enter_from_fork as _); + + stack.push(cr3 as _); + + stack.push(frame.rbp as _); + stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame + stack.push(frame.r15 as _); + stack.push(frame.r14 as _); + stack.push(frame.r13 as _); + stack.push(frame.r12 as _); + stack.push(frame.rbx as _); + + let sp = stack.build(); + let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; + + Ok(Self { + inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + fpu_context: UnsafeCell::new(FpuContext::new()), + stack_base_phys, + stack_size: USER_TASK_PAGES * 0x1000, + _alloc: PhantomData, + _table_manager: PhantomData, + }) + } +} + +unsafe impl> Sync + for TaskContextImpl +{ +} + +impl> + TaskContext for TaskContextImpl +{ + const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; + const USER_STACK_EXTRA_ALIGN: usize = 8; + + fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { + const KERNEL_TASK_PAGES: usize = 32; + + let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?; + let stack_base = stack_base_phys.raw_virtualize::(); + + let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); + + // Entry and argument + stack.push(entry as _); + stack.push(arg); + + // XXX + setup_common_context( + &mut stack, + __x86_64_task_enter_kernel as _, + unsafe { KERNEL_TABLES.as_physical_address().into_raw() }, + 0, + ); + + let sp = stack.build(); + + // TODO stack is leaked + + Ok(Self { + inner: UnsafeCell::new(Inner { sp, tss_rsp0: 0 }), + fpu_context: UnsafeCell::new(FpuContext::new()), + stack_base_phys, + stack_size: KERNEL_TASK_PAGES * 0x1000, + + _alloc: PhantomData, + _table_manager: PhantomData, + }) + } + + fn user( + entry: usize, + arg: usize, + cr3: u64, + user_stack_sp: usize, + fs_base: usize, + ) -> Result { + const USER_TASK_PAGES: usize = 8; + + let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.raw_virtualize::(); + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + stack.push(entry as _); + stack.push(arg); + stack.push(user_stack_sp); + + setup_common_context(&mut stack, __x86_64_task_enter_user as _, cr3, fs_base); + + let sp = stack.build(); + let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; + + Ok(Self { + inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + fpu_context: UnsafeCell::new(FpuContext::new()), + stack_base_phys, + stack_size: USER_TASK_PAGES * 0x1000, + + _alloc: PhantomData, + _table_manager: PhantomData, + }) + } + + unsafe fn enter(&self) -> ! { + FpuContext::restore(self.fpu_context.get()); + + __x86_64_enter_task(self.inner.get()) + } + + unsafe fn switch(&self, from: &Self) { + let dst = self.inner.get(); + let src = from.inner.get(); + + if dst != src { + // Save the old context + FpuContext::save(from.fpu_context.get()); + // Load next context + FpuContext::restore(self.fpu_context.get()); + + __x86_64_switch_task(dst, src); + } + } + + unsafe fn switch_and_drop(&self, thread: *const ()) { + let dst = self.inner.get(); + + FpuContext::restore(self.fpu_context.get()); + + __x86_64_switch_and_drop(dst, thread) + } +} + +impl> Drop + for TaskContextImpl +{ + fn drop(&mut self) { + assert_eq!(self.stack_size % 0x1000, 0); + + for offset in (0..self.stack_size).step_by(0x1000) { + unsafe { + PA::free_page(self.stack_base_phys.add(offset)); + } + } + } +} + +fn setup_common_context(builder: &mut StackBuilder, entry: usize, cr3: u64, fs_base: usize) { + builder.push(entry); + + builder.push(cr3 as _); + + builder.push(0); // %rbp + builder.push(fs_base); // %fs_base + builder.push(0); // %r15 + builder.push(0); // %r14 + builder.push(0); // %r13 + builder.push(0); // %r12 + builder.push(0); // %rbx +} + +extern "C" { + fn __x86_64_task_enter_kernel(); + fn __x86_64_task_enter_user(); + fn __x86_64_task_enter_from_fork(); + fn __x86_64_enter_task(to: *mut Inner) -> !; + fn __x86_64_switch_task(to: *mut Inner, from: *mut Inner); + fn __x86_64_switch_and_drop(to: *mut Inner, from: *const ()); +} + +global_asm!( + include_str!("context.S"), + context_size = const COMMON_CONTEXT_SIZE, + options(att_syntax) +); diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index fcc8dee5..0fe9aae8 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -1,5 +1,5 @@ #![no_std] -#![feature(effects, strict_provenance)] +#![feature(effects, strict_provenance, asm_const)] extern crate alloc; @@ -10,9 +10,11 @@ use libk_mm_interface::address::PhysicalAddress; use registers::MSR_IA32_KERNEL_GS_BASE; use tock_registers::interfaces::Writeable; +pub mod context; pub mod mem; pub mod registers; +pub use context::TaskContextImpl; pub use mem::{process::ProcessAddressSpaceImpl, KernelTableManagerImpl}; pub struct ArchitectureImpl; diff --git a/libk-mm/src/phys/mod.rs b/libk-mm/src/phys/mod.rs index f531b346..3b28a567 100644 --- a/libk-mm/src/phys/mod.rs +++ b/libk-mm/src/phys/mod.rs @@ -1,6 +1,6 @@ use core::ops::Range; -use kernel_arch::absolute_address; +use kernel_arch::{absolute_address, mem::PhysicalMemoryAllocator}; use libk_mm_interface::address::{FromRaw, IntoRaw, PhysicalAddress}; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; @@ -18,6 +18,8 @@ use self::manager::{PhysicalMemoryManager, TRACKED_PAGE_LIMIT}; mod manager; pub mod reserved; +pub struct GlobalPhysicalAllocator; + /// Defines an usable memory region #[derive(Clone, Copy, Debug)] pub struct PhysicalMemoryRegion { @@ -58,6 +60,22 @@ impl PhysicalMemoryRegion { } } +impl PhysicalMemoryAllocator for GlobalPhysicalAllocator { + type Address = PhysicalAddress; + + fn allocate_page() -> Result { + alloc_page() + } + + fn allocate_contiguous_pages(count: usize) -> Result { + alloc_pages_contiguous(count) + } + + unsafe fn free_page(page: Self::Address) { + free_page(page) + } +} + /// Allocates a single physical page from the global manager pub fn alloc_page() -> Result { PHYSICAL_MEMORY.get().lock().alloc_page() diff --git a/libk/src/lib.rs b/libk/src/lib.rs index ee9ebf5c..472b66b0 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -12,7 +12,8 @@ maybe_uninit_uninit_array, const_maybe_uninit_uninit_array, new_uninit, - inline_const + inline_const, + trait_alias )] use core::time::Duration; diff --git a/libk/src/thread.rs b/libk/src/thread.rs index 92742d86..0a30fa63 100644 --- a/libk/src/thread.rs +++ b/libk/src/thread.rs @@ -1,4 +1,4 @@ -use core::{fmt, ops::Deref}; +use core::ops::Deref; use alloc::{ boxed::Box, @@ -6,46 +6,24 @@ use alloc::{ sync::{Arc, Weak}, }; use futures_util::task::ArcWake; +use kernel_arch::KernelTableManagerImpl; +use libk_mm::phys::GlobalPhysicalAllocator; use libk_util::sync::IrqGuard; use yggdrasil_abi::{error::Error, process::ExitCode}; use crate::api; +pub use kernel_arch::task::{TaskContext, TaskFrame, Termination}; + +pub type TaskContextImpl = + kernel_arch::TaskContextImpl; +pub trait ForkFrame = kernel_arch::task::ForkFrame; + #[repr(C)] pub(crate) struct Thread(!); #[repr(C)] pub(crate) struct CurrentThread(Arc, IrqGuard); -/// Conversion trait to allow multiple kernel closure return types -pub trait Termination { - /// Converts the closure return type into [ExitCode] - fn into_exit_code(self) -> ExitCode; -} - -impl Termination for Result { - fn into_exit_code(self) -> ExitCode { - match self { - Ok(_) => ExitCode::SUCCESS, - Err(err) => { - log::warn!("Kernel thread failed: {:?}", err); - ExitCode::Exited(1) - } - } - } -} - -impl Termination for ExitCode { - fn into_exit_code(self) -> ExitCode { - self - } -} - -impl Termination for () { - fn into_exit_code(self) -> ExitCode { - ExitCode::SUCCESS - } -} - impl Thread { pub fn spawn, T: Termination, F: FnOnce() -> T + Send + 'static>( name: S, diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs deleted file mode 100644 index 4c61955d..00000000 --- a/src/arch/aarch64/context.rs +++ /dev/null @@ -1,181 +0,0 @@ -//! AArch64-specific task context implementation -use core::{arch::global_asm, cell::UnsafeCell}; - -use abi::error::Error; -use alloc::sync::Arc; -use libk_mm::{ - address::{PhysicalAddress, Virtualize}, - phys, -}; - -use crate::task::{context::TaskContextImpl, thread::Thread}; - -struct StackBuilder { - base: usize, - sp: usize, -} - -#[repr(C, align(0x10))] -struct TaskContextInner { - // 0x00 - sp: usize, -} - -/// AArch64 implementation of a task context -#[allow(unused)] -pub struct TaskContext { - inner: UnsafeCell, - stack_base_phys: PhysicalAddress, - stack_size: usize, -} - -const COMMON_CONTEXT_SIZE: usize = 8 * 14; - -unsafe impl Sync for TaskContext {} - -impl StackBuilder { - fn new(base: usize, size: usize) -> Self { - Self { - base, - sp: base + size, - } - } - - fn push(&mut self, value: usize) { - if self.sp == self.base { - panic!(); - } - self.sp -= 8; - unsafe { - (self.sp as *mut usize).write_volatile(value); - } - } - - fn _skip(&mut self, count: usize) { - self.sp -= count * 8; - if self.sp < self.base { - panic!(); - } - } - - fn build(self) -> usize { - self.sp - } - - fn init_common(&mut self, entry: usize, ttbr0: u64, tpidr_el0: u64) { - self.push(ttbr0 as _); // ttbr0_el1 - self.push(tpidr_el0 as _); // tpidr_el0 - - self.push(entry); // x30/lr - self.push(0); // x29 - self.push(0); // x28 - self.push(0); // x27 - self.push(0); // x26 - self.push(0); // x25 - self.push(0); // x24 - self.push(0); // x23 - self.push(0); // x22 - self.push(0); // x21 - self.push(0); // x20 - self.push(0); // x19 - } -} - -impl TaskContextImpl for TaskContext { - const USER_STACK_EXTRA_ALIGN: usize = 0; - const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; - - fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { - const KERNEL_TASK_PAGES: usize = 8; - let stack_base_phys = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize(); - - let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); - - // Entry and argument - stack.push(entry as _); - stack.push(arg); - - stack.init_common(__aarch64_task_enter_kernel as _, 0, 0); - - let sp = stack.build(); - - // TODO stack is leaked - - Ok(Self { - inner: UnsafeCell::new(TaskContextInner { sp }), - stack_base_phys, - stack_size: KERNEL_TASK_PAGES * 0x1000, - }) - } - - fn user( - entry: usize, - arg: usize, - ttbr0: u64, - user_stack_sp: usize, - tpidr_el0: usize, - ) -> Result { - const USER_TASK_PAGES: usize = 16; - let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize(); - - let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); - - stack.push(entry as _); - stack.push(arg); - stack.push(0); - stack.push(user_stack_sp); - - stack.init_common(__aarch64_task_enter_user as _, ttbr0, tpidr_el0 as _); - - let sp = stack.build(); - - Ok(Self { - inner: UnsafeCell::new(TaskContextInner { sp }), - stack_base_phys, - stack_size: USER_TASK_PAGES * 0x1000, - }) - } - - unsafe fn enter(&self) -> ! { - __aarch64_enter_task(self.inner.get()) - } - - unsafe fn switch(&self, from: &Self) { - __aarch64_switch_task(self.inner.get(), from.inner.get()) - } - - unsafe fn switch_and_drop(&self, thread: Arc) { - __aarch64_switch_task_and_drop(self.inner.get(), Arc::into_raw(thread) as _); - } -} - -impl Drop for TaskContext { - fn drop(&mut self) { - assert_eq!(self.stack_size % 0x1000, 0); - - for offset in (0..self.stack_size).step_by(0x1000) { - unsafe { - phys::free_page(self.stack_base_phys.add(offset)); - } - } - } -} - -#[no_mangle] -unsafe extern "C" fn __aarch64_drop_context(thread_ptr: *const Thread) { - let thread = Arc::from_raw(thread_ptr); - Thread::remove_from_list(thread.id); - Arc::decrement_strong_count(thread_ptr); -} - -extern "C" { - fn __aarch64_enter_task(to: *mut TaskContextInner) -> !; - fn __aarch64_switch_task(to: *mut TaskContextInner, from: *mut TaskContextInner); - fn __aarch64_switch_task_and_drop(to: *mut TaskContextInner, thread: *const ()) -> !; - fn __aarch64_task_enter_kernel(); - fn __aarch64_task_enter_user(); -} - -global_asm!(include_str!("context.S"), context_size = const COMMON_CONTEXT_SIZE); diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 258e7dfa..2f2c91a6 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -1,5 +1,6 @@ //! Exception and interrupt management functions -use core::{arch::global_asm, fmt}; + +use core::arch::global_asm; use aarch64_cpu::{ asm::barrier, @@ -9,100 +10,15 @@ use aarch64_cpu::{ }, }; use abi::{ - arch::SavedFrame, process::{Signal, SignalEntryData}, syscall::SyscallFunction, }; -use kernel_arch::{Architecture, ArchitectureImpl}; +use kernel_arch::{task::TaskFrame, Architecture, ArchitectureImpl}; +use kernel_arch_aarch64::context::ExceptionFrame; use libk::device::external_interrupt_controller; use tock_registers::interfaces::{Readable, Writeable}; -use crate::{ - debug::LogLevel, - syscall::raw_syscall_handler, - task::{context::TaskFrame, thread::Thread}, -}; - -/// Struct for register values saved when taking an exception -#[repr(C)] -pub struct ExceptionFrame { - /// General-purpose registers - pub r: [u64; 32], - /// SPSR_EL1, userspace flags register - pub spsr_el1: u64, - /// ELR_EL1, userspace program counter - pub elr_el1: u64, - /// SP_EL0, userspace stack pointer - pub sp_el0: u64, - _x: u64, - // ... -} - -impl TaskFrame for ExceptionFrame { - fn store(&self) -> SavedFrame { - SavedFrame { - gp_regs: self.r, - spsr_el1: self.spsr_el1, - elr_el1: self.elr_el1, - sp_el0: self.sp_el0, - } - } - - fn restore(&mut self, saved: &SavedFrame) { - self.r = saved.gp_regs; - self.spsr_el1 = saved.spsr_el1; - self.elr_el1 = saved.elr_el1; - self.sp_el0 = saved.sp_el0; - } - - fn argument(&self) -> u64 { - self.r[0] - } - - fn user_ip(&self) -> usize { - self.elr_el1 as _ - } - - fn user_sp(&self) -> usize { - self.sp_el0 as _ - } - - fn set_argument(&mut self, value: u64) { - self.r[0] = value; - } - - fn set_return_value(&mut self, value: u64) { - self.r[0] = value; - } - - fn set_user_ip(&mut self, value: usize) { - self.elr_el1 = value as _; - } - - fn set_user_sp(&mut self, value: usize) { - self.sp_el0 = value as _; - } -} - -impl fmt::Debug for ExceptionFrame { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for i in (0..32).step_by(2) { - write!( - f, - "x{:<2} = {:#020x}\tx{:<2} = {:#020x}", - i, - self.r[i], - i + 1, - self.r[i + 1] - )?; - if i != 30 { - f.write_str("\n")?; - } - } - - Ok(()) - } -} +use crate::{debug::LogLevel, syscall::raw_syscall_handler, task::thread::Thread}; /// Initializes the exception/interrupt vectors. May be called repeatedly (though that makes no /// sense). @@ -176,15 +92,6 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { _ => (), } - // unsafe { - // let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); - // let far = FAR_EL1.get() as usize; - // space.walk(far, |level, raw| { - // log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); - // true - // }); - // } - log_print_raw!(LogLevel::Fatal, "System register dump:\n"); log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); log_print_raw!(LogLevel::Fatal, "TCR_EL1 = {:#x}\n", TCR_EL1.get()); @@ -197,26 +104,8 @@ extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) { assert!(ArchitectureImpl::interrupt_mask()); let frame = unsafe { &mut *frame }; - // let current_id = Process::get_current().map(|p| p.id()); - // log_print_raw!( - // LogLevel::Fatal, - // "sync_entry {:?}: x30 = {:#x} @ {:p}\n", - // current_id, - // frame.r[30], - // &frame.r[30] - // ); - el0_sync_inner(frame); - // let current_id = Process::get_current().map(|p| p.id()); - // log_print_raw!( - // LogLevel::Fatal, - // "sync_exit {:?}: x30 = {:#x} @ {:p}\n", - // current_id, - // frame.r[30], - // &frame.r[30] - // ); - unsafe { let thread = Thread::current(); thread.handle_pending_signals(frame); @@ -228,30 +117,8 @@ extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) { assert!(ArchitectureImpl::interrupt_mask()); let frame = unsafe { &mut *frame }; - // let current_id = Process::get_current().map(|p| p.id()); - // log_print_raw!(LogLevel::Fatal, "+++ irq_handler ENTRY +++\n"); - // log_print_raw!( - // LogLevel::Fatal, - // "{:?}: x30 = {:#x} @ {:p}\n", - // current_id, - // frame.r[30], - // &frame.r[30] - // ); - // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); - irq_common(); - // let current_id = Process::get_current().map(|p| p.id()); - // log_print_raw!(LogLevel::Fatal, "+++ irq_handler EXIT +++\n"); - // log_print_raw!( - // LogLevel::Fatal, - // "{:?}: x30 = {:#x} @ {:p}\n", - // current_id, - // frame.r[30], - // &frame.r[30] - // ); - // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); - unsafe { let thread = Thread::current(); thread.handle_pending_signals(frame); diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 73099c97..cde08fae 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -43,7 +43,6 @@ use self::gic::Gic; use super::Platform; pub mod boot; -pub mod context; pub mod exception; pub mod gic; pub mod smp; diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 60488011..555ffce0 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -2,12 +2,13 @@ use core::arch::global_asm; +use kernel_arch_x86_64::context::IrqFrame; use libk::device::{external_interrupt_controller, message_interrupt_controller}; use static_assertions::{const_assert, const_assert_eq}; use crate::task::{thread::Thread, Cpu}; -use super::exception::{self, IrqFrame}; +use super::exception; pub mod ioapic; pub mod local; diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs deleted file mode 100644 index d63e2f75..00000000 --- a/src/arch/x86_64/context.rs +++ /dev/null @@ -1,264 +0,0 @@ -//! x86-64 task context setup and switching -use core::{arch::global_asm, cell::UnsafeCell}; - -use abi::error::Error; -use alloc::sync::Arc; -use kernel_arch_x86_64::{mem::KERNEL_TABLES, registers::FpuContext}; -use libk_mm::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress, Virtualize}, - phys, -}; - -use crate::task::{context::TaskContextImpl, thread::Thread}; - -use super::syscall::SyscallFrame; - -struct StackBuilder { - base: usize, - sp: usize, -} - -#[repr(C, align(0x10))] -struct Inner { - // 0x00 - sp: usize, - // 0x08 - tss_rsp0: usize, -} - -/// x86-64 implementation of a task context -#[allow(dead_code)] -pub struct TaskContext { - inner: UnsafeCell, - fpu_context: UnsafeCell, - stack_base_phys: PhysicalAddress, - stack_size: usize, -} - -// 8 registers + return address (which is not included) -const COMMON_CONTEXT_SIZE: usize = 8 * 8; - -impl StackBuilder { - fn new(base: usize, size: usize) -> Self { - Self { - base, - sp: base + size, - } - } - - fn push(&mut self, value: usize) { - if self.sp == self.base { - panic!(); - } - self.sp -= 8; - unsafe { - (self.sp as *mut usize).write_volatile(value); - } - } - - fn _skip(&mut self, count: usize) { - self.sp -= count * 8; - if self.sp < self.base { - panic!(); - } - } - - fn build(self) -> usize { - self.sp - } - - fn init_common(&mut self, entry: usize, cr3: u64, fs_base: usize) { - self.push(entry); // address for ret - - // End of common context - self.push(cr3 as _); // %cr3 - - self.push(0); // %rbp - self.push(fs_base); // fs_base - self.push(0); // %r15 - self.push(0); // %r14 - self.push(0); // %r13 - self.push(0); // %r12 - self.push(0); // %rbx - } -} - -unsafe impl Sync for TaskContext {} - -impl TaskContext { - /// Constructs a new task context from a "forked" syscall frame - pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result { - const USER_TASK_PAGES: usize = 8; - - let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize(); - - let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); - - // iretq frame - stack.push(0x1B); - stack.push(frame.user_sp as _); - stack.push(0x200); - stack.push(0x23); - stack.push(frame.user_ip as _); - - stack.push(frame.args[5] as _); // r9 - stack.push(frame.args[4] as _); // r8 - stack.push(frame.args[3] as _); // r10 - stack.push(frame.args[2] as _); // rdx - stack.push(frame.args[1] as _); // rsi - stack.push(frame.args[0] as _); // rdi - - // callee-saved registers - stack.push(__x86_64_task_enter_from_fork as _); - - stack.push(cr3 as _); - - stack.push(frame.rbp as _); - stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame - stack.push(frame.r15 as _); - stack.push(frame.r14 as _); - stack.push(frame.r13 as _); - stack.push(frame.r12 as _); - stack.push(frame.rbx as _); - - let sp = stack.build(); - let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; - - Ok(Self { - inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), - fpu_context: UnsafeCell::new(FpuContext::new()), - stack_base_phys, - stack_size: USER_TASK_PAGES * 0x1000, - }) - } -} - -impl TaskContextImpl for TaskContext { - const SIGNAL_STACK_EXTRA_ALIGN: usize = 8; - const USER_STACK_EXTRA_ALIGN: usize = 8; - - fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { - const KERNEL_TASK_PAGES: usize = 32; - - let stack_base_phys = phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize(); - - let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); - - // Entry and argument - stack.push(entry as _); - stack.push(arg); - - stack.init_common( - __x86_64_task_enter_kernel as _, - unsafe { KERNEL_TABLES.as_physical_address().into_raw() }, - 0, - ); - - let sp = stack.build(); - - // TODO stack is leaked - - Ok(Self { - inner: UnsafeCell::new(Inner { sp, tss_rsp0: 0 }), - fpu_context: UnsafeCell::new(FpuContext::new()), - stack_base_phys, - stack_size: KERNEL_TASK_PAGES * 0x1000, - }) - } - - fn user( - entry: usize, - arg: usize, - cr3: u64, - user_stack_sp: usize, - fs_base: usize, - ) -> Result { - const USER_TASK_PAGES: usize = 8; - - let stack_base_phys = phys::alloc_pages_contiguous(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.virtualize(); - - let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); - - stack.push(entry as _); - stack.push(arg); - stack.push(user_stack_sp); - - stack.init_common(__x86_64_task_enter_user as _, cr3, fs_base); - - let sp = stack.build(); - let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; - - Ok(Self { - inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), - fpu_context: UnsafeCell::new(FpuContext::new()), - stack_base_phys, - stack_size: USER_TASK_PAGES * 0x1000, - }) - } - - unsafe fn enter(&self) -> ! { - FpuContext::restore(self.fpu_context.get()); - - __x86_64_enter_task(self.inner.get()) - } - - unsafe fn switch(&self, from: &Self) { - let dst = self.inner.get(); - let src = from.inner.get(); - - if dst != src { - // Save the old context - FpuContext::save(from.fpu_context.get()); - // Load next context - FpuContext::restore(self.fpu_context.get()); - - __x86_64_switch_task(dst, src); - } - } - - unsafe fn switch_and_drop(&self, thread: Arc) { - let dst = self.inner.get(); - - FpuContext::restore(self.fpu_context.get()); - - __x86_64_switch_and_drop(dst, Arc::into_raw(thread) as _) - } -} - -impl Drop for TaskContext { - fn drop(&mut self) { - assert_eq!(self.stack_size % 0x1000, 0); - - for offset in (0..self.stack_size).step_by(0x1000) { - unsafe { - phys::free_page(self.stack_base_phys.add(offset)); - } - } - } -} - -// TODO merge with aarch64 -#[no_mangle] -unsafe extern "C" fn __x86_64_drop_thread(thread_ptr: *const Thread) { - let thread = Arc::from_raw(thread_ptr); - Thread::remove_from_list(thread.id); - Arc::decrement_strong_count(thread_ptr); -} - -extern "C" { - fn __x86_64_task_enter_kernel(); - fn __x86_64_task_enter_user(); - fn __x86_64_task_enter_from_fork(); - fn __x86_64_enter_task(to: *mut Inner) -> !; - fn __x86_64_switch_task(to: *mut Inner, from: *mut Inner); - fn __x86_64_switch_and_drop(to: *mut Inner, from: *const ()); -} - -global_asm!( - include_str!("context.S"), - context_size = const COMMON_CONTEXT_SIZE, - options(att_syntax) -); diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 5c5d2ea7..9017bd8c 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -1,14 +1,13 @@ //! x86-64 exception and interrupt handling use core::{arch::global_asm, mem::size_of, ptr::addr_of}; -use abi::{arch::SavedFrame, primitive_enum, process::Signal}; -use kernel_arch_x86_64::registers::CR3; +use abi::{primitive_enum, process::Signal}; +use kernel_arch_x86_64::{context::ExceptionFrame, registers::CR3}; use tock_registers::interfaces::Readable; use crate::{ arch::x86_64::apic, - debug, - task::{context::TaskFrame, thread::Thread, Cpu}, + task::{thread::Thread, Cpu}, }; use super::PLATFORM; @@ -59,63 +58,6 @@ impl ExceptionKind { } } -/// Frame saved onto the stack when taking an IRQ -#[derive(Debug)] -#[repr(C)] -pub struct IrqFrame { - rax: u64, - rcx: u64, - rdx: u64, - rbx: u64, - rsi: u64, - rdi: u64, - rbp: u64, - r8: u64, - r9: u64, - r10: u64, - r11: u64, - r12: u64, - r13: u64, - r14: u64, - r15: u64, - - rip: u64, - cs: u64, - rflags: u64, - rsp: u64, - ss: u64, -} - -/// Set of registers saved when taking an exception/interrupt -#[derive(Debug)] -#[repr(C)] -pub struct ExceptionFrame { - rax: u64, - rcx: u64, - rdx: u64, - rbx: u64, - rsi: u64, - rdi: u64, - rbp: u64, - r8: u64, - r9: u64, - r10: u64, - r11: u64, - r12: u64, - r13: u64, - r14: u64, - r15: u64, - - exc_number: u64, - exc_code: u64, - - rip: u64, - cs: u64, - rflags: u64, - rsp: u64, - ss: u64, -} - /// Exception table entry #[allow(dead_code)] #[derive(Clone, Copy)] @@ -137,166 +79,52 @@ struct Pointer { offset: usize, } -impl ExceptionFrame { - fn dump(&self, level: debug::LogLevel) { - log_print_raw!(level, " CS:RIP = {:#x}:{:#x}\n", self.cs, self.rip); - log_print_raw!(level, " SS:RSP = {:#x}:{:#x}\n", self.ss, self.rsp); - - log_print_raw!( - level, - "RAX = {:#018x}, RCX = {:#018x}\n", - self.rax, - self.rcx - ); - log_print_raw!( - level, - "RDX = {:#018x}, RBX = {:#018x}\n", - self.rdx, - self.rbx - ); - log_print_raw!( - level, - "RSI = {:#018x}, RDI = {:#018x}\n", - self.rsi, - self.rdi - ); - log_print_raw!(level, "RBP = {:#018x}\n\n", self.rbp); - - log_print_raw!(level, " R8 = {:#018x}, R9 = {:#018x}\n", self.r8, self.r9); - log_print_raw!( - level, - "R10 = {:#018x}, R11 = {:#018x}\n", - self.r10, - self.r11 - ); - log_print_raw!( - level, - "R12 = {:#018x}, R13 = {:#018x}\n", - self.r12, - self.r13 - ); - log_print_raw!( - level, - "R14 = {:#018x}, R15 = {:#018x}\n", - self.r14, - self.r15 - ); - } -} - -impl TaskFrame for IrqFrame { - fn store(&self) -> SavedFrame { - SavedFrame { - rax: self.rax, - rcx: self.rcx, - rdx: self.rdx, - rbx: self.rbx, - rsi: self.rsi, - rdi: self.rdi, - rbp: self.rbp, - r8: self.r8, - r9: self.r9, - r10: self.r10, - r11: self.r11, - r12: self.r12, - r13: self.r13, - r14: self.r14, - r15: self.r15, - user_ip: self.rip, - user_sp: self.rsp, - rflags: self.rflags, - } - } - - fn restore(&mut self, _saved: &SavedFrame) { - todo!() - } - - fn argument(&self) -> u64 { - self.rdi as _ - } - - fn user_ip(&self) -> usize { - self.rip as _ - } - - fn user_sp(&self) -> usize { - self.rsp as _ - } - - fn set_argument(&mut self, value: u64) { - self.rdi = value; - } - - fn set_return_value(&mut self, value: u64) { - self.rax = value; - } - - fn set_user_ip(&mut self, value: usize) { - self.rip = value as _; - } - - fn set_user_sp(&mut self, value: usize) { - self.rsp = value as _; - } -} - -impl TaskFrame for ExceptionFrame { - fn store(&self) -> SavedFrame { - SavedFrame { - rax: self.rax, - rcx: self.rcx, - rdx: self.rdx, - rbx: self.rbx, - rsi: self.rsi, - rdi: self.rdi, - rbp: self.rbp, - r8: self.r8, - r9: self.r9, - r10: self.r10, - r11: self.r11, - r12: self.r12, - r13: self.r13, - r14: self.r14, - r15: self.r15, - user_ip: self.rip, - user_sp: self.rsp, - rflags: self.rflags, - } - } - - fn restore(&mut self, _saved: &SavedFrame) { - todo!() - } - - fn argument(&self) -> u64 { - 0 - } - - fn user_sp(&self) -> usize { - self.rsp as _ - } - - fn user_ip(&self) -> usize { - self.rip as _ - } - - fn set_user_sp(&mut self, value: usize) { - self.rsp = value as _; - } - - fn set_user_ip(&mut self, value: usize) { - self.rip = value as _; - } - - fn set_return_value(&mut self, _value: u64) { - // Not in syscall, do not overwrite - } - - fn set_argument(&mut self, value: u64) { - self.rdi = value; - } -} +// impl ExceptionFrame { +// fn dump(&self, level: debug::LogLevel) { +// log_print_raw!(level, " CS:RIP = {:#x}:{:#x}\n", self.cs, self.rip); +// log_print_raw!(level, " SS:RSP = {:#x}:{:#x}\n", self.ss, self.rsp); +// +// log_print_raw!( +// level, +// "RAX = {:#018x}, RCX = {:#018x}\n", +// self.rax, +// self.rcx +// ); +// log_print_raw!( +// level, +// "RDX = {:#018x}, RBX = {:#018x}\n", +// self.rdx, +// self.rbx +// ); +// log_print_raw!( +// level, +// "RSI = {:#018x}, RDI = {:#018x}\n", +// self.rsi, +// self.rdi +// ); +// log_print_raw!(level, "RBP = {:#018x}\n\n", self.rbp); +// +// log_print_raw!(level, " R8 = {:#018x}, R9 = {:#018x}\n", self.r8, self.r9); +// log_print_raw!( +// level, +// "R10 = {:#018x}, R11 = {:#018x}\n", +// self.r10, +// self.r11 +// ); +// log_print_raw!( +// level, +// "R12 = {:#018x}, R13 = {:#018x}\n", +// self.r12, +// self.r13 +// ); +// log_print_raw!( +// level, +// "R14 = {:#018x}, R15 = {:#018x}\n", +// self.r14, +// self.r15 +// ); +// } +// } const SIZE: usize = 256; @@ -333,12 +161,13 @@ impl Entry { static mut IDT: [Entry; SIZE] = [Entry::NULL; SIZE]; -fn user_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) { +fn user_exception_inner(kind: ExceptionKind, _frame: &ExceptionFrame) { let thread = Thread::current(); let cr3 = CR3.get(); warnln!("{:?} in {} {:?}", kind, thread.id, thread.name); - frame.dump(debug::LogLevel::Warning); + // XXX + // frame.dump(debug::LogLevel::Warning); warnln!("CR3 = {:#x}", cr3); match kind { @@ -381,7 +210,8 @@ fn kernel_exception_inner(kind: ExceptionKind, frame: &ExceptionFrame) -> ! { fatalln!("cr2 = {:#x}", cr2); } - frame.dump(debug::LogLevel::Fatal); + // XXX + // frame.dump(debug::LogLevel::Fatal); panic!("Irrecoverable exception"); } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index c2664785..8a8d9dff 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -32,8 +32,6 @@ use ygg_driver_pci::PciBusManager; mod acpi; mod apic; mod boot; -pub mod context; -// pub mod cpu; mod cpuid; mod exception; mod gdt; diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 21c37090..d1b42eeb 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -2,131 +2,19 @@ use core::arch::global_asm; -use abi::{arch::SavedFrame, error::Error, process::SignalEntryData, syscall::SyscallFunction}; -use kernel_arch_x86_64::registers::{ - MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR, +use abi::{process::SignalEntryData, syscall::SyscallFunction}; +use kernel_arch::task::TaskFrame; +use kernel_arch_x86_64::{ + context::SyscallFrame, + registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, }; use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ syscall::raw_syscall_handler, - task::{ - context::{ForkFrame, TaskFrame}, - process::Process, - thread::Thread, - TaskContext, - }, + task::{process::Process, thread::Thread}, }; -/// Set of registers saved when taking a syscall instruction -#[derive(Debug)] -#[repr(C)] -pub(super) struct SyscallFrame { - pub rax: u64, - pub args: [u64; 6], - - pub rcx: u64, - pub r11: u64, - - pub user_ip: u64, - pub user_sp: u64, - pub user_flags: u64, - - pub rbx: u64, - pub rbp: u64, - pub r12: u64, - pub r13: u64, - pub r14: u64, - pub r15: u64, -} - -impl ForkFrame for SyscallFrame { - unsafe fn fork(&self, address_space: u64) -> Result { - TaskContext::from_syscall_frame(self, address_space) - } - - fn set_return_value(&mut self, value: u64) { - self.rax = value; - } -} - -impl TaskFrame for SyscallFrame { - fn store(&self) -> SavedFrame { - SavedFrame { - rax: self.rax, - rcx: self.rcx, - rdx: self.args[2], - rbx: self.rbx, - rsi: self.args[1], - rdi: self.args[0], - rbp: self.rbp, - r8: self.args[4], - r9: self.args[5], - r10: self.args[3], - r11: self.r11, - r12: self.r12, - r13: self.r13, - r14: self.r14, - r15: self.r15, - user_ip: self.user_ip, - user_sp: self.user_sp, - rflags: self.user_flags, - } - } - - fn restore(&mut self, saved: &SavedFrame) { - self.rax = saved.rax; - self.args[0] = saved.rdi; - self.args[1] = saved.rsi; - self.args[2] = saved.rdx; - self.args[3] = saved.r10; - self.args[4] = saved.r8; - self.args[5] = saved.r9; - - self.rcx = saved.rcx; - self.r11 = saved.r11; - - self.user_ip = saved.user_ip; - self.user_sp = saved.user_sp; - self.user_flags = saved.rflags; - - self.rbx = saved.rbx; - self.rbp = saved.rbp; - self.r12 = saved.r12; - self.r13 = saved.r13; - self.r14 = saved.r14; - self.r15 = saved.r15; - } - - fn argument(&self) -> u64 { - self.args[0] - } - - fn user_sp(&self) -> usize { - self.user_sp as _ - } - - fn user_ip(&self) -> usize { - self.user_ip as _ - } - - fn set_user_sp(&mut self, value: usize) { - self.user_sp = value as _; - } - - fn set_user_ip(&mut self, value: usize) { - self.user_ip = value as _; - } - - fn set_return_value(&mut self, value: u64) { - self.rax = value; - } - - fn set_argument(&mut self, value: u64) { - self.args[0] = value; - } -} - fn syscall_inner(frame: &mut SyscallFrame) { if frame.rax == usize::from(SyscallFunction::ExitSignal) as u64 { unsafe { diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 6ecf3535..81f17738 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -14,6 +14,8 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; +use kernel_arch::task::TaskContext; +use libk::thread::TaskContextImpl; use libk_mm::{pointer::PhysicalRefMut, table::MapAttributes}; use vfs::{FileRef, IoContext, Read, Seek}; @@ -24,10 +26,8 @@ use crate::{ }, proc, task::{ - context::TaskContextImpl, process::{Process, ProcessImage}, thread::Thread, - TaskContext, }, }; @@ -121,7 +121,7 @@ fn setup_context( image: &ProcessImage, args: &Vec, envs: &Vec, -) -> Result { +) -> Result { const USER_STACK_PAGES: usize = 32; let virt_stack_base = 0x3000000; @@ -145,13 +145,14 @@ fn setup_context( virt_args_base ); - let user_sp = virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContext::USER_STACK_EXTRA_ALIGN; + let user_sp = + virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN; // Fill with some sentinel value to detect stack underflows let ptr = user_sp as *mut u64; #[allow(clippy::reversed_empty_ranges)] - for i in 0..TaskContext::USER_STACK_EXTRA_ALIGN / 8 { + for i in 0..TaskContextImpl::USER_STACK_EXTRA_ALIGN / 8 { unsafe { ptr.add(i).write_foreign_volatile(space, 0xDEADC0DE); } @@ -262,7 +263,7 @@ pub fn load_into>( path: P, args: Vec, envs: Vec, -) -> Result<(TaskContext, ProcessImage), Error> { +) -> Result<(TaskContextImpl, ProcessImage), Error> { let mut io = process.io.lock(); // Have to make the Path owned, going to drop the address space from which it came let path = path.as_ref().to_owned(); diff --git a/src/task/context.rs b/src/task/context.rs index 478a5f94..382c915d 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -1,119 +1 @@ //! Platform-specific task context manipulation interfaces - -use abi::{arch::SavedFrame, error::Error}; -use alloc::{boxed::Box, sync::Arc}; -use cfg_if::cfg_if; -use libk::thread::Termination; - -use crate::task::thread::Thread; - -cfg_if! { - if #[cfg(target_arch = "aarch64")] { - pub use crate::arch::aarch64::context::TaskContext; - } else if #[cfg(target_arch = "x86_64")] { - pub use crate::arch::x86_64::context::TaskContext; - } -} - -/// Interface for task state save/restore mechanisms -pub trait TaskFrame { - /// Creates a "snapshot" of a exception/syscall frame - fn store(&self) -> SavedFrame; - - /// Restores the exception/syscall frame from its saved state - fn restore(&mut self, saved: &SavedFrame); - - /// Replaces the return value in the frame (or does nothing, if the frame is not a part of a - /// syscall signal handler) - fn set_return_value(&mut self, value: u64); - - /// Replaces the userspace stack pointer in the frame - fn set_user_sp(&mut self, value: usize); - - /// Replaces the userspace instruction pointer in the frame - fn set_user_ip(&mut self, value: usize); - - /// Replaces the argument in the frame - fn set_argument(&mut self, value: u64); - - /// Returns the argument (if any) of the frame being processed - fn argument(&self) -> u64; - - /// Returns the userspace stack pointer - fn user_sp(&self) -> usize; - /// Returns the userspace instruction pointer - fn user_ip(&self) -> usize; -} - -/// Interface for performing context fork operations -pub trait ForkFrame: Sized { - /// Constructs a "forked" task context by copying the registers from this one and supplying a - /// new address space to it. - /// - /// # Safety - /// - /// Unsafe: accepts raw frames and address space address. - unsafe fn fork(&self, address_space: u64) -> Result; - - /// Replaces the return value inside the frame with a new one - fn set_return_value(&mut self, value: u64); -} - -/// Platform-specific task context implementation -pub trait TaskContextImpl: Sized { - /// Number of bytes to offset the signal stack pointer by - const SIGNAL_STACK_EXTRA_ALIGN: usize; - /// Number of bytes to offset the user stack pointer by - const USER_STACK_EXTRA_ALIGN: usize; - - /// Constructs a kernel-space task context - fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result; - - /// Constructs a user thread context. The caller is responsible for allocating the userspace - /// stack and setting up a valid address space for the context. - fn user( - entry: usize, - arg: usize, - cr3: u64, - user_stack_sp: usize, - tls_address: usize, - ) -> Result; - - /// Performs an entry into a context. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code. - unsafe fn enter(&self) -> !; - - /// Performs a context switch between two contexts. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code. - unsafe fn switch(&self, from: &Self); - - /// Performs a context switch and drops the source thread. - /// - /// # Safety - /// - /// Only meant to be called from the scheduler code after the `thread` has terminated. - unsafe fn switch_and_drop(&self, thread: Arc); - - /// Constructs a safe wrapper process to execute a kernel-space closure - fn kernel_closure T + Send + 'static>( - f: F, - ) -> Result { - extern "C" fn closure_wrapper T + Send + 'static>( - closure_addr: usize, - ) -> ! { - let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; - let result = closure(); - Thread::current().exit(result.into_exit_code()) - } - - let closure = Box::new(f); - debugln!("closure: {:p}", closure); - Self::kernel(closure_wrapper::, Box::into_raw(closure) as usize) - } -} diff --git a/src/task/mod.rs b/src/task/mod.rs index e15eafb2..f97bd7c9 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -7,8 +7,11 @@ use core::ops::{Deref, DerefMut}; use abi::error::Error; use alloc::{string::String, vec::Vec}; use device_api::interrupt::IpiMessage; -use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl, LocalCpuImpl}; -use libk::{runtime, thread::Termination}; +use kernel_arch::{task::TaskContext, Architecture, ArchitectureImpl, CpuImpl, LocalCpuImpl}; +use libk::{ + runtime, + thread::{TaskContextImpl, Termination}, +}; use libk_util::sync::{IrqGuard, SpinFence}; use crate::{ @@ -16,16 +19,12 @@ use crate::{ task::{sched::CpuQueue, thread::Thread}, }; -use self::context::TaskContextImpl; - pub mod context; pub mod process; pub mod sched; pub mod sync; pub mod thread; -pub use context::TaskContext; - /// Kernel wrapper for local CPU info structure. See [kernel_arch::LocalCpuImpl]. pub struct LocalCpu<'a>(LocalCpuImpl<'a, CpuQueue>); /// Kernel wrapper for per-CPU info structure. See [kernel_arch::LocalCpuImpl]. @@ -125,7 +124,13 @@ pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send name: S, f: F, ) -> Result<(), Error> { - let thread = Thread::new_kthread(name, TaskContext::kernel_closure(f)?); + let thread = Thread::new_kthread( + name, + TaskContextImpl::kernel_closure(move || { + let result = f(); + Thread::current().exit(result.into_exit_code()); + })?, + ); thread.enqueue(); Ok(()) diff --git a/src/task/process.rs b/src/task/process.rs index f4b06c3b..c84f36d0 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -16,6 +16,8 @@ use alloc::{ sync::{Arc, Weak}, vec::Vec, }; +use kernel_arch::task::TaskContext; +use libk::thread::{ForkFrame, TaskContextImpl}; use libk_util::{ event::OneTimeEvent, sync::{ @@ -28,14 +30,12 @@ use vfs::{IoContext, NodeRef}; use crate::{ mem::process::ProcessAddressSpace, proc::{self, io::ProcessIo}, - task::{context::TaskContextImpl, sched::CpuQueue}, + task::sched::CpuQueue, }; use super::{ - context::ForkFrame, sync::UserspaceMutex, thread::{Thread, ThreadId}, - TaskContext, }; /// Unique number assigned to each [Process] @@ -169,7 +169,7 @@ impl Process { name: S, parent: Option>, space: Arc, - context: TaskContext, + context: TaskContextImpl, image: Option, ) -> (Arc, Arc) { let name = name.into(); @@ -221,7 +221,7 @@ impl Process { 0 }; - let context = TaskContext::user( + let context = TaskContextImpl::user( options.entry as _, options.argument as _, space.as_address_with_asid(), @@ -238,7 +238,10 @@ impl Process { Ok(id) } - unsafe fn fork_inner(self: &Arc, frame: &F) -> Result { + unsafe fn fork_inner>( + self: &Arc, + frame: &F, + ) -> Result { let src_inner = self.inner.read(); let new_space = src_inner.space.as_ref().unwrap().fork()?; let new_context = frame.fork(new_space.as_address_with_asid())?; @@ -279,7 +282,7 @@ impl Process { /// /// Unsafe: frame must be a valid frame to be forked, the function is not yet stable and does /// not yet properly fork all the necessary context details. - pub unsafe fn raw_fork(frame: &mut F) { + pub unsafe fn raw_fork>(frame: &mut F) { let src_thread = Thread::current(); let src_process = src_thread.process(); diff --git a/src/task/sched.rs b/src/task/sched.rs index 04940552..c665a528 100644 --- a/src/task/sched.rs +++ b/src/task/sched.rs @@ -5,22 +5,25 @@ use core::cell::Cell; use alloc::{sync::Arc, vec::Vec}; use cfg_if::cfg_if; use crossbeam_queue::SegQueue; -use kernel_arch::{task::Scheduler, Architecture, ArchitectureImpl}; +use kernel_arch::{ + task::{Scheduler, TaskContext}, + Architecture, ArchitectureImpl, +}; +use libk::thread::TaskContextImpl; use libk_util::{sync::IrqGuard, OneTimeInit}; use crate::task::thread::ThreadState; use super::{ - context::TaskContextImpl, thread::{Thread, ThreadId}, - Cpu, TaskContext, + Cpu, }; /// Per-CPU queue pub struct CpuQueue { queue: SegQueue, index: usize, - idle: Cell, + idle: Cell, } impl Scheduler for CpuQueue { @@ -30,7 +33,7 @@ impl Scheduler for CpuQueue { impl CpuQueue { /// Creates a new [CpuQueue] for CPU with given `index` pub fn new(index: usize) -> Self { - let idle = TaskContext::kernel(__idle, Cpu::local().id() as usize) + let idle = TaskContextImpl::kernel(__idle, Cpu::local().id() as usize) .expect("Could not construct an idle task"); Self { @@ -167,7 +170,7 @@ impl CpuQueue { if !core::ptr::eq(current_ctx, next_ctx) { // Perform the switch if drop_current { - (*next_ctx).switch_and_drop(current.unwrap()); + (*next_ctx).switch_and_drop(Arc::into_raw(current.unwrap()) as _); } else { (*next_ctx).switch(&*current_ctx); } diff --git a/src/task/thread.rs b/src/task/thread.rs index a7c1bfe8..1f9afaa1 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -15,18 +15,24 @@ use abi::{ use alloc::{collections::BTreeMap, string::String, sync::Arc}; use atomic_enum::atomic_enum; use crossbeam_queue::SegQueue; -use libk::block; +use kernel_arch::task::{TaskContext, TaskFrame}; +use libk::{block, thread::TaskContextImpl}; use libk_util::{ event::BoolEvent, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, }; -use crate::{ - mem::{process::ProcessAddressSpace, ForeignPointer}, - task::context::TaskContextImpl, -}; +use crate::mem::{process::ProcessAddressSpace, ForeignPointer}; -use super::{context::TaskFrame, process::Process, sched::CpuQueue, Cpu, TaskContext}; +use super::{process::Process, sched::CpuQueue, Cpu}; + +// TODO this is ugly? +#[no_mangle] +unsafe extern "C" fn __arch_drop_thread(thread_ptr: *const Thread) { + let thread = Arc::from_raw(thread_ptr); + Thread::remove_from_list(thread.id); + Arc::decrement_strong_count(thread_ptr); +} /// Represents the states a thread can be at some point in time #[atomic_enum] @@ -95,7 +101,7 @@ pub struct Thread { /// Scheduling information pub sched: IrqSafeSpinlock, /// Low-level context details - pub context: Cell, + pub context: Cell, process: Option>, space: Option>, @@ -168,7 +174,7 @@ impl Thread { name: Option, process: Option>, space: Option>, - context: TaskContext, + context: TaskContextImpl, ) -> Arc { let thread = Arc::new(Self { id, @@ -195,7 +201,7 @@ impl Thread { } /// Constructs a new kernel-space thread - pub fn new_kthread>(name: S, context: TaskContext) -> Arc { + pub fn new_kthread>(name: S, context: TaskContextImpl) -> Arc { Self::new( ThreadId::next_kernel(), Some(name.into()), @@ -209,7 +215,7 @@ impl Thread { pub fn new_uthread( parent: Arc, space: Arc, - context: TaskContext, + context: TaskContextImpl, ) -> Arc { Self::new( ThreadId::next_user(), @@ -227,7 +233,7 @@ impl Thread { /// # Safety /// /// Unsafe: directly sets the thread's context. - pub unsafe fn replace_context(&self, context: TaskContext) -> TaskContext { + pub unsafe fn replace_context(&self, context: TaskContextImpl) -> TaskContextImpl { self.context.replace(context) } @@ -381,12 +387,6 @@ impl Thread { } } -impl Drop for Thread { - fn drop(&mut self) { - debugln!("DROP {:p}", self); - } -} - impl CurrentThread { /// Terminate the current thread pub fn exit(&self, code: ExitCode) -> ! { @@ -454,7 +454,7 @@ impl CurrentThread { // Setup signal frame let usp = ((entry.stack - size_of::()) & !0xF) - - TaskContext::SIGNAL_STACK_EXTRA_ALIGN; + - TaskContextImpl::SIGNAL_STACK_EXTRA_ALIGN; let frame_ptr = usp as *mut SignalEntryData; let saved_frame = frame.store(); @@ -468,7 +468,7 @@ impl CurrentThread { // Setup return to signal handler debugln!( - "Syscall entry @ pc={:#x}, sp={:#x} (top = {:#x})", + "Signal entry @ pc={:#x}, sp={:#x} (top = {:#x})", entry.entry, usp, entry.stack @@ -541,7 +541,7 @@ fn __create_kthread( func: extern "C" fn(usize) -> !, arg: usize, ) -> Result, Error> { - let context = TaskContext::kernel(func, arg)?; + let context = TaskContextImpl::kernel(func, arg)?; Ok(Thread::new_kthread(name, context)) } From c44ba57791ca17dcf786de0960679ca7cee3b922 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sat, 10 Feb 2024 14:03:04 +0200 Subject: [PATCH 188/211] libk: move ProcessAddressSpace to libk-mm --- libk-mm/src/process.rs | 132 +++++++++++++++++++++++++++++++++++------ src/mem/mod.rs | 4 +- src/mem/process.rs | 109 ---------------------------------- src/proc/elf.rs | 2 +- src/proc/exec.rs | 11 ++-- src/syscall/mod.rs | 2 +- src/task/process.rs | 2 +- src/task/thread.rs | 3 +- 8 files changed, 125 insertions(+), 140 deletions(-) delete mode 100644 src/mem/process.rs diff --git a/libk-mm/src/process.rs b/libk-mm/src/process.rs index bdb27eb7..9abcf7a1 100644 --- a/libk-mm/src/process.rs +++ b/libk-mm/src/process.rs @@ -1,5 +1,6 @@ use core::ops::Range; +use alloc::sync::Arc; use kernel_arch::ProcessAddressSpaceImpl; use libk_mm_interface::{ address::PhysicalAddress, @@ -10,27 +11,120 @@ use libk_util::sync::IrqSafeSpinlock; use vmalloc::{RangeData, VirtualMemoryAllocator}; use yggdrasil_abi::error::Error; -use crate::{phys, PageProvider, TableAllocatorImpl, L3_PAGE_SIZE}; +use crate::{ + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + PageProvider, TableAllocatorImpl, L3_PAGE_SIZE, +}; -struct Inner { - allocator: VirtualMemoryAllocator, +/// Describes how the physical memory is provided for the mapping +#[derive(Clone)] +pub enum VirtualRangeBacking { + /// Memory is taken from regular "anonymous" physical memory + Anonymous, + /// Mapping is backed by file blocks/device memory + File(FileBacking), +} + +/// Describes a file-backed memory range provider +#[derive(Clone)] +pub struct FileBacking { + offset: u64, + file: Arc, +} + +impl VirtualRangeBacking { + /// Creates a file-backed memory range provider + pub fn file(offset: u64, file: Arc) -> Result { + // XXX + // if !(offset as usize).is_page_aligned_for::() { + // todo!(); + // } + + Ok(Self::File(FileBacking { offset, file })) + } + + /// Creates a range of anonymous memory + pub fn anonymous() -> Self { + Self::Anonymous + } +} + +impl PageProvider for VirtualRangeBacking { + fn get_page(&self, offset: u64) -> Result { + match self { + Self::Anonymous => phys::alloc_page(), + Self::File(f) => f.file.get_page(f.offset + offset), + } + } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + match self { + Self::Anonymous => unsafe { + phys::free_page(phys); + Ok(()) + }, + Self::File(f) => f.file.release_page(f.offset + offset, phys), + } + } + + fn clone_page( + &self, + _offset: u64, + src_phys: PhysicalAddress, + _src_attrs: MapAttributes, + ) -> Result { + match self { + Self::Anonymous => { + let dst_page = phys::alloc_page()?; + let src_map = unsafe { PhysicalRef::<[u8; 4096]>::map(src_phys) }; + let mut dst_map = unsafe { PhysicalRefMut::<[u8; 4096]>::map(dst_page) }; + dst_map.copy_from_slice(src_map.as_ref()); + Ok(dst_page) + } + Self::File(_) => todo!(), + } + } +} + +impl PartialEq for VirtualRangeBacking { + fn eq(&self, other: &Self) -> bool { + matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous) + } +} + +impl Eq for VirtualRangeBacking {} + +impl RangeData for VirtualRangeBacking {} + +impl core::fmt::Debug for VirtualRangeBacking { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Anonymous => f.debug_struct("VirtualRangeBacking::Anonymous").finish(), + Self::File(fb) => f + .debug_struct("VirtualRangeBacking::File") + .field("offset", &fb.offset) + .finish(), + } + } +} + +struct Inner { + allocator: VirtualMemoryAllocator, table: ProcessAddressSpaceImpl, } /// Data structure for managing the address translation and allocation for a single process -pub struct ProcessAddressSpace< - PP: PageProvider + RangeData, - TA: TableAllocator = TableAllocatorImpl, -> { - inner: IrqSafeSpinlock>, +pub struct ProcessAddressSpace { + inner: IrqSafeSpinlock>, } -impl Inner { +impl Inner { fn try_map_pages( &mut self, address: usize, page_count: usize, - backing: &PP, + backing: &VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), (usize, Error)> { for i in 0..page_count { @@ -81,7 +175,7 @@ impl Inner { &mut self, address: usize, page_count: usize, - backing: PP, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), Error> { // If inserting fails, the range cannot be mapped @@ -104,7 +198,7 @@ impl Inner { fn map_single( &mut self, address: usize, - backing: PP, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { let start_pfn = address / L3_PAGE_SIZE; @@ -138,7 +232,7 @@ impl Inner { fn alloc_range( &mut self, page_count: usize, - backing: PP, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { let start_pfn = self.allocator.allocate(page_count, backing.clone())?; @@ -201,7 +295,7 @@ impl Inner { &mut self, source: &Self, pfn_range: Range, - backing: &PP, + backing: &VirtualRangeBacking, ) -> Result<(), Error> { self.allocator .insert(pfn_range.start, pfn_range.len(), backing.clone()) @@ -226,7 +320,7 @@ impl Inner { } } -impl ProcessAddressSpace { +impl ProcessAddressSpace { /// Constructs a new [ProcessAddressSpace] pub fn new() -> Result { let table = ProcessAddressSpaceImpl::new()?; @@ -276,7 +370,7 @@ impl ProcessAddressSpace, size: usize, - backing: PP, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { assert_eq!(size & (L3_PAGE_SIZE - 1), 0); @@ -291,7 +385,7 @@ impl ProcessAddressSpace Result<(), Error> { assert_eq!(address & (L3_PAGE_SIZE - 1), 0); @@ -306,7 +400,7 @@ impl ProcessAddressSpace Result { assert_eq!(address & (L3_PAGE_SIZE - 1), 0); @@ -350,7 +444,7 @@ impl ProcessAddressSpace Drop for ProcessAddressSpace { +impl Drop for ProcessAddressSpace { fn drop(&mut self) { self.clear().ok(); } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index e5dd5d33..b4700c66 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -10,14 +10,12 @@ use abi::error::Error; use libk_mm::{ address::{PhysicalAddress, Virtualize}, device::DeviceMemoryMapping, + process::ProcessAddressSpace, }; use crate::arch::{Platform, PlatformImpl}; pub mod heap; -pub mod process; - -use self::process::ProcessAddressSpace; /// Offset applied to the physical kernel image when translating it into the virtual address space pub const KERNEL_VIRT_OFFSET: usize = PlatformImpl::KERNEL_VIRT_OFFSET; diff --git a/src/mem/process.rs b/src/mem/process.rs deleted file mode 100644 index 779e3af2..00000000 --- a/src/mem/process.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Process address space structures and management functions - -use abi::error::Error; -use libk_mm::{ - address::PhysicalAddress, - phys, - pointer::{PhysicalRef, PhysicalRefMut}, - table::{EntryLevelExt, MapAttributes}, - PageProvider, -}; -use vfs::FileRef; -use vmalloc::RangeData; - -use crate::arch::L3; - -/// Represents a process address space. See [libk_mm::process::ProcessAddressSpace]. -pub type ProcessAddressSpace = libk_mm::process::ProcessAddressSpace; - -/// Describes how the physical memory is provided for the mapping -#[derive(Clone)] -pub enum VirtualRangeBacking { - /// Memory is taken from regular "anonymous" physical memory - Anonymous, - /// Mapping is backed by file blocks/device memory - File(FileBacking), -} - -/// Describes a file-backed memory range provider -#[derive(Clone)] -pub struct FileBacking { - offset: u64, - file: FileRef, -} - -impl VirtualRangeBacking { - /// Creates a file-backed memory range provider - pub fn file(offset: u64, file: FileRef) -> Result { - if !(offset as usize).is_page_aligned_for::() { - todo!(); - } - - Ok(Self::File(FileBacking { offset, file })) - } - - /// Creates a range of anonymous memory - pub fn anonymous() -> Self { - Self::Anonymous - } -} - -impl PageProvider for VirtualRangeBacking { - fn get_page(&self, offset: u64) -> Result { - match self { - Self::Anonymous => phys::alloc_page(), - Self::File(f) => f.file.get_page(f.offset + offset), - } - } - - fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { - match self { - Self::Anonymous => unsafe { - phys::free_page(phys); - Ok(()) - }, - Self::File(f) => f.file.release_page(f.offset + offset, phys), - } - } - - fn clone_page( - &self, - _offset: u64, - src_phys: PhysicalAddress, - _src_attrs: MapAttributes, - ) -> Result { - match self { - Self::Anonymous => { - let dst_page = phys::alloc_page()?; - let src_map = unsafe { PhysicalRef::<[u8; 4096]>::map(src_phys) }; - let mut dst_map = unsafe { PhysicalRefMut::<[u8; 4096]>::map(dst_page) }; - dst_map.copy_from_slice(src_map.as_ref()); - Ok(dst_page) - } - Self::File(_) => todo!(), - } - } -} - -impl PartialEq for VirtualRangeBacking { - fn eq(&self, other: &Self) -> bool { - matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous) - } -} - -impl Eq for VirtualRangeBacking {} - -impl RangeData for VirtualRangeBacking {} - -impl core::fmt::Debug for VirtualRangeBacking { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Anonymous => f.debug_struct("VirtualRangeBacking::Anonymous").finish(), - Self::File(fb) => f - .debug_struct("VirtualRangeBacking::File") - .field("offset", &fb.offset) - .field("file", fb.file.as_ref()) - .finish(), - } - } -} diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 7fa09f97..4d3066c5 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -9,6 +9,7 @@ use elf::{ }; use libk_mm::{ pointer::{PhysicalRef, PhysicalRefMut}, + process::{ProcessAddressSpace, VirtualRangeBacking}, table::{EntryLevelExt, MapAttributes}, }; use vfs::{FileRef, Read, Seek}; @@ -16,7 +17,6 @@ use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ arch::L3, - mem::process::{ProcessAddressSpace, VirtualRangeBacking}, task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 81f17738..5229de2c 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -16,14 +16,15 @@ use alloc::{ }; use kernel_arch::task::TaskContext; use libk::thread::TaskContextImpl; -use libk_mm::{pointer::PhysicalRefMut, table::MapAttributes}; +use libk_mm::{ + pointer::PhysicalRefMut, + process::{ProcessAddressSpace, VirtualRangeBacking}, + table::MapAttributes, +}; use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ - mem::{ - process::{ProcessAddressSpace, VirtualRangeBacking}, - ForeignPointer, - }, + mem::ForeignPointer, proc, task::{ process::{Process, ProcessImage}, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 371ea71c..ee4f8a2e 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -21,6 +21,7 @@ use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; use libk::{block, runtime}; use libk_mm::{ phys, + process::VirtualRangeBacking, table::{EntryLevelExt, MapAttributes}, }; use libk_util::sync::IrqSafeSpinlockGuard; @@ -32,7 +33,6 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::process::VirtualRangeBacking, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, diff --git a/src/task/process.rs b/src/task/process.rs index c84f36d0..0a83195e 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -18,6 +18,7 @@ use alloc::{ }; use kernel_arch::task::TaskContext; use libk::thread::{ForkFrame, TaskContextImpl}; +use libk_mm::process::ProcessAddressSpace; use libk_util::{ event::OneTimeEvent, sync::{ @@ -28,7 +29,6 @@ use libk_util::{ use vfs::{IoContext, NodeRef}; use crate::{ - mem::process::ProcessAddressSpace, proc::{self, io::ProcessIo}, task::sched::CpuQueue, }; diff --git a/src/task/thread.rs b/src/task/thread.rs index 1f9afaa1..aadb7ee7 100644 --- a/src/task/thread.rs +++ b/src/task/thread.rs @@ -17,12 +17,13 @@ use atomic_enum::atomic_enum; use crossbeam_queue::SegQueue; use kernel_arch::task::{TaskContext, TaskFrame}; use libk::{block, thread::TaskContextImpl}; +use libk_mm::process::ProcessAddressSpace; use libk_util::{ event::BoolEvent, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, }; -use crate::mem::{process::ProcessAddressSpace, ForeignPointer}; +use crate::mem::ForeignPointer; use super::{process::Process, sched::CpuQueue, Cpu}; From 00982dbdbe2876b5d920e838266ac21629528893 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 12 Feb 2024 12:09:53 +0200 Subject: [PATCH 189/211] libk: move thread/process to libk-threa --- Cargo.toml | 1 + arch/aarch64/src/lib.rs | 35 +- arch/interface/src/lib.rs | 4 + arch/interface/src/task.rs | 14 + arch/x86_64/src/lib.rs | 41 +- driver/block/nvme/src/drive.rs | 2 +- driver/block/nvme/src/lib.rs | 2 +- libk-thread/Cargo.toml | 18 + libk-thread/src/api.rs | 7 + src/task/sync.rs => libk-thread/src/futex.rs | 2 - libk-thread/src/lib.rs | 24 + libk-thread/src/mem.rs | 193 ++++++ libk-thread/src/process.rs | 430 ++++++++++++++ {libk => libk-thread}/src/runtime/executor.rs | 30 +- {libk => libk-thread}/src/runtime/macros.rs | 0 {libk => libk-thread}/src/runtime/mod.rs | 0 {libk => libk-thread}/src/runtime/task.rs | 0 .../src/runtime/task_queue.rs | 0 {libk => libk-thread}/src/runtime/timer.rs | 0 {src/task => libk-thread/src}/sched.rs | 121 ++-- {src/task => libk-thread/src}/thread.rs | 336 ++++------- libk-thread/src/types.rs | 220 +++++++ libk/Cargo.toml | 1 + libk/src/api.rs | 22 +- libk/src/arch.rs | 85 +++ libk/src/lib.rs | 17 +- libk/src/sync/mutex.rs | 6 +- libk/src/thread.rs | 89 --- src/arch/aarch64/exception.rs | 7 +- src/arch/aarch64/gic/mod.rs | 4 +- src/arch/aarch64/mod.rs | 5 +- src/arch/aarch64/timer.rs | 8 +- src/arch/mod.rs | 14 +- src/arch/x86_64/apic/local.rs | 6 +- src/arch/x86_64/apic/mod.rs | 15 +- src/arch/x86_64/exception.rs | 9 +- src/arch/x86_64/mod.rs | 5 +- src/arch/x86_64/smp.rs | 3 +- src/arch/x86_64/syscall.rs | 7 +- src/debug.rs | 7 +- src/device/display/linear_fb.rs | 8 +- src/device/tty.rs | 11 +- src/main.rs | 3 +- src/mem/mod.rs | 194 +----- src/panic.rs | 2 +- src/proc/exec.rs | 21 +- src/proc/io.rs | 43 +- src/syscall/arg.rs | 7 +- src/syscall/mod.rs | 137 ++--- src/task/context.rs | 1 - src/task/mod.rs | 121 +--- src/task/process.rs | 552 +----------------- 52 files changed, 1466 insertions(+), 1424 deletions(-) create mode 100644 libk-thread/Cargo.toml create mode 100644 libk-thread/src/api.rs rename src/task/sync.rs => libk-thread/src/futex.rs (97%) create mode 100644 libk-thread/src/lib.rs create mode 100644 libk-thread/src/mem.rs create mode 100644 libk-thread/src/process.rs rename {libk => libk-thread}/src/runtime/executor.rs (71%) rename {libk => libk-thread}/src/runtime/macros.rs (100%) rename {libk => libk-thread}/src/runtime/mod.rs (100%) rename {libk => libk-thread}/src/runtime/task.rs (100%) rename {libk => libk-thread}/src/runtime/task_queue.rs (100%) rename {libk => libk-thread}/src/runtime/timer.rs (100%) rename {src/task => libk-thread/src}/sched.rs (71%) rename {src/task => libk-thread/src}/thread.rs (70%) create mode 100644 libk-thread/src/types.rs create mode 100644 libk/src/arch.rs delete mode 100644 src/task/context.rs diff --git a/Cargo.toml b/Cargo.toml index 2363a7eb..0e1e8e28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ device-api = { path = "lib/device-api", features = ["derive"] } libk = { path = "libk" } libk-util = { path = "libk-util" } libk-mm = { path = "libk-mm" } +libk-thread = { path = "libk-thread" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index 0311809b..768c92d8 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -1,12 +1,17 @@ #![no_std] -#![feature(effects, strict_provenance, asm_const)] +#![feature(effects, strict_provenance, asm_const, naked_functions)] extern crate alloc; -use aarch64_cpu::registers::{DAIF, TPIDR_EL1}; -use alloc::vec::Vec; +use aarch64_cpu::registers::{DAIF, MPIDR_EL1, TPIDR_EL1}; +use alloc::{boxed::Box, vec::Vec}; use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; -use kernel_arch_interface::{cpu::IpiQueue, util::OneTimeInit, Architecture}; +use kernel_arch_interface::{ + cpu::{CpuImpl, IpiQueue}, + task::Scheduler, + util::OneTimeInit, + Architecture, +}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; pub mod context; @@ -25,6 +30,13 @@ pub struct PerCpuData { static IPI_QUEUES: OneTimeInit>> = OneTimeInit::new(); +#[naked] +extern "C" fn idle_task(_: usize) -> ! { + unsafe { + core::arch::asm!("1: nop; b 1b", options(noreturn)); + } +} + impl ArchitectureImpl { pub fn local_cpu_data() -> Option<&'static mut PerCpuData> { unsafe { (Self::local_cpu() as *mut PerCpuData).as_mut() } @@ -56,6 +68,17 @@ impl Architecture for ArchitectureImpl { TPIDR_EL1.set(cpu as _); } + unsafe fn init_local_cpu(id: Option, data: Self::PerCpuData) { + assert!( + id.is_none(), + "AArch64 uses MPIDR_EL1 instead of manual ID set" + ); + let id = (MPIDR_EL1.get() & 0xFF) as u32; + let cpu = Box::leak(Box::new(CpuImpl::::new(id, data))); + + cpu.set_local(); + } + fn local_cpu() -> *mut () { TPIDR_EL1.get() as _ } @@ -64,6 +87,10 @@ impl Architecture for ArchitectureImpl { IPI_QUEUES.init(queues); } + fn idle_task() -> extern "C" fn(usize) -> ! { + idle_task + } + fn local_interrupt_controller() -> &'static dyn LocalInterruptController { let local = Self::local_cpu_data().unwrap(); *local.gic.get() diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs index 4c92bab6..c324e2b5 100644 --- a/arch/interface/src/lib.rs +++ b/arch/interface/src/lib.rs @@ -4,6 +4,7 @@ use alloc::vec::Vec; use cpu::IpiQueue; use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; +use task::Scheduler; extern crate alloc; @@ -23,6 +24,9 @@ pub trait Architecture: Sized { unsafe fn set_local_cpu(cpu: *mut ()); fn local_cpu() -> *mut (); unsafe fn init_ipi_queues(queues: Vec>); + unsafe fn init_local_cpu(id: Option, data: Self::PerCpuData); + + fn idle_task() -> extern "C" fn(usize) -> !; // Interrupt management fn interrupt_mask() -> bool; diff --git a/arch/interface/src/task.rs b/arch/interface/src/task.rs index d011e71a..2a262a61 100644 --- a/arch/interface/src/task.rs +++ b/arch/interface/src/task.rs @@ -7,6 +7,20 @@ use crate::mem::{KernelTableManager, PhysicalMemoryAllocator}; pub trait Scheduler { type ThreadId: Copy; + + fn for_cpu(index: usize) -> &'static Self; + fn for_affinity_mask(mask: u64) -> &'static Self; + fn local() -> &'static Self; + + fn is_local(&self) -> bool; + fn push(&self, task: Self::ThreadId); + + /// Selects a new thread from the queue and performs a context switch if necessary. + /// + /// # Safety + /// + /// Only meant to be called from within the timer handler or the thread impl. + unsafe fn yield_cpu(&self) -> bool; } /// Conversion trait to allow multiple kernel closure return types diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index 0fe9aae8..16c1bd86 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -1,11 +1,18 @@ #![no_std] -#![feature(effects, strict_provenance, asm_const)] +#![feature(effects, strict_provenance, asm_const, naked_functions)] extern crate alloc; +use core::ops::DerefMut; + use alloc::vec::Vec; use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; -use kernel_arch_interface::{cpu::IpiQueue, util::OneTimeInit, Architecture}; +use kernel_arch_interface::{ + cpu::{CpuImpl, IpiQueue}, + task::Scheduler, + util::OneTimeInit, + Architecture, +}; use libk_mm_interface::address::PhysicalAddress; use registers::MSR_IA32_KERNEL_GS_BASE; use tock_registers::interfaces::Writeable; @@ -53,6 +60,20 @@ impl PerCpuData { static IPI_QUEUES: OneTimeInit>> = OneTimeInit::new(); +#[naked] +extern "C" fn idle_task(_: usize) -> ! { + unsafe { + core::arch::asm!( + r#" + 1: + nop + jmp 1b + "#, + options(noreturn, att_syntax) + ); + } +} + impl ArchitectureImpl { fn local_cpu_data() -> Option<&'static mut PerCpuData> { unsafe { (Self::local_cpu() as *mut PerCpuData).as_mut() } @@ -79,6 +100,22 @@ impl Architecture for ArchitectureImpl { IPI_QUEUES.init(queues); } + unsafe fn init_local_cpu(id: Option, data: Self::PerCpuData) { + use alloc::boxed::Box; + + let cpu = Box::leak(Box::new(CpuImpl::::new( + id.expect("x86_64 required manual CPU ID set"), + data, + ))); + cpu.this = cpu.deref_mut(); + + cpu.set_local(); + } + + fn idle_task() -> extern "C" fn(usize) -> ! { + idle_task + } + fn interrupt_mask() -> bool { let mut flags: u64; unsafe { diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index 68e30ff6..c353f2b5 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -87,7 +87,7 @@ impl NgBlockDevice for NvmeDrive { async fn submit_request(&self, request: IoRequest<'_>) -> Result { let queue_id = cpu_index(); - let ioq = &self.controller.ioqs.get()[queue_id]; + let ioq = &self.controller.ioqs.get()[queue_id as usize]; let command_id = match request.operation { IoOperation::Read { lba, count } => { diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index cc70209a..08b1e231 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -268,7 +268,7 @@ impl NvmeController { ) -> Result<(), NvmeError> { let _guard = IrqGuard::acquire(); let cpu_index = cpu_index(); - let ioq = &self.ioqs.get()[cpu_index]; + let ioq = &self.ioqs.get()[cpu_index as usize]; log::debug!( "{:?} ioq #{}, nsid={}, lba={:#x}", diff --git a/libk-thread/Cargo.toml b/libk-thread/Cargo.toml new file mode 100644 index 00000000..ae55d8c8 --- /dev/null +++ b/libk-thread/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "libk-thread" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libk-util = { path = "../libk-util" } +libk-mm = { path = "../libk-mm" } +kernel-arch = { path = "../arch" } + +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } + +log = "0.4.20" +atomic_enum = "0.2.0" +crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/libk-thread/src/api.rs b/libk-thread/src/api.rs new file mode 100644 index 00000000..82d9549d --- /dev/null +++ b/libk-thread/src/api.rs @@ -0,0 +1,7 @@ +use core::time::Duration; + +use yggdrasil_abi::error::Error; + +extern "Rust" { + pub fn __monotonic_timestamp() -> Result; +} diff --git a/src/task/sync.rs b/libk-thread/src/futex.rs similarity index 97% rename from src/task/sync.rs rename to libk-thread/src/futex.rs index 262261ad..af82a9da 100644 --- a/src/task/sync.rs +++ b/libk-thread/src/futex.rs @@ -1,5 +1,3 @@ -//! Higher-order task synchronization primitives - use core::{ pin::Pin, sync::atomic::{AtomicU32, Ordering}, diff --git a/libk-thread/src/lib.rs b/libk-thread/src/lib.rs new file mode 100644 index 00000000..aa2fb686 --- /dev/null +++ b/libk-thread/src/lib.rs @@ -0,0 +1,24 @@ +#![no_std] +#![feature(trait_alias, never_type, inline_const, arbitrary_self_types)] + +extern crate alloc; + +use kernel_arch::KernelTableManagerImpl; +use libk_mm::phys::GlobalPhysicalAllocator; + +pub(crate) mod api; + +#[macro_use] +pub mod runtime; + +pub mod futex; +pub mod mem; +pub mod process; +pub mod sched; +pub mod thread; +pub mod types; + +pub type TaskContextImpl = + kernel_arch::TaskContextImpl; + +pub use types::{AtomicThreadState, ThreadAffinity, ThreadId, ThreadState}; diff --git a/libk-thread/src/mem.rs b/libk-thread/src/mem.rs new file mode 100644 index 00000000..0631e33c --- /dev/null +++ b/libk-thread/src/mem.rs @@ -0,0 +1,193 @@ +use core::{alloc::Layout, mem::size_of}; + +use libk_mm::{address::Virtualize, process::ProcessAddressSpace}; +use yggdrasil_abi::error::Error; + +// XXX +const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; + +/// Helper trait to allow cross-address space access to pointers +pub trait ForeignPointer: Sized { + /// Perform a volatile pointer write without dropping the old value. + /// + /// # Panics + /// + /// The function panics if any of the following conditions is met: + /// + /// * The address of the pointer is not mapped in the `space`. + /// * The pointer is not writable. + /// * The pointer is misaligned. + /// + /// # Safety + /// + /// As this function allows direct memory writes, it is inherently unsafe. + unsafe fn write_foreign_volatile(self: *mut Self, space: &ProcessAddressSpace, value: Self); + + /// Performs pointer validation for given address space: + /// + /// * Checks if the pointer has proper alignment for the type. + /// * Checks if the pointer is mapped in the address space. + /// * Checks if the pointer is above the userspace memory boundary. + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_ptr<'a>( + self: *const Self, + space: &ProcessAddressSpace, + ) -> Result<&'a Self, Error>; + + /// [ForeignPointer::validate_user_ptr], with extra "writability" check. + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_mut<'a>( + self: *mut Self, + space: &ProcessAddressSpace, + ) -> Result<&'a mut Self, Error>; + + /// [ForeignPointer::validate_user_ptr], but for slices + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_slice<'a>( + self: *const Self, + len: usize, + space: &ProcessAddressSpace, + ) -> Result<&'a [Self], Error>; + + /// [ForeignPointer::validate_user_slice], but for mutable slices + /// + /// # Safety + /// + /// Even though this function does the necessary checks, it is still a raw pointer to reference + /// conversion, and thus is unsafe. + unsafe fn validate_user_slice_mut<'a>( + self: *mut Self, + len: usize, + space: &ProcessAddressSpace, + ) -> Result<&'a mut [Self], Error>; +} + +impl ForeignPointer for T { + unsafe fn write_foreign_volatile(self: *mut Self, space: &ProcessAddressSpace, value: T) { + // TODO check align + let addr = self as usize; + let start_page = addr & !0xFFF; + let end_page = (addr + size_of::() - 1) & !0xFFF; + let page_offset = addr & 0xFFF; + + if start_page != end_page { + todo!("Foreign pointer write crossed a page boundary"); + } + + let phys_page = space + .translate(start_page) + .expect("Address is not mapped in the target address space"); + + let virt_ptr = phys_page.add(page_offset).virtualize() as *mut T; + virt_ptr.write_volatile(value); + } + + unsafe fn validate_user_slice_mut<'a>( + self: *mut Self, + len: usize, + space: &ProcessAddressSpace, + ) -> Result<&'a mut [Self], Error> { + let base = self as usize; + let layout = Layout::array::(len).unwrap(); + + validate_user_align_size(base, &layout)?; + validate_user_region(space, base, layout.size(), true)?; + + Ok(core::slice::from_raw_parts_mut(self, len)) + } + + unsafe fn validate_user_slice<'a>( + self: *const Self, + len: usize, + space: &ProcessAddressSpace, + ) -> Result<&'a [Self], Error> { + let base = self as usize; + let layout = Layout::array::(len).unwrap(); + + validate_user_align_size(base, &layout)?; + validate_user_region(space, base, layout.size(), false)?; + + Ok(core::slice::from_raw_parts(self, len)) + } + + unsafe fn validate_user_mut<'a>( + self: *mut Self, + space: &ProcessAddressSpace, + ) -> Result<&'a mut Self, Error> { + let addr = self as usize; + let layout = Layout::new::(); + + // Common validation + validate_user_align_size(addr, &layout)?; + + // Validate that the pages covered by this address are mapped as writable by the process + // TODO for CoW this may differ + validate_user_region(space, addr, layout.size(), true)?; + + Ok(&mut *self) + } + + unsafe fn validate_user_ptr<'a>( + self: *const Self, + space: &ProcessAddressSpace, + ) -> Result<&'a Self, Error> { + let addr = self as usize; + let layout = Layout::new::(); + + // Common validation + validate_user_align_size(addr, &layout)?; + validate_user_region(space, addr, layout.size(), false)?; + + Ok(&*self) + } +} + +fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { + // Explicitly disallow NULL + if addr == 0 { + return Err(Error::InvalidArgument); + } + // Validate alignment + if addr % layout.align() != 0 { + return Err(Error::InvalidArgument); + } + if addr + layout.size() > KERNEL_VIRT_OFFSET { + todo!(); + } + + Ok(()) +} + +/// Validates access to given userspace memory region with given constraints +pub fn validate_user_region( + space: &ProcessAddressSpace, + base: usize, + len: usize, + _need_write: bool, +) -> Result<(), Error> { + if base + len > crate::mem::KERNEL_VIRT_OFFSET { + panic!("Invalid argument"); + } + + let aligned_start = base & !0xFFF; + let aligned_end = (base + len + 0xFFF) & !0xFFF; + + for page in (aligned_start..aligned_end).step_by(0x1000) { + // TODO check writability + space.translate(page)?; + } + + Ok(()) +} diff --git a/libk-thread/src/process.rs b/libk-thread/src/process.rs new file mode 100644 index 00000000..1ef594b9 --- /dev/null +++ b/libk-thread/src/process.rs @@ -0,0 +1,430 @@ +use core::marker::PhantomData; + +use alloc::{ + collections::BTreeMap, + string::String, + sync::{Arc, Weak}, + vec::Vec, +}; +use futures_util::Future; +use kernel_arch::KernelTableManagerImpl; +use libk_mm::{phys::GlobalPhysicalAllocator, process::ProcessAddressSpace}; +use libk_util::{ + event::OneTimeEvent, + sync::{ + spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, + IrqSafeSpinlock, + }, +}; +use yggdrasil_abi::{ + error::{Error, SyscallResult}, + process::{ExitCode, Signal, ThreadSpawnOptions}, +}; + +use crate::{ + futex::UserspaceMutex, + thread::Thread, + types::{ProcessId, ProcessTlsInfo}, + TaskContextImpl, ThreadId, +}; + +pub trait ForkFrame = kernel_arch::task::ForkFrame; + +pub trait Process: Send + Sync { + /// Returns the [ProcessId] of this process + fn id(&self) -> ProcessId; + + /// Returns the name of the process + fn name(&self) -> &str; + + fn handle_thread_exit(&self, id: ThreadId, code: ExitCode); + fn terminate_others(&self, except: ThreadId) -> impl Future + Send; +} + +pub trait ProcessIo: Sized { + type Node; + + fn new() -> Self; + fn handle_exit(&mut self); + fn fork_from(&mut self, src: &Self) -> Result<(), Error>; +} + +pub trait ProcessManager: Send + Sync { + type Process: Process; + + fn register_process(process: Arc); + fn get(id: ProcessId) -> Option>; + fn for_each)>(f: F); +} + +/// Describes information about a program's image in memory +#[derive(Clone)] +pub struct ProcessImage { + /// Entry point address + pub entry: usize, + /// Thread-local storage information + pub tls: Option, +} + +pub struct ProcessInner { + session_id: ProcessId, + group_id: ProcessId, + + session_terminal: Option>, + threads: Vec>, + mutexes: BTreeMap>, + space: Option>, + image: Option, +} + +/// Describes a process within the system +pub struct ProcessImpl, IO: ProcessIo> { + name: String, + id: ProcessId, + + #[allow(unused)] + parent: Option>>, + + inner: IrqSafeRwLock>, + + pub(crate) exit: OneTimeEvent, + + /// Process I/O information + pub io: IrqSafeSpinlock, + + _pm: PhantomData, +} + +impl, IO: ProcessIo> Process for ProcessImpl { + fn id(&self) -> ProcessId { + self.id + } + + fn name(&self) -> &str { + self.name.as_ref() + } + + fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { + log::debug!("Thread {} of process {}: {:?}", thread, self.id, code); + let mut inner = self.inner.write(); + + assert!(inner.remove_thread(thread)); + + let last_thread = inner.threads.is_empty(); + + if last_thread { + log::debug!("Last thread of {} exited", self.id); + self.cleanup(inner); + self.exit.signal(code); + } + } + + async fn terminate_others(&self, except: ThreadId) { + let mut inner = self.inner.write(); + + for thread in inner.threads.iter() { + if thread.id == except { + continue; + } + + log::info!("Terminate thread {}", thread.id); + thread.terminate().await; + } + + inner.retain_thread(except); + } +} + +impl, IO: ProcessIo> ProcessImpl { + /// Creates a new process with given main thread + pub fn new_with_main>( + name: S, + parent: Option>, + space: Arc, + context: TaskContextImpl, + image: Option, + ) -> (Arc, Arc) { + let name = name.into(); + let id = ProcessId::next(); + + let process = Arc::new(Self { + name, + id, + parent, + + inner: IrqSafeRwLock::new(ProcessInner::new(id, Some(space.clone()), image)), + + exit: OneTimeEvent::new(), + io: IrqSafeSpinlock::new(IO::new()), + + _pm: PhantomData, + }); + + // Create "main" thread + let thread = Thread::new_uthread(process.id, space, context); + process.inner.write().threads.push(thread.clone()); + + PM::register_process(process.clone()); + + (process, thread) + } + + /// Spawns a new child thread within the process + pub fn spawn_thread(self: &Arc, options: &ThreadSpawnOptions) -> Result { + log::debug!( + "Spawn thread in {} with options: {:#x?}", + self.id(), + options + ); + // let mut inner = self.inner.write(); + + // let space = inner.space.clone().unwrap(); + + todo!() + // XXX + // let tls_address = if let Some(image) = inner.image.as_ref() { + // proc::elf::clone_tls(&space, image)? + // } else { + // 0 + // }; + + // let context = TaskContextImpl::user( + // options.entry as _, + // options.argument as _, + // space.as_address_with_asid(), + // options.stack_top, + // tls_address, + // )?; + // let thread = Thread::new_uthread(self.clone(), space.clone(), context); + // let id = thread.id; + + // inner.add_thread(thread.clone()); + + // thread.enqueue(); + + // Ok(id) + } + + unsafe fn fork_inner>( + self: &Arc, + frame: &F, + ) -> Result { + let src_inner = self.inner.read(); + let new_space = src_inner.space.as_ref().unwrap().fork()?; + let new_context = frame.fork(new_space.as_address_with_asid())?; + + let (new_process, new_main) = Self::new_with_main( + self.name(), + Some(Arc::downgrade(self)), + Arc::new(new_space), + new_context, + src_inner.image.clone(), + ); + + { + let mut dst_io = new_process.io.lock(); + let src_io = self.io.lock(); + + dst_io.fork_from(&src_io)?; + } + + new_process.inherit(self)?; + + log::info!("Process::fork -> {:?}", new_process.id()); + new_main.enqueue(); + + Ok(new_process.id()) + } + + /// Performs a "fork" operation on the process, creating an identical copy of it, cloning + /// the register state from provided [ForkFrame]. + /// + /// # Safety + /// + /// Unsafe: frame must be a valid frame to be forked, the function is not yet stable and does + /// not yet properly fork all the necessary context details. + pub unsafe fn raw_fork>(frame: &mut F) { + let src_thread = Thread::current(); + let src_process = src_thread.process::(); + + let rax = src_process + .fork_inner(frame) + .map(|ProcessId(p)| p as u32) + .into_syscall_result(); + + frame.set_return_value(rax as _); + } + + /// Replaces the process address space with a new one, loaded from the specified program + pub fn exec(&self, _program: &str, _argv: Vec, _envp: Vec) -> Result { + // XXX + todo!() + // if self.inner.read().thread_count() != 1 { + // todo!(); + // } + + // let (context, image) = proc::exec::load_into(self, program, argv, envp)?; + // let mut inner = self.inner.write(); + // let main = &inner.threads()[0]; + + // let old_context = unsafe { main.replace_context(context) }; + // let new_context = main.context.as_ptr(); + // inner.set_image(Some(image)); + + // drop(inner); + + // // TODO old context is leaked + // unsafe { (*new_context).switch(&old_context) } + // unreachable!() + } + + /// Returns the address space of the process + pub fn space(&self) -> Arc { + self.inner.read().space.clone().unwrap() + } + + /// Returns the process group ID of the process + pub fn group_id(&self) -> ProcessId { + self.inner.read().group_id + } + + /// Returns the process session ID of the process + pub fn session_id(&self) -> ProcessId { + self.inner.read().session_id + } + + /// Changes the process's group ID + pub fn set_group_id(&self, id: ProcessId) { + self.inner.write().group_id = id; + } + + /// Changes the process's session ID + pub fn set_session_id(&self, id: ProcessId) { + self.inner.write().session_id = id; + } + + // Resources + + /// Returns the current session terminal of the process, if set + pub fn session_terminal(&self) -> Option> { + self.inner.read().session_terminal.clone() + } + + /// Changes the current session terminal of the process + pub fn set_session_terminal(&self, node: Arc) { + self.inner.write().session_terminal = Some(node); + } + + /// Resets the current session terminal of the process + pub fn clear_session_terminal(&self) -> Option> { + self.inner.write().session_terminal.take() + } + + /// Inherits the process information from the `parent` + pub fn inherit(&self, parent: &Self) -> Result<(), Error> { + let mut our_inner = self.inner.write(); + let their_inner = parent.inner.read(); + + our_inner.session_id = their_inner.session_id; + our_inner.group_id = their_inner.group_id; + our_inner.session_terminal = their_inner.session_terminal.clone(); + + Ok(()) + } + + // State + + /// Returns the [ExitCode] of the process, if it has exited + pub fn get_exit_status(&self) -> Option { + self.exit.try_read_copy() + } + + /// Returns `true` if the process has exited + pub fn has_exited(&self) -> bool { + self.exit.is_signalled() + } + + pub async fn wait_for_exit(&self) -> ExitCode { + self.exit.wait_copy().await + } + + /// Cleans up process resources + fn cleanup(&self, mut inner: IrqSafeRwLockWriteGuard>) { + self.io.lock().handle_exit(); + inner.space = None; + } + + /// Raises a signal for the specified process + pub fn raise_signal(self: &Arc, signal: Signal) { + let thread = self.inner.read().threads[0].clone(); + thread.raise_signal(signal); + } + + /// Raises a signal for the specified process group + pub fn signal_group(group_id: ProcessId, signal: Signal) { + PM::for_each(|_, proc| { + let inner = proc.inner.read(); + if !proc.has_exited() && inner.group_id == group_id { + log::debug!( + "Deliver group ({}) signal to {}: {:?}", + group_id, + proc.id(), + signal + ); + drop(inner); + proc.raise_signal(signal); + } + }); + } + + /// Returns a [UserspaceMutex] associated with the `address`. If one does not exist, will + /// create it. + pub fn get_or_insert_mutex(&self, address: usize) -> Arc { + let mut inner = self.inner.write(); + inner + .mutexes + .entry(address) + .or_insert_with(|| Arc::new(UserspaceMutex::new(address))) + .clone() + } + + // Process list + + /// Returns the process with given [ProcessId], if it exists + pub fn get(id: ProcessId) -> Option> { + PM::get(id) + } +} + +impl ProcessInner { + pub fn new( + id: ProcessId, + space: Option>, + image: Option, + ) -> Self { + Self { + session_id: id, + group_id: id, + session_terminal: None, + threads: Vec::new(), + + mutexes: BTreeMap::new(), + image, + space: space.clone(), + } + } + + pub fn add_thread(&mut self, thread: Arc) { + self.threads.push(thread); + } + + pub fn remove_thread(&mut self, id: ThreadId) -> bool { + let n = self.threads.len(); + self.threads.retain(|t| t.id != id); + n != self.threads.len() + } + + pub fn retain_thread(&mut self, except: ThreadId) { + self.threads.retain(|t| t.id == except) + } +} diff --git a/libk/src/runtime/executor.rs b/libk-thread/src/runtime/executor.rs similarity index 71% rename from libk/src/runtime/executor.rs rename to libk-thread/src/runtime/executor.rs index 486e02ec..debf16b6 100644 --- a/libk/src/runtime/executor.rs +++ b/libk-thread/src/runtime/executor.rs @@ -2,10 +2,11 @@ use core::task::{Context, Poll, Waker}; use alloc::{boxed::Box, format, sync::Arc}; use futures_util::{task::waker_ref, Future}; +use kernel_arch::task::TaskContext; use libk_util::waker::WakeWeak; use yggdrasil_abi::error::Error; -use crate::thread::Thread; +use crate::{thread::Thread, TaskContextImpl}; use super::{ task::{Task, Termination}, @@ -21,19 +22,26 @@ pub fn enqueue(task: Arc) -> Result<(), Error> { pub fn spawn_async_worker(index: usize) -> Result<(), Error> { let name = format!("[async-worker-{}]", index); - Thread::spawn(name, move || loop { - let task = task_queue::pop_task().unwrap(); - let mut future_slot = task.future.lock(); + let thread = Thread::new_kthread( + name, + TaskContextImpl::kernel_closure(move || loop { + let task = task_queue::pop_task().unwrap(); + let mut future_slot = task.future.lock(); - if let Some(mut future) = future_slot.take() { - let waker = waker_ref(&task); - let context = &mut Context::from_waker(&waker); + if let Some(mut future) = future_slot.take() { + let waker = waker_ref(&task); + let context = &mut Context::from_waker(&waker); - if future.as_mut().poll(context).is_pending() { - *future_slot = Some(future); + if future.as_mut().poll(context).is_pending() { + *future_slot = Some(future); + } } - } - }) + })?, + ); + + thread.enqueue(); + + Ok(()) } /// Creates a new task for the [Future] and queues it for execution in background diff --git a/libk/src/runtime/macros.rs b/libk-thread/src/runtime/macros.rs similarity index 100% rename from libk/src/runtime/macros.rs rename to libk-thread/src/runtime/macros.rs diff --git a/libk/src/runtime/mod.rs b/libk-thread/src/runtime/mod.rs similarity index 100% rename from libk/src/runtime/mod.rs rename to libk-thread/src/runtime/mod.rs diff --git a/libk/src/runtime/task.rs b/libk-thread/src/runtime/task.rs similarity index 100% rename from libk/src/runtime/task.rs rename to libk-thread/src/runtime/task.rs diff --git a/libk/src/runtime/task_queue.rs b/libk-thread/src/runtime/task_queue.rs similarity index 100% rename from libk/src/runtime/task_queue.rs rename to libk-thread/src/runtime/task_queue.rs diff --git a/libk/src/runtime/timer.rs b/libk-thread/src/runtime/timer.rs similarity index 100% rename from libk/src/runtime/timer.rs rename to libk-thread/src/runtime/timer.rs diff --git a/src/task/sched.rs b/libk-thread/src/sched.rs similarity index 71% rename from src/task/sched.rs rename to libk-thread/src/sched.rs index c665a528..09a5d7c2 100644 --- a/src/task/sched.rs +++ b/libk-thread/src/sched.rs @@ -1,23 +1,14 @@ -//! Per-CPU queue implementation - use core::cell::Cell; use alloc::{sync::Arc, vec::Vec}; -use cfg_if::cfg_if; use crossbeam_queue::SegQueue; use kernel_arch::{ task::{Scheduler, TaskContext}, - Architecture, ArchitectureImpl, + Architecture, ArchitectureImpl, CpuImpl, }; -use libk::thread::TaskContextImpl; use libk_util::{sync::IrqGuard, OneTimeInit}; -use crate::task::thread::ThreadState; - -use super::{ - thread::{Thread, ThreadId}, - Cpu, -}; +use crate::{thread::Thread, TaskContextImpl, ThreadId, ThreadState}; /// Per-CPU queue pub struct CpuQueue { @@ -26,15 +17,16 @@ pub struct CpuQueue { idle: Cell, } -impl Scheduler for CpuQueue { - type ThreadId = ThreadId; -} +static QUEUES: OneTimeInit> = OneTimeInit::new(); impl CpuQueue { /// Creates a new [CpuQueue] for CPU with given `index` pub fn new(index: usize) -> Self { - let idle = TaskContextImpl::kernel(__idle, Cpu::local().id() as usize) - .expect("Could not construct an idle task"); + let idle = TaskContextImpl::kernel( + ArchitectureImpl::idle_task(), + CpuImpl::::local().id() as usize, + ) + .expect("Could not construct an idle task"); Self { queue: SegQueue::new(), @@ -60,11 +52,6 @@ impl CpuQueue { (*self.idle.as_ptr()).enter() } - /// "Pushes" a thread to the queue for execution - pub fn push(&self, tid: ThreadId) { - self.queue.push(tid); - } - fn pop(&self) -> (Option>, Option) { while let Some(id) = self.queue.pop() { let Some(thread) = Thread::get(id) else { @@ -99,16 +86,43 @@ impl CpuQueue { (None, None) } +} - /// Selects a new thread from the queue and performs a context switch if necessary. - /// - /// # Safety - /// - /// Only meant to be called from within the timer handler or the thread impl. - pub unsafe fn yield_cpu(&self) -> bool { +impl Scheduler for CpuQueue { + type ThreadId = ThreadId; + + fn for_cpu(index: usize) -> &'static Self { + &QUEUES.get()[index] + } + + fn for_affinity_mask(mask: u64) -> &'static Self { + debug_assert_ne!(mask, 0); + + QUEUES + .get() + .iter() + .filter(|c| mask & (1 << c.index) != 0) + .min_by_key(|c| c.queue.len()) + .unwrap() + } + + fn local() -> &'static Self { + CpuImpl::local().scheduler() + } + + fn is_local(&self) -> bool { + assert!(ArchitectureImpl::interrupt_mask()); + core::ptr::eq(Self::local(), self) + } + + fn push(&self, task: Self::ThreadId) { + self.queue.push(task); + } + + unsafe fn yield_cpu(&self) -> bool { assert!(ArchitectureImpl::interrupt_mask()); - let mut cpu = Cpu::local(); + let mut cpu = CpuImpl::::local(); let current_id = cpu.current_thread_id(); let current = current_id.and_then(Thread::get); @@ -180,60 +194,9 @@ impl CpuQueue { false } } - - /// Returns a queue for given CPU `index` - pub fn for_cpu(index: usize) -> &'static CpuQueue { - &QUEUES.get()[index] - } - - /// Returns a queue for the local CPU - pub fn local() -> &'static CpuQueue { - Cpu::local().scheduler() - } - - /// Returns the preferred queue for a thread to be scheduled into. Takes affinity mask into - /// account when picking a queue. - pub fn for_affinity_mask(mask: u64) -> &'static CpuQueue { - debug_assert_ne!(mask, 0); - - QUEUES - .get() - .iter() - .filter(|c| mask & (1 << c.index) != 0) - .min_by_key(|c| c.queue.len()) - .unwrap() - } - - /// Returns `true` if the queue is local to the current CPU - pub fn is_local(&self) -> bool { - assert!(ArchitectureImpl::interrupt_mask()); - core::ptr::eq(Self::local(), self) - } } -#[naked] -extern "C" fn __idle(_x: usize) -> ! { - unsafe { - cfg_if! { - if #[cfg(target_arch = "aarch64")] { - core::arch::asm!("1: nop; b 1b", options(noreturn)); - } else if #[cfg(target_arch = "x86_64")] { - core::arch::asm!(r#" - 1: - nop - jmp 1b - "#, options(noreturn, att_syntax)); - } else { - core::arch::asm!("", options(noreturn)); - } - } - } -} - -static QUEUES: OneTimeInit> = OneTimeInit::new(); - /// Initializes the global queue list pub fn init_queues(queues: Vec) { - debugln!("init_queues!!!"); QUEUES.init(queues); } diff --git a/src/task/thread.rs b/libk-thread/src/thread.rs similarity index 70% rename from src/task/thread.rs rename to libk-thread/src/thread.rs index aadb7ee7..8157386c 100644 --- a/src/task/thread.rs +++ b/libk-thread/src/thread.rs @@ -1,84 +1,33 @@ -//! Thread data structures and management +use core::{cell::Cell, mem::size_of, ops::Deref}; -use core::{ - cell::Cell, - fmt, - mem::size_of, - ops::Deref, - sync::atomic::{AtomicU64, Ordering}, +use alloc::{ + collections::BTreeMap, + string::String, + sync::{Arc, Weak}, }; - -use abi::{ - error::Error, - process::{ExitCode, Signal, SignalEntryData}, -}; -use alloc::{collections::BTreeMap, string::String, sync::Arc}; -use atomic_enum::atomic_enum; use crossbeam_queue::SegQueue; -use kernel_arch::task::{TaskContext, TaskFrame}; -use libk::{block, thread::TaskContextImpl}; +use futures_util::task::ArcWake; +use kernel_arch::{ + task::{Scheduler, TaskContext, TaskFrame}, + CpuImpl, +}; use libk_mm::process::ProcessAddressSpace; use libk_util::{ event::BoolEvent, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, }; +use yggdrasil_abi::{ + error::Error, + process::{ExitCode, Signal, SignalEntryData}, +}; -use crate::mem::ForeignPointer; - -use super::{process::Process, sched::CpuQueue, Cpu}; - -// TODO this is ugly? -#[no_mangle] -unsafe extern "C" fn __arch_drop_thread(thread_ptr: *const Thread) { - let thread = Arc::from_raw(thread_ptr); - Thread::remove_from_list(thread.id); - Arc::decrement_strong_count(thread_ptr); -} - -/// Represents the states a thread can be at some point in time -#[atomic_enum] -#[derive(PartialEq)] -pub enum ThreadState { - /// Thread is ready for execution and is present in some CPU's queue - Ready, - /// Thread is currently being executed by some CPU - Running, - /// Thread is present in a global list, but is not queued for execution until it is resumed - Suspended, - /// Thread is terminated and waits to be reaped - Terminated, -} - -/// Unique number describing a single kernel or userspace thread -#[derive(Debug, PartialEq, Eq, Clone, Copy, Ord, PartialOrd)] -pub enum ThreadId { - /// Describes a kernel-space thread - Kernel(u64), - /// Describes an user-space thread - User(u64), -} - -/// Wrapper which guarantees the thread referred to is the current one on the current CPU -#[repr(C)] -pub struct CurrentThread(Arc, IrqGuard); - -/// Mask describing CPUs a thread is allowed to be scheduled to -#[derive(Debug)] -#[repr(transparent)] -pub struct ThreadAffinity(AtomicU64); - -struct SignalEntry { - entry: usize, - stack: usize, -} - -struct ThreadInner { - signal_entry: Option, -} - -struct GlobalThreadList { - data: BTreeMap>, -} +use crate::{ + mem::ForeignPointer, + process::{Process, ProcessManager}, + sched::CpuQueue, + types::{ProcessId, ThreadAffinity, ThreadId, ThreadState}, + TaskContextImpl, +}; /// Provides details about how the thread is scheduled onto CPUs pub struct ThreadSchedulingInfo { @@ -91,7 +40,14 @@ pub struct ThreadSchedulingInfo { pub queue: Option<&'static CpuQueue>, } -static THREADS: IrqSafeRwLock = IrqSafeRwLock::new(GlobalThreadList::new()); +struct SignalEntry { + entry: usize, + stack: usize, +} + +struct ThreadInner { + signal_entry: Option, +} /// Describes a single thread within the system pub struct Thread { @@ -103,77 +59,40 @@ pub struct Thread { pub sched: IrqSafeSpinlock, /// Low-level context details pub context: Cell, - - process: Option>, + process: Option, space: Option>, inner: IrqSafeSpinlock, signal_queue: SegQueue, - pub(super) exit: BoolEvent, - + pub exit: BoolEvent, /// CPU scheduling affinity mask pub affinity: ThreadAffinity, } -unsafe impl Sync for Thread {} +/// Wrapper which guarantees the thread referred to is the current one on the current CPU +#[repr(C)] +pub struct CurrentThread(Arc, IrqGuard); -impl GlobalThreadList { - pub const fn new() -> Self { - Self { - data: BTreeMap::new(), - } - } - - #[inline] - pub fn get(&self, id: ThreadId) -> Option<&Arc> { - self.data.get(&id) - } - - pub fn insert(&mut self, thread: Arc) { - let id = thread.id; - debug_assert!(!self.data.contains_key(&id)); - self.data.insert(id, thread); - } - - pub fn remove(&mut self, id: ThreadId) -> Option> { - self.data.remove(&id) - } +struct GlobalThreadList { + data: BTreeMap>, } -impl ThreadAffinity { - /// Mask value for a thread to be scheduled onto any CPU - pub const ANY_CPU: u64 = u64::MAX; +static THREADS: IrqSafeRwLock = IrqSafeRwLock::new(GlobalThreadList::new()); - /// Constructs an affinity mask allowing a thread to be scheduled onto any CPU - pub const fn any_cpu() -> Self { - Self(AtomicU64::new(Self::ANY_CPU)) - } - - /// Constructs an affinity mask targeting a single CPU - pub const fn only_cpu(index: usize) -> Self { - Self(AtomicU64::new(1 << index)) - } - - /// Returns the current value of the thread's CPU affinity mask - #[inline] - pub fn get(&self) -> u64 { - self.0.load(Ordering::Relaxed) - } - - /// Updates the thread's CPU affinity mask - #[inline] - pub fn set(&self, value: u64) { - debug_assert_ne!(value, 0); - self.0.store(value, Ordering::Relaxed); - } +// TODO this is ugly? +#[no_mangle] +unsafe extern "C" fn __arch_drop_thread(thread_ptr: *const Thread) { + let thread = Arc::from_raw(thread_ptr); + Thread::remove_from_list(thread.id); + Arc::decrement_strong_count(thread_ptr); } impl Thread { fn new( id: ThreadId, name: Option, - process: Option>, + process: Option, space: Option>, context: TaskContextImpl, ) -> Arc { @@ -202,7 +121,7 @@ impl Thread { } /// Constructs a new kernel-space thread - pub fn new_kthread>(name: S, context: TaskContextImpl) -> Arc { + pub fn new_kthread>(name: N, context: TaskContextImpl) -> Arc { Self::new( ThreadId::next_kernel(), Some(name.into()), @@ -214,7 +133,7 @@ impl Thread { /// Constructs a new user-space thread pub fn new_uthread( - parent: Arc, + parent: ProcessId, space: Arc, context: TaskContextImpl, ) -> Arc { @@ -257,9 +176,19 @@ impl Thread { self.space.as_ref().unwrap() } - /// Returns the thread's parent process, panics if there's none - pub fn process(&self) -> &Arc { - self.process.as_ref().unwrap() + /// Returns the thread's parent process ID, panics if there's none + pub fn process_id(&self) -> ProcessId { + self.process.unwrap() + } + + /// Returns the thread's parent process reference + pub fn try_get_process(&self) -> Option> { + self.process.and_then(PM::get) + } + + /// Returns the thread's parent process reference, panics if there's none + pub fn process(&self) -> Arc { + PM::get(self.process.unwrap()).unwrap() } /// Removes the thread from the thread list @@ -333,7 +262,8 @@ impl Thread { queue.yield_cpu(); }, (false, ThreadState::Running) => { - debugln!("deq remote {:?}", self.id); + // XXX + // debugln!("deq remote {:?}", self.id); debug_assert!(sched.in_queue); } (_, ThreadState::Ready) => { @@ -376,7 +306,7 @@ impl Thread { /// Returns the current thread on the CPU, if any is present pub fn get_current() -> Option { // IrqGuard is held throughout - let cpu = Cpu::local(); + let cpu = CpuImpl::::local(); let thread = cpu.current_thread_id().and_then(Self::get); thread.map(|t| CurrentThread(t, cpu.into_guard())) @@ -386,12 +316,53 @@ impl Thread { pub fn get(id: ThreadId) -> Option> { THREADS.read().get(id).cloned() } + + pub unsafe fn upgrade(weak: &Weak) -> Option { + let guard = IrqGuard::acquire(); + let strong = weak.upgrade()?; + Some(CurrentThread(strong, guard)) + } +} + +impl GlobalThreadList { + pub const fn new() -> Self { + Self { + data: BTreeMap::new(), + } + } + + #[inline] + pub fn get(&self, id: ThreadId) -> Option<&Arc> { + self.data.get(&id) + } + + pub fn insert(&mut self, thread: Arc) { + let id = thread.id; + debug_assert!(!self.data.contains_key(&id)); + self.data.insert(id, thread); + } + + pub fn remove(&mut self, id: ThreadId) -> Option> { + self.data.remove(&id) + } +} + +unsafe impl Sync for Thread {} + +impl ArcWake for Thread { + fn wake_by_ref(arc_self: &Arc) { + arc_self.clone().enqueue() + } } impl CurrentThread { + pub fn downgrade(self) -> Weak { + Arc::downgrade(&self.0) + } + /// Terminate the current thread - pub fn exit(&self, code: ExitCode) -> ! { - if let Some(process) = self.process.as_ref() { + pub fn exit(&self, code: ExitCode) -> ! { + if let Some(process) = self.try_get_process::() { process.handle_thread_exit(self.id, code); } self.dequeue(ThreadState::Terminated); @@ -401,13 +372,10 @@ impl CurrentThread { // TODO: test multithreaded process exit /// Terminate the parent process of the thread, including all other threads and the current /// thread itself - pub fn exit_process(&self, code: ExitCode) -> ! { + pub fn exit_process(&self, code: ExitCode) -> ! { let _guard = IrqGuard::acquire(); - let process = self - .process - .clone() - .expect("exit_process() called on a detached thread"); + let process = self.process::(); let p = process.clone(); @@ -416,7 +384,7 @@ impl CurrentThread { } .unwrap(); - self.exit(code) + self.exit::(code) } /// Suspends a thread from further execution until it is awoken @@ -436,7 +404,7 @@ impl CurrentThread { /// /// This function is only meant to be called right before returning from an userspace /// exception handler. - pub unsafe fn handle_pending_signals(&self, frame: &mut F) { + pub unsafe fn handle_pending_signals(&self, frame: &mut F) { if !self.id.is_user() { return; } @@ -446,7 +414,7 @@ impl CurrentThread { let Some(entry) = inner.signal_entry.as_ref() else { drop(inner); - self.exit_process(ExitCode::BySignal(signal)); + self.exit_process::(ExitCode::BySignal(signal)); }; // TODO check if really in a syscall, lol @@ -468,7 +436,7 @@ impl CurrentThread { ); // Setup return to signal handler - debugln!( + log::debug!( "Signal entry @ pc={:#x}, sp={:#x} (top = {:#x})", entry.entry, usp, @@ -491,85 +459,3 @@ impl Deref for CurrentThread { &self.0 } } - -impl ThreadId { - /// Generates a new kernel-space thread ID - pub fn next_kernel() -> Self { - static COUNT: AtomicU64 = AtomicU64::new(1); - let id = COUNT.fetch_add(1, Ordering::SeqCst); - Self::Kernel(id) - } - - /// Generates a new user-space thread ID - pub fn next_user() -> Self { - static COUNT: AtomicU64 = AtomicU64::new(1); - let id = COUNT.fetch_add(1, Ordering::SeqCst); - Self::User(id) - } - - /// Returns the number of the userspace thread represented by this ID. - /// - /// # Panics - /// - /// Will panic if this [ThreadId] does not represent a user-space thread. - pub fn as_user(&self) -> u64 { - match self { - Self::Kernel(_) => panic!(), - &Self::User(id) => id, - } - } - - /// Returns `true` if the [ThreadId] represents a user-space thread - pub fn is_user(&self) -> bool { - matches!(self, ThreadId::User(_)) - } -} - -impl fmt::Display for ThreadId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Kernel(id) => write!(f, "#[{id}]"), - Self::User(id) => write!(f, "#{id}"), - } - } -} - -// External API - -#[no_mangle] -fn __create_kthread( - name: String, - func: extern "C" fn(usize) -> !, - arg: usize, -) -> Result, Error> { - let context = TaskContextImpl::kernel(func, arg)?; - Ok(Thread::new_kthread(name, context)) -} - -#[no_mangle] -fn __enqueue(t: &Arc) { - t.enqueue(); -} - -#[no_mangle] -fn __current_thread() -> CurrentThread { - Thread::current() -} - -#[no_mangle] -fn __suspend_current(t: &CurrentThread) -> Result<(), Error> { - t.suspend() -} - -#[no_mangle] -fn __exit_current(t: &CurrentThread, code: ExitCode) -> ! { - t.exit(code); -} - -#[no_mangle] -unsafe fn __yield() { - // XXX - todo!(); - // let _irq = IrqGuard::acquire(); - // Cpu::local().queue().yield_cpu(); -} diff --git a/libk-thread/src/types.rs b/libk-thread/src/types.rs new file mode 100644 index 00000000..1cc3c2cb --- /dev/null +++ b/libk-thread/src/types.rs @@ -0,0 +1,220 @@ +use core::{ + fmt, + mem::size_of, + sync::atomic::{AtomicU64, Ordering}, +}; + +use atomic_enum::atomic_enum; + +/// Represents the states a thread can be at some point in time +#[atomic_enum] +#[derive(PartialEq)] +pub enum ThreadState { + /// Thread is ready for execution and is present in some CPU's queue + Ready, + /// Thread is currently being executed by some CPU + Running, + /// Thread is present in a global list, but is not queued for execution until it is resumed + Suspended, + /// Thread is terminated and waits to be reaped + Terminated, +} + +/// Mask describing CPUs a thread is allowed to be scheduled to +#[derive(Debug)] +#[repr(transparent)] +pub struct ThreadAffinity(AtomicU64); + +/// Unique number describing a single kernel or userspace thread +#[derive(Debug, PartialEq, Eq, Clone, Copy, Ord, PartialOrd)] +pub enum ThreadId { + /// Describes a kernel-space thread + Kernel(u64), + /// Describes an user-space thread + User(u64), +} + +/// Unique number assigned to each [Process] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct ProcessId(pub u64); + +// TLS layout (x86-64): +// | mem_size | uthread_size | +// | Data .......| self, ??? | +// +// TLS layout (aarch64): +// | uthread_size (0x10?) | mem_size | +// | ??? | Data .....| + +/// Describes Thread-Local Storage of a process +#[derive(Clone, Debug)] +pub struct ProcessTlsInfo { + /// Location of the TLS master copy within the process's memory + pub master_copy_base: usize, + /// Layout of the TLS + pub layout: ProcessTlsLayout, +} + +/// Describes TLS layout for a program image +#[derive(Clone, Debug)] +pub struct ProcessTlsLayout { + /// Data offset from the TLS base + pub data_offset: usize, + /// struct uthread offset from the TLS base + pub uthread_offset: usize, + /// Pointer offset from the TLS base. The pointer is passed to the userspace + pub ptr_offset: usize, + + /// Data size of the TLS segment + pub data_size: usize, + /// Memory size of the TLS segment (mem_size >= data_size) + pub mem_size: usize, + + /// Overall allocation size of the TLS data + pub full_size: usize, +} + +impl ThreadAffinity { + /// Mask value for a thread to be scheduled onto any CPU + pub const ANY_CPU: u64 = u64::MAX; + + /// Constructs an affinity mask allowing a thread to be scheduled onto any CPU + pub const fn any_cpu() -> Self { + Self(AtomicU64::new(Self::ANY_CPU)) + } + + /// Constructs an affinity mask targeting a single CPU + pub const fn only_cpu(index: usize) -> Self { + Self(AtomicU64::new(1 << index)) + } + + /// Returns the current value of the thread's CPU affinity mask + #[inline] + pub fn get(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } + + /// Updates the thread's CPU affinity mask + #[inline] + pub fn set(&self, value: u64) { + debug_assert_ne!(value, 0); + self.0.store(value, Ordering::Relaxed); + } +} + +impl ThreadId { + /// Generates a new kernel-space thread ID + pub fn next_kernel() -> Self { + static COUNT: AtomicU64 = AtomicU64::new(1); + let id = COUNT.fetch_add(1, Ordering::SeqCst); + Self::Kernel(id) + } + + /// Generates a new user-space thread ID + pub fn next_user() -> Self { + static COUNT: AtomicU64 = AtomicU64::new(1); + let id = COUNT.fetch_add(1, Ordering::SeqCst); + Self::User(id) + } + + /// Returns the number of the userspace thread represented by this ID. + /// + /// # Panics + /// + /// Will panic if this [ThreadId] does not represent a user-space thread. + pub fn as_user(&self) -> u64 { + match self { + Self::Kernel(_) => panic!(), + &Self::User(id) => id, + } + } + + /// Returns `true` if the [ThreadId] represents a user-space thread + pub fn is_user(&self) -> bool { + matches!(self, ThreadId::User(_)) + } +} + +impl fmt::Display for ThreadId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Kernel(id) => write!(f, "#[{id}]"), + Self::User(id) => write!(f, "#{id}"), + } + } +} + +impl fmt::Display for ProcessId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "", self.0) + } +} + +// XXX TODO Remove this +impl From for u32 { + fn from(value: ProcessId) -> Self { + value.0 as _ + } +} + +impl From for ProcessId { + fn from(value: u32) -> Self { + Self(value as _) + } +} + +impl ProcessId { + /// Generates a new [ProcessId] + pub fn next() -> Self { + static COUNTER: AtomicU64 = AtomicU64::new(1); + let id = COUNTER.fetch_add(1, Ordering::SeqCst); + Self(id) + } +} + +#[cfg(target_arch = "aarch64")] +impl ProcessTlsLayout { + /// Constructs a new thread-local storage layout info struct + pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { + debug_assert!(align.is_power_of_two()); + let tls_block0_offset = (size_of::() * 2 + align - 1) & !(align - 1); + + let full_size = (tls_block0_offset + mem_size + align - 1) & !(align - 1); + + Self { + data_offset: tls_block0_offset, + uthread_offset: 0, + ptr_offset: 0, + + data_size, + mem_size, + full_size, + } + } +} + +#[cfg(target_arch = "x86_64")] +impl ProcessTlsLayout { + /// Constructs a new thread-local storage layout info struct + pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { + // The static TLS blocks are placed below TP + // TP points to the TCB + debug_assert!(align.is_power_of_two()); + let back_size = (mem_size + align - 1) & !(align - 1); + // Self-pointer + let forward_size = size_of::(); + + let full_size = back_size + forward_size; + + Self { + data_offset: 0, + uthread_offset: back_size, + ptr_offset: back_size, + + data_size, + mem_size, + full_size, + } + } +} diff --git a/libk/Cargo.toml b/libk/Cargo.toml index 414a7248..35073b8f 100644 --- a/libk/Cargo.toml +++ b/libk/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] libk-mm = { path = "../libk-mm" } libk-util = { path = "../libk-util" } +libk-thread = { path = "../libk-thread" } kernel-arch = { path = "../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/libk/src/api.rs b/libk/src/api.rs index c3fc5add..657f9567 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -1,30 +1,10 @@ use core::time::Duration; -use alloc::{string::String, sync::Arc}; -use yggdrasil_abi::{ - error::Error, - process::{ExitCode, Signal}, -}; - -use crate::thread::{CurrentThread, Thread}; +use yggdrasil_abi::{error::Error, process::Signal}; extern "Rust" { - pub fn __cpu_index() -> usize; pub fn __cpu_count() -> usize; - // SAFETY: Both the kernel-side and api-side Threads are sized and Arc<___> has the same value - // for them - pub fn __create_kthread( - name: String, - func: extern "C" fn(usize) -> !, - arg: usize, - ) -> Result, Error>; - pub fn __enqueue(t: &Arc); - pub fn __current_thread() -> CurrentThread; - pub fn __suspend_current(t: &CurrentThread) -> Result<(), Error>; - pub fn __exit_current(t: &CurrentThread, code: ExitCode) -> !; - pub fn __yield(); - pub fn __monotonic_timestamp() -> Result; pub fn __signal_process_group(group_id: u32, signal: Signal); diff --git a/libk/src/arch.rs b/libk/src/arch.rs new file mode 100644 index 00000000..933f7c5d --- /dev/null +++ b/libk/src/arch.rs @@ -0,0 +1,85 @@ +use core::ops::{Deref, DerefMut}; + +use device_api::interrupt::IpiMessage; +use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl, LocalCpuImpl}; +use libk_thread::sched::CpuQueue; +use libk_util::sync::IrqGuard; + +/// Kernel wrapper for local CPU info structure. See [kernel_arch::LocalCpuImpl]. +#[repr(transparent)] +pub struct LocalCpu<'a>(LocalCpuImpl<'a, CpuQueue>); +/// Kernel wrapper for per-CPU info structure. See [kernel_arch::LocalCpuImpl]. +#[repr(transparent)] +pub struct Cpu(CpuImpl); + +impl Cpu { + pub unsafe fn init_local( + id: Option, + data: ::PerCpuData, + ) { + ArchitectureImpl::init_local_cpu::(id, data) + } + + /// Returns local CPU reference + #[inline] + pub fn local<'a>() -> LocalCpu<'a> { + LocalCpu(CpuImpl::local()) + } + + /// Returns local CPU reference or None if it hasn't yet been initialized + #[inline] + pub fn try_local<'a>() -> Option> { + CpuImpl::try_local().map(LocalCpu) + } + + /// Pushes a message to the IPI queue + #[inline] + pub fn push_ipi_queue(cpu_id: u32, msg: IpiMessage) { + CpuImpl::::push_ipi_queue(cpu_id, msg) + } + + /// Initialize the IPI queues for all present CPUs + #[inline] + pub fn init_ipi_queues(cpu_count: usize) { + CpuImpl::::init_ipi_queues(cpu_count) + } +} + +impl Deref for Cpu { + type Target = CpuImpl; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Cpu { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl<'a> LocalCpu<'a> { + /// Converts the local CPU handle into its IRQ guard + pub fn into_guard(self) -> IrqGuard { + self.0.into_guard() + } +} + +impl<'a> Deref for LocalCpu<'a> { + type Target = CpuImpl; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl<'a> DerefMut for LocalCpu<'a> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.deref_mut() + } +} diff --git a/libk/src/lib.rs b/libk/src/lib.rs index 472b66b0..a8b97f51 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -18,27 +18,32 @@ use core::time::Duration; +use arch::Cpu; +use kernel_arch::CpuImpl; use yggdrasil_abi::{error::Error, process::Signal}; extern crate alloc; +pub use libk_thread::{block, runtime}; + pub(crate) mod api; +pub mod arch; pub mod device; -pub mod runtime; pub mod sync; pub mod thread; -#[inline] -pub fn cpu_index() -> usize { - unsafe { api::__cpu_index() } -} - #[inline] pub fn cpu_count() -> usize { unsafe { api::__cpu_count() } } +/// Returns local CPU index +#[inline] +pub fn cpu_index() -> u32 { + Cpu::try_local().as_deref().map(CpuImpl::id).unwrap_or(0) +} + #[inline] pub fn signal_process_group(group_id: u32, signal: Signal) { unsafe { api::__signal_process_group(group_id, signal) } diff --git a/libk/src/sync/mutex.rs b/libk/src/sync/mutex.rs index ac4e3308..c0ec9003 100644 --- a/libk/src/sync/mutex.rs +++ b/libk/src/sync/mutex.rs @@ -6,11 +6,11 @@ use core::{ use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; +use kernel_arch::task::Scheduler; +use libk_thread::{sched::CpuQueue, thread::Thread}; use libk_util::sync::LockMethod; use yggdrasil_abi::error::Error; -use crate::{api, thread::Thread}; - struct ThreadedMutexInner { queue: ArrayQueue>, lock: AtomicU32, @@ -77,7 +77,7 @@ impl<'q> LockMethod<'q> for ThreadedMutexInner { // Yield current thread to avoid congestion unsafe { - api::__yield(); + CpuQueue::local().yield_cpu(); } } } diff --git a/libk/src/thread.rs b/libk/src/thread.rs index 0a30fa63..2d66b87f 100644 --- a/libk/src/thread.rs +++ b/libk/src/thread.rs @@ -1,97 +1,8 @@ -use core::ops::Deref; - -use alloc::{ - boxed::Box, - string::String, - sync::{Arc, Weak}, -}; -use futures_util::task::ArcWake; use kernel_arch::KernelTableManagerImpl; use libk_mm::phys::GlobalPhysicalAllocator; -use libk_util::sync::IrqGuard; -use yggdrasil_abi::{error::Error, process::ExitCode}; - -use crate::api; pub use kernel_arch::task::{TaskContext, TaskFrame, Termination}; pub type TaskContextImpl = kernel_arch::TaskContextImpl; pub trait ForkFrame = kernel_arch::task::ForkFrame; - -#[repr(C)] -pub(crate) struct Thread(!); -#[repr(C)] -pub(crate) struct CurrentThread(Arc, IrqGuard); - -impl Thread { - pub fn spawn, T: Termination, F: FnOnce() -> T + Send + 'static>( - name: S, - f: F, - ) -> Result<(), Error> { - extern "C" fn closure_wrapper T + Send + 'static>( - closure_addr: usize, - ) -> ! { - let closure = unsafe { Box::from_raw(closure_addr as *mut F) }; - let result = closure(); - Thread::current().exit(result.into_exit_code()); - unreachable!(); - } - - let closure = Box::new(f); - log::debug!("closure: {:p}", closure); - let thread = unsafe { - api::__create_kthread( - name.into(), - closure_wrapper::, - Box::into_raw(closure) as usize, - ) - }?; - - thread.enqueue(); - - Ok(()) - } - - pub fn current() -> CurrentThread { - unsafe { api::__current_thread() } - } - - pub fn enqueue(self: &Arc) { - unsafe { api::__enqueue(self) } - } - - pub unsafe fn upgrade(weak: &Weak) -> Option { - let guard = IrqGuard::acquire(); - let strong = weak.upgrade()?; - Some(CurrentThread(strong, guard)) - } -} - -impl CurrentThread { - pub fn suspend(&self) -> Result<(), Error> { - unsafe { api::__suspend_current(self) } - } - - pub fn exit(&self, code: ExitCode) { - unsafe { api::__exit_current(self, code) } - } - - pub fn downgrade(self) -> Weak { - Arc::downgrade(&self.0) - } -} - -impl ArcWake for Thread { - fn wake_by_ref(arc_self: &Arc) { - arc_self.clone().enqueue() - } -} - -impl Deref for CurrentThread { - type Target = Arc; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 2f2c91a6..eb8fe2a6 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -16,9 +16,10 @@ use abi::{ use kernel_arch::{task::TaskFrame, Architecture, ArchitectureImpl}; use kernel_arch_aarch64::context::ExceptionFrame; use libk::device::external_interrupt_controller; +use libk_thread::thread::Thread; use tock_registers::interfaces::{Readable, Writeable}; -use crate::{debug::LogLevel, syscall::raw_syscall_handler, task::thread::Thread}; +use crate::{debug::LogLevel, syscall::raw_syscall_handler, task::process::ProcessManagerImpl}; /// Initializes the exception/interrupt vectors. May be called repeatedly (though that makes no /// sense). @@ -108,7 +109,7 @@ extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) { unsafe { let thread = Thread::current(); - thread.handle_pending_signals(frame); + thread.handle_pending_signals::(frame); } } @@ -121,7 +122,7 @@ extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) { unsafe { let thread = Thread::current(); - thread.handle_pending_signals(frame); + thread.handle_pending_signals::(frame); } } diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 8e497b3a..e9f275dc 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -15,15 +15,13 @@ use device_api::{ }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; use kernel_arch_aarch64::GicInterface; -use libk::device::register_external_interrupt_controller; +use libk::{arch::Cpu, cpu_index, device::register_external_interrupt_controller}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::{DeviceMemoryIo, RawDeviceMemoryMapping}, }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::{arch::cpu_index, task::Cpu}; - use self::{gicc::Gicc, gicd::Gicd}; use super::{smp::CPU_COUNT, AArch64}; diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index cde08fae..f564b08c 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -18,7 +18,7 @@ use kernel_arch_aarch64::{ }, ArchitectureImpl, PerCpuData, }; -use libk::device::external_interrupt_controller; +use libk::{arch::Cpu, device::external_interrupt_controller}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, phys::PhysicalMemoryRegion, @@ -35,7 +35,6 @@ use crate::{ device::{self, power::arm_psci::Psci}, fs::{Initrd, INITRD_DATA}, mem::heap, - task::Cpu, }; use self::gic::Gic; @@ -255,7 +254,7 @@ impl AArch64 { let per_cpu = PerCpuData { gic: OneTimeInit::new(), }; - Cpu::init_local(per_cpu); + Cpu::init_local(None, per_cpu); if is_bsp { ygg_driver_pci::register_vendor_driver( diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 932341f4..84b9ade0 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -11,13 +11,11 @@ use device_api::{ Device, }; use device_tree::device_tree_driver; -use libk::{device::external_interrupt_controller, runtime}; +use kernel_arch::task::Scheduler; +use libk::{arch::Cpu, device::external_interrupt_controller, runtime}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use crate::{ - arch::{Platform, PLATFORM}, - task::Cpu, -}; +use crate::arch::{Platform, PLATFORM}; /// ARM Generic Timer driver pub struct ArmTimer { diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 162dcf0d..f165380a 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -10,11 +10,9 @@ use device_api::{ timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl}; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk_mm::table::EntryLevel; -use crate::task::Cpu; - cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod aarch64; @@ -103,16 +101,6 @@ pub trait Platform { // External API for architecture specifics -/// Returns local CPU index -pub fn cpu_index() -> u32 { - Cpu::try_local().as_deref().map(CpuImpl::id).unwrap_or(0) -} - -#[no_mangle] -fn __cpu_index() -> usize { - Cpu::local().id() as _ -} - #[no_mangle] fn __cpu_count() -> usize { PlatformImpl::cpu_count() diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index fcc79f92..56e51007 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -11,6 +11,7 @@ use device_api::{ Device, }; use kernel_arch_x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, LocalApicInterface}; +use libk::arch::Cpu; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, @@ -26,10 +27,7 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use crate::{ - arch::x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}, - task::Cpu, -}; +use crate::arch::x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}; use super::{ APIC_IPI_VECTOR, APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR, diff --git a/src/arch/x86_64/apic/mod.rs b/src/arch/x86_64/apic/mod.rs index 555ffce0..89d8ce4c 100644 --- a/src/arch/x86_64/apic/mod.rs +++ b/src/arch/x86_64/apic/mod.rs @@ -2,11 +2,16 @@ use core::arch::global_asm; +use kernel_arch::task::Scheduler; use kernel_arch_x86_64::context::IrqFrame; -use libk::device::{external_interrupt_controller, message_interrupt_controller}; +use libk::{ + arch::Cpu, + device::{external_interrupt_controller, message_interrupt_controller}, +}; +use libk_thread::thread::Thread; use static_assertions::{const_assert, const_assert_eq}; -use crate::task::{thread::Thread, Cpu}; +use crate::task::process::ProcessManagerImpl; use super::exception; @@ -67,7 +72,7 @@ unsafe extern "C" fn irq_handler(vector: usize, frame: *mut IrqFrame) { cpu.local_apic().clear_interrupt(); if let Some(thread) = Thread::get_current() { - thread.handle_pending_signals(frame); + thread.handle_pending_signals::(frame); } } @@ -83,7 +88,7 @@ unsafe extern "C" fn msi_handler(vector: usize, frame: *mut IrqFrame) { cpu.local_apic().clear_interrupt(); if let Some(thread) = Thread::get_current() { - thread.handle_pending_signals(frame); + thread.handle_pending_signals::(frame); } } @@ -98,7 +103,7 @@ unsafe extern "C" fn local_timer_irq_handler(frame: *mut IrqFrame) { } if let Some(thread) = Thread::get_current() { - thread.handle_pending_signals(frame); + thread.handle_pending_signals::(frame); } } diff --git a/src/arch/x86_64/exception.rs b/src/arch/x86_64/exception.rs index 9017bd8c..1c032cfd 100644 --- a/src/arch/x86_64/exception.rs +++ b/src/arch/x86_64/exception.rs @@ -3,12 +3,11 @@ use core::{arch::global_asm, mem::size_of, ptr::addr_of}; use abi::{primitive_enum, process::Signal}; use kernel_arch_x86_64::{context::ExceptionFrame, registers::CR3}; +use libk::arch::Cpu; +use libk_thread::thread::Thread; use tock_registers::interfaces::Readable; -use crate::{ - arch::x86_64::apic, - task::{thread::Thread, Cpu}, -}; +use crate::{arch::x86_64::apic, task::process::ProcessManagerImpl}; use super::PLATFORM; @@ -224,7 +223,7 @@ extern "C" fn __x86_64_exception_handler(frame: *mut ExceptionFrame) { user_exception_inner(kind, frame); unsafe { - Thread::current().handle_pending_signals(frame); + Thread::current().handle_pending_signals::(frame); } } else { if kind == ExceptionKind::NonMaskableInterrupt { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 8a8d9dff..06f13397 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -19,7 +19,7 @@ use kernel_arch_x86_64::{ PerCpuData, }; use kernel_fs::devfs; -use libk::device::register_external_interrupt_controller; +use libk::{arch::Cpu, device::register_external_interrupt_controller}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, @@ -52,7 +52,6 @@ use crate::{ }, fs::{Initrd, INITRD_DATA}, mem::heap, - task::Cpu, }; use self::{ @@ -299,7 +298,7 @@ impl X86_64 { local_apic, }; cpuid::enable_features(); - Cpu::init_local(cpu_id as _, cpu_data); + Cpu::init_local(Some(cpu_id as _), cpu_data); syscall::init_syscall(); if cpu_id == 0 { diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index 62a01d78..f4fa1197 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -8,6 +8,7 @@ use kernel_arch_x86_64::mem::{ table::{PageAttributes, PageEntry, PageTable, L1, L2}, KERNEL_TABLES, }; +use libk::arch::Cpu; use libk_mm::{ address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys, @@ -15,7 +16,7 @@ use libk_mm::{ TableAllocatorImpl, }; -use crate::{arch::x86_64::boot::__x86_64_ap_entry, task::Cpu}; +use crate::arch::x86_64::boot::__x86_64_ap_entry; use super::acpi::AcpiAllocator; diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index d1b42eeb..faff7bc4 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -8,11 +8,12 @@ use kernel_arch_x86_64::{ context::SyscallFrame, registers::{MSR_IA32_EFER, MSR_IA32_LSTAR, MSR_IA32_SFMASK, MSR_IA32_STAR}, }; +use libk_thread::thread::Thread; use tock_registers::interfaces::{ReadWriteable, Writeable}; use crate::{ syscall::raw_syscall_handler, - task::{process::Process, thread::Thread}, + task::process::{ProcessImpl, ProcessManagerImpl}, }; fn syscall_inner(frame: &mut SyscallFrame) { @@ -24,7 +25,7 @@ fn syscall_inner(frame: &mut SyscallFrame) { } if frame.rax == usize::from(SyscallFunction::Fork) as u64 { unsafe { - Process::raw_fork(frame); + ProcessImpl::raw_fork(frame); return; } } @@ -40,7 +41,7 @@ extern "C" fn __x86_64_syscall_handler(frame: *mut SyscallFrame) { let thread = Thread::current(); unsafe { - thread.handle_pending_signals(frame); + thread.handle_pending_signals::(frame); } } diff --git a/src/debug.rs b/src/debug.rs index 4f020e9a..5c28f4b7 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -2,10 +2,9 @@ use core::fmt::{self, Arguments}; use abi::error::Error; +use libk_thread::process::Process; use libk_util::{ring::RingBuffer, sync::IrqSafeSpinlock, StaticVector}; -use crate::task::process::Process; - const MAX_DEBUG_SINKS: usize = 4; const RING_LOGGER_CAPACITY: usize = 65536; @@ -64,7 +63,7 @@ macro_rules! log_print_raw { macro_rules! log_print { ($level:expr, $($args:tt)+) => { - log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::arch::cpu_index(), file!(), line!(), format_args!($($args)+)) + log_print_raw!($level, "cpu{}:{}:{}: {}", libk::cpu_index(), file!(), line!(), format_args!($($args)+)) }; } @@ -241,7 +240,7 @@ pub fn add_sink(sink: &'static dyn DebugSink, level: LogLevel) { } /// Print a trace message coming from a process -pub fn program_trace(process: &Process, message: &str) { +pub fn program_trace(process: &P, message: &str) { log_print_raw!( LogLevel::Trace, "[trace {} {:?}] {}\n", diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index bd400a8e..f72b51eb 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -13,13 +13,11 @@ use libk_mm::{ table::{EntryLevel, MapAttributes}, PageProvider, }; +use libk_thread::thread::Thread; use libk_util::sync::IrqSafeSpinlock; use ygg_driver_block::BlockDevice; -use crate::{ - arch::L3, - task::{process::ProcessId, thread::Thread}, -}; +use crate::{arch::L3, task::process::ProcessId}; use super::{DisplayDevice, DisplayDimensions}; @@ -143,7 +141,7 @@ impl BlockDevice for LinearFramebuffer { let thread = Thread::current(); match req { DeviceRequest::AcquireDevice => { - self.inner.lock().holder.replace(thread.process().id()); + self.inner.lock().holder.replace(thread.process_id()); Ok(()) } _ => todo!(), diff --git a/src/device/tty.rs b/src/device/tty.rs index 2317a348..bb279942 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -12,9 +12,13 @@ use abi::{ }; use device_api::serial::SerialDevice; use futures_util::Future; +use libk_thread::process::ProcessImpl; use libk_util::{ring::RingBuffer, sync::IrqSafeSpinlock, waker::QueueWaker}; -use crate::task::process::{Process, ProcessId}; +use crate::{ + proc::io::ProcessIoImpl, + task::process::{ProcessId, ProcessManagerImpl}, +}; struct TerminalRing { buffer: IrqSafeSpinlock>, @@ -167,7 +171,10 @@ pub trait TtyDevice: SerialDevice { if let Some(pgrp) = pgrp { drop(inner); - Process::signal_group(pgrp, Signal::Interrupted); + ProcessImpl::::signal_group( + pgrp, + Signal::Interrupted, + ); return; } else { debugln!("Terminal has no process group attached"); diff --git a/src/main.rs b/src/main.rs index 3cb0c578..d848477d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,13 +39,14 @@ #![no_main] use arch::Platform; +use libk::arch::Cpu; use libk_util::sync::SpinFence; use crate::{ arch::{PlatformImpl, PLATFORM}, fs::sysfs, mem::heap, - task::{spawn_kernel_closure, Cpu}, + task::spawn_kernel_closure, }; extern crate yggdrasil_abi as abi; diff --git a/src/mem/mod.rs b/src/mem/mod.rs index b4700c66..a6010756 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -1,17 +1,11 @@ //! Memory management utilities and types use core::{ - alloc::Layout, ffi::c_void, mem::{align_of, size_of}, }; -use abi::error::Error; -use libk_mm::{ - address::{PhysicalAddress, Virtualize}, - device::DeviceMemoryMapping, - process::ProcessAddressSpace, -}; +use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; use crate::arch::{Platform, PlatformImpl}; @@ -54,192 +48,6 @@ pub unsafe fn write_memory(address: PhysicalAddress, value: T) { } } -/// Helper trait to allow cross-address space access to pointers -pub trait ForeignPointer: Sized { - /// Perform a volatile pointer write without dropping the old value. - /// - /// # Panics - /// - /// The function panics if any of the following conditions is met: - /// - /// * The address of the pointer is not mapped in the `space`. - /// * The pointer is not writable. - /// * The pointer is misaligned. - /// - /// # Safety - /// - /// As this function allows direct memory writes, it is inherently unsafe. - unsafe fn write_foreign_volatile(self: *mut Self, space: &ProcessAddressSpace, value: Self); - - /// Performs pointer validation for given address space: - /// - /// * Checks if the pointer has proper alignment for the type. - /// * Checks if the pointer is mapped in the address space. - /// * Checks if the pointer is above the userspace memory boundary. - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_ptr<'a>( - self: *const Self, - space: &ProcessAddressSpace, - ) -> Result<&'a Self, Error>; - - /// [ForeignPointer::validate_user_ptr], with extra "writability" check. - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_mut<'a>( - self: *mut Self, - space: &ProcessAddressSpace, - ) -> Result<&'a mut Self, Error>; - - /// [ForeignPointer::validate_user_ptr], but for slices - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_slice<'a>( - self: *const Self, - len: usize, - space: &ProcessAddressSpace, - ) -> Result<&'a [Self], Error>; - - /// [ForeignPointer::validate_user_slice], but for mutable slices - /// - /// # Safety - /// - /// Even though this function does the necessary checks, it is still a raw pointer to reference - /// conversion, and thus is unsafe. - unsafe fn validate_user_slice_mut<'a>( - self: *mut Self, - len: usize, - space: &ProcessAddressSpace, - ) -> Result<&'a mut [Self], Error>; -} - -impl ForeignPointer for T { - unsafe fn write_foreign_volatile(self: *mut Self, space: &ProcessAddressSpace, value: T) { - // TODO check align - let addr = self as usize; - let start_page = addr & !0xFFF; - let end_page = (addr + size_of::() - 1) & !0xFFF; - let page_offset = addr & 0xFFF; - - if start_page != end_page { - todo!("Foreign pointer write crossed a page boundary"); - } - - let phys_page = space - .translate(start_page) - .expect("Address is not mapped in the target address space"); - - let virt_ptr = phys_page.add(page_offset).virtualize() as *mut T; - virt_ptr.write_volatile(value); - } - - unsafe fn validate_user_slice_mut<'a>( - self: *mut Self, - len: usize, - space: &ProcessAddressSpace, - ) -> Result<&'a mut [Self], Error> { - let base = self as usize; - let layout = Layout::array::(len).unwrap(); - - validate_user_align_size(base, &layout)?; - validate_user_region(space, base, layout.size(), true)?; - - Ok(core::slice::from_raw_parts_mut(self, len)) - } - - unsafe fn validate_user_slice<'a>( - self: *const Self, - len: usize, - space: &ProcessAddressSpace, - ) -> Result<&'a [Self], Error> { - let base = self as usize; - let layout = Layout::array::(len).unwrap(); - - validate_user_align_size(base, &layout)?; - validate_user_region(space, base, layout.size(), false)?; - - Ok(core::slice::from_raw_parts(self, len)) - } - - unsafe fn validate_user_mut<'a>( - self: *mut Self, - space: &ProcessAddressSpace, - ) -> Result<&'a mut Self, Error> { - let addr = self as usize; - let layout = Layout::new::(); - - // Common validation - validate_user_align_size(addr, &layout)?; - - // Validate that the pages covered by this address are mapped as writable by the process - // TODO for CoW this may differ - validate_user_region(space, addr, layout.size(), true)?; - - Ok(&mut *self) - } - - unsafe fn validate_user_ptr<'a>( - self: *const Self, - space: &ProcessAddressSpace, - ) -> Result<&'a Self, Error> { - let addr = self as usize; - let layout = Layout::new::(); - - // Common validation - validate_user_align_size(addr, &layout)?; - validate_user_region(space, addr, layout.size(), false)?; - - Ok(&*self) - } -} - -fn validate_user_align_size(addr: usize, layout: &Layout) -> Result<(), Error> { - // Explicitly disallow NULL - if addr == 0 { - return Err(Error::InvalidArgument); - } - // Validate alignment - if addr % layout.align() != 0 { - return Err(Error::InvalidArgument); - } - if addr + layout.size() > KERNEL_VIRT_OFFSET { - todo!(); - } - - Ok(()) -} - -/// Validates access to given userspace memory region with given constraints -pub fn validate_user_region( - space: &ProcessAddressSpace, - base: usize, - len: usize, - _need_write: bool, -) -> Result<(), Error> { - if base + len > crate::mem::KERNEL_VIRT_OFFSET { - panic!("Invalid argument"); - } - - let aligned_start = base & !0xFFF; - let aligned_end = (base + len + 0xFFF) & !0xFFF; - - for page in (aligned_start..aligned_end).step_by(0x1000) { - // TODO check writability - space.translate(page)?; - } - - Ok(()) -} - #[no_mangle] unsafe extern "C" fn memcpy(p0: *mut c_void, p1: *const c_void, len: usize) -> *mut c_void { compiler_builtins::mem::memcpy(p0 as _, p1 as _, len) as _ diff --git a/src/panic.rs b/src/panic.rs index c196f8d3..0d433ee8 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -3,13 +3,13 @@ use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use device_api::interrupt::{IpiDeliveryTarget, IpiMessage}; use kernel_arch::{Architecture, ArchitectureImpl}; +use libk::arch::Cpu; use libk_util::sync::{hack_locks, SpinFence}; use crate::{ arch::{Platform, PlatformImpl, PLATFORM}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, - task::Cpu, }; static PANIC_HANDLED_FENCE: SpinFence = SpinFence::new(); diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 5229de2c..4cddf3a4 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -21,15 +21,12 @@ use libk_mm::{ process::{ProcessAddressSpace, VirtualRangeBacking}, table::MapAttributes, }; +use libk_thread::{mem::ForeignPointer, process::Process, thread::Thread}; use vfs::{FileRef, IoContext, Read, Seek}; use crate::{ - mem::ForeignPointer, proc, - task::{ - process::{Process, ProcessImage}, - thread::Thread, - }, + task::process::{ProcessImage, ProcessImpl}, }; struct BufferPlacer<'a> { @@ -179,15 +176,15 @@ fn setup_context( fn setup_binary>( name: S, - parent: Option>, + parent: Option>, space: ProcessAddressSpace, image: ProcessImage, args: &Vec, envs: &Vec, -) -> Result<(Arc, Arc), Error> { +) -> Result<(Arc, Arc), Error> { let context = setup_context(&space, &image, args, envs)?; let (process, main) = - Process::new_with_main(name, parent, Arc::new(space), context, Some(image)); + ProcessImpl::new_with_main(name, parent, Arc::new(space), context, Some(image)); infoln!("exec::setup_binary -> {:?}", process.id()); Ok((process, main)) } @@ -244,11 +241,11 @@ fn xxx_load_program>( /// Loads a program from given `path` pub fn load>( ioctx: &mut IoContext, - parent: Option>, + parent: Option>, path: P, args: &[&str], envs: &[&str], -) -> Result<(Arc, Arc), Error> { +) -> Result<(Arc, Arc), Error> { let path = path.as_ref(); let args = args.iter().map(|&s| s.to_owned()).collect(); let envs = envs.iter().map(|&s| s.to_owned()).collect(); @@ -260,7 +257,7 @@ pub fn load>( /// XXX pub fn load_into>( - process: &Process, + process: &ProcessImpl, path: P, args: Vec, envs: Vec, @@ -270,7 +267,7 @@ pub fn load_into>( let path = path.as_ref().to_owned(); let space = process.space(); space.clear()?; - let (image, args, envs) = xxx_load_program(&space, io.ioctx(), &path, args, envs)?; + let (image, args, envs) = xxx_load_program(&space, io.ioctx_mut(), &path, args, envs)?; let context = setup_context(&space, &image, &args, &envs)?; Ok((context, image)) diff --git a/src/proc/io.rs b/src/proc/io.rs index 19a06351..19990b82 100644 --- a/src/proc/io.rs +++ b/src/proc/io.rs @@ -1,36 +1,57 @@ //! Process I/O management -use vfs::{FileSet, IoContext}; +use abi::error::Error; +use libk_thread::process::ProcessIo; +use vfs::{FileSet, IoContext, Node}; /// I/O context of a process, contains information like root, current directory and file /// descriptor table -pub struct ProcessIo { +pub struct ProcessIoImpl { ioctx: Option, /// Process' set of files pub files: FileSet, } -impl ProcessIo { - /// Constructs an uninitialized I/O context - pub fn new() -> Self { +impl ProcessIo for ProcessIoImpl { + type Node = Node; + + fn new() -> Self { Self { ioctx: None, files: FileSet::new(), } } + fn handle_exit(&mut self) { + self.files.close_all(); + } + + fn fork_from(&mut self, src: &Self) -> Result<(), Error> { + let new_ioctx = IoContext::inherit(src.ioctx()); + self.set_ioctx(new_ioctx); + + for (old_fd, old_file) in src.files.iter() { + let new_file = old_file.send()?; + self.files.set_file(*old_fd, new_file)?; + } + + Ok(()) + } +} + +impl ProcessIoImpl { /// Returns the associated [IoContext] reference - pub fn ioctx(&mut self) -> &mut IoContext { + pub fn ioctx_mut(&mut self) -> &mut IoContext { self.ioctx.as_mut().unwrap() } + /// Returns the associated [IoContext] reference + pub fn ioctx(&self) -> &IoContext { + self.ioctx.as_ref().unwrap() + } + /// Changes the [IoContext] of this struct pub fn set_ioctx(&mut self, ioctx: IoContext) { self.ioctx = Some(ioctx); } - - /// Handles process exit by closing all of its files - pub fn handle_exit(&mut self) { - self.files.close_all(); - } } diff --git a/src/syscall/arg.rs b/src/syscall/arg.rs index b772c851..f7049d8f 100644 --- a/src/syscall/arg.rs +++ b/src/syscall/arg.rs @@ -1,10 +1,9 @@ use abi::io::RawFd; -use yggdrasil_abi::error::Error; - -use crate::{ +use libk_thread::{ mem::{validate_user_region, ForeignPointer}, - task::thread::Thread, + thread::Thread, }; +use yggdrasil_abi::error::Error; // XXX diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index ee4f8a2e..b6f7ce09 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -24,6 +24,10 @@ use libk_mm::{ process::VirtualRangeBacking, table::{EntryLevelExt, MapAttributes}, }; +use libk_thread::{ + process::{Process, ProcessManager}, + thread::{CurrentThread, Thread}, +}; use libk_util::sync::IrqSafeSpinlockGuard; use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; @@ -33,23 +37,26 @@ use crate::{ arch::L3, debug::LogLevel, fs, - proc::{self, io::ProcessIo, random}, - task::{ - process::{Process, ProcessId}, - thread::{CurrentThread, Thread}, - }, + proc::{self, io::ProcessIoImpl, random}, + task::process::{ProcessId, ProcessImpl, ProcessManagerImpl}, }; mod arg; use arg::*; -fn run_with_io) -> T>(proc: &Process, f: F) -> T { +fn run_with_io) -> T>( + proc: &ProcessImpl, + f: F, +) -> T { let io = proc.io.lock(); f(io) } -fn run_with_io_at) -> Result>( - proc: &Process, +fn run_with_io_at< + T, + F: FnOnce(NodeRef, IrqSafeSpinlockGuard) -> Result, +>( + proc: &ProcessImpl, at: Option, f: F, ) -> Result { @@ -62,13 +69,13 @@ fn run_with_io_at) -> Resu }) .transpose()? // at = None - .unwrap_or_else(|| io.ioctx().cwd().clone()); + .unwrap_or_else(|| io.ioctx_mut().cwd().clone()); f(at, io) } fn sys_execve(thread: CurrentThread, options: &ExecveOptions) -> Result { - let process = thread.process(); + let process = thread.process::(); // Clone the options into the kernel let mut argv = Vec::new(); @@ -86,7 +93,7 @@ fn sys_execve(thread: CurrentThread, options: &ExecveOptions) -> Result Result { let thread = Thread::current(); - let process = thread.process(); + let process = thread.process::(); match func { SyscallFunction::DebugTrace => { @@ -123,7 +130,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result len = len.page_align_up::(); - run_with_io(process, |io| { + run_with_io(&process, |io| { let backing = match source { MappingSource::Anonymous => VirtualRangeBacking::anonymous(), &MappingSource::File(fd, offset) => { @@ -173,8 +180,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let opts = OpenOptions::from(args[3] as u32); let mode = FileMode::from(args[4] as u32); - run_with_io_at(process, at, |at, mut io| { - let file = io.ioctx().open(Some(at), path, opts, mode)?; + run_with_io_at(&process, at, |at, mut io| { + let file = io.ioctx_mut().open(Some(at), path, opts, mode)?; // TODO NO_CTTY? if process.session_terminal().is_none() @@ -193,9 +200,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - run_with_io_at(process, at, |at, mut io| { - let node = io.ioctx().find(Some(at), path, true, true)?; - let access = io.ioctx().check_access(vfs::Action::Read, &node)?; + run_with_io_at(&process, at, |at, mut io| { + let node = io.ioctx_mut().find(Some(at), path, true, true)?; + let access = io.ioctx_mut().check_access(vfs::Action::Read, &node)?; let file = node.open_directory(access)?; let fd = io.files.place_file(file, true)?; @@ -206,19 +213,19 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - run_with_io(process, |io| io.files.file(fd)?.read(data)) + run_with_io(&process, |io| io.files.file(fd)?.read(data)) } SyscallFunction::Write => { let fd = RawFd(args[0] as u32); let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - run_with_io(process, |io| io.files.file(fd)?.write(data)) + run_with_io(&process, |io| io.files.file(fd)?.write(data)) } SyscallFunction::Seek => { let fd = RawFd(args[0] as u32); let pos = SeekFrom::from(args[1]); - run_with_io(process, |io| { + run_with_io(&process, |io| { io.files.file(fd)?.seek(pos).map(|v| v as usize) }) } @@ -229,12 +236,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result args[2] as usize, )?; - run_with_io(process, |io| io.files.file(fd)?.read_dir(buffer)) + run_with_io(&process, |io| io.files.file(fd)?.read_dir(buffer)) } SyscallFunction::Close => { let fd = RawFd(args[0] as u32); - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let res = io.files.close_file(fd); if res == Err(Error::InvalidFile) { @@ -249,9 +256,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::Mount => { let options = arg_user_ref::(args[0] as usize)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let fs_root = fs::create_filesystem(options)?; - io.ioctx().mount(options.target, fs_root)?; + io.ioctx_mut().mount(options.target, fs_root)?; Ok(0) }) } @@ -262,7 +269,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let fd = RawFd(args[0] as u32); let req = arg_user_mut::(args[1] as usize)?; - run_with_io(process, |io| { + run_with_io(&process, |io| { let file = io.files.file(fd)?; // let node = file.node().ok_or(Error::InvalidFile)?; file.device_request(req)?; @@ -275,12 +282,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let buffer = arg_user_mut::>(args[3] as usize)?; let follow = args[4] != 0; - run_with_io_at(process, at, |at, mut io| { + run_with_io_at(&process, at, |at, mut io| { let node = if path.is_empty() { at // at.ok_or(Error::InvalidArgument)? } else { - io.ioctx().find(Some(at), path, follow, true)? + io.ioctx_mut().find(Some(at), path, follow, true)? }; let metadata = node.metadata()?; @@ -303,7 +310,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let _update = arg_user_ref::(args[3] as usize)?; let _follow = args[4] != 0; - run_with_io_at(process, at, |_at, _io| { + run_with_io_at(&process, at, |_at, _io| { todo!(); }) } @@ -312,8 +319,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let path = arg_user_str(args[1] as usize, args[2] as usize)?; let mode = FileMode::from(args[3] as u32); - run_with_io_at(process, at, |at, mut io| { - io.ioctx().create_directory(Some(at), path, mode)?; + run_with_io_at(&process, at, |at, mut io| { + io.ioctx_mut().create_directory(Some(at), path, mode)?; Ok(0) }) } @@ -321,8 +328,8 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let at = arg_option_fd(args[0] as u32); let path = arg_user_str(args[1] as usize, args[2] as usize)?; - run_with_io_at(process, at, |at, mut io| { - io.ioctx().remove_file(Some(at), path)?; + run_with_io_at(&process, at, |at, mut io| { + io.ioctx_mut().remove_file(Some(at), path)?; Ok(0) }) } @@ -332,7 +339,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::CreatePipe => { let ends: &mut [MaybeUninit; 2] = arg_user_mut(args[0] as usize)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let (read, write) = File::new_pipe_pair(256); let read_fd = io.files.place_file(read, true)?; @@ -347,7 +354,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result Ok(0) }) } - SyscallFunction::CreatePoll => run_with_io(process, |mut io| { + SyscallFunction::CreatePoll => run_with_io(&process, |mut io| { let poll = File::new_poll_channel(); let fd = io.files.place_file(poll, true)?; @@ -359,7 +366,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result PollControl::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; let fd = RawFd::from(args[2] as u32); - run_with_io(process, |io| { + run_with_io(&process, |io| { let poll_file = io.files.file(poll_fd)?; let poll = poll_file.as_poll_channel()?; @@ -378,7 +385,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let timeout = arg_user_ref::>(args[1] as usize)?; let output = arg_user_mut::)>>(args[2] as usize)?; - run_with_io(process, |io| { + run_with_io(&process, |io| { let poll_file = io.files.file(poll_fd)?; let poll = poll_file.as_poll_channel()?; @@ -395,7 +402,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result debugln!("OpenChannel {:?}, with_sub={}", name, with_sub); - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = File::new_message_channel(name, with_sub); let fd = io.files.place_file(file, true)?; @@ -407,7 +414,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let message = arg_user_ref::(args[1] as usize)?; let destination = MessageDestination::from(args[2]); - run_with_io(process, |io| { + run_with_io(&process, |io| { let file = io.files.file(fd)?; let channel = file.as_message_channel()?; @@ -432,7 +439,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let buf = arg_buffer_mut(args[2] as usize, args[3] as usize)?; let from = arg_user_mut::>(args[4] as usize)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = io.files.file(fd)?; let channel = file.as_message_channel()?; @@ -467,7 +474,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let size = args[0] as usize; let size = size.page_align_up::(); - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = File::new_shared_memory(size)?; let fd = io.files.place_file(file, true)?; Ok(fd.0 as usize) @@ -478,7 +485,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let size = arg_user_ref::(args[1] as usize)?; let output = arg_user_mut::>(args[2] as usize)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let (master, slave) = File::new_pseudo_terminal(*options, *size)?; let master_fd = io.files.place_file(master, true)?; let slave_fd = io.files.place_file(slave, true)?; @@ -492,7 +499,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let source_fd = RawFd::from(args[0] as u32); let target_fd = RawFd::from(args[1] as u32); - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = io.files.file(source_fd)?.clone(); let fd = if target_fd != RawFd::NONE { @@ -508,7 +515,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::CreateTimer => { let repeat = args[0] != 0; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = File::new_timer(repeat); let fd = io.files.place_file(file, true)?; Ok(fd.0 as _) @@ -518,11 +525,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::SpawnProcess => { let options = arg_user_ref::(args[0] as usize)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { // Setup a new process from the file let (child_process, child_main) = proc::exec::load( - io.ioctx(), - Some(Arc::downgrade(process)), + io.ioctx_mut(), + Some(Arc::downgrade(&process)), options.program, options.arguments, options.environment, @@ -530,11 +537,11 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let pid: u32 = child_process.id().into(); // Inherit group and session from the creator - child_process.inherit(process)?; + child_process.inherit(&process)?; // Inherit root from the creator // let child_ioctx = IoContext::new(io.ioctx().root().clone()); - let child_ioctx = IoContext::inherit(io.ioctx()); + let child_ioctx = IoContext::inherit(io.ioctx_mut()); let mut child_io = child_process.io.lock(); child_io.set_ioctx(child_ioctx); @@ -601,20 +608,20 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } SyscallFunction::ExitThread => { let code = ExitCode::from(args[0] as i32); - thread.exit(code) + thread.exit::(code) } SyscallFunction::ExitProcess => { // A bit different from thread exit: wait for other threads to finish and exit only // after that let code = ExitCode::from(args[0] as i32); - thread.exit_process(code) + thread.exit_process::(code) } SyscallFunction::SendSignal => { let pid = ProcessId::from(args[0] as u32); let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - let target = Process::get(pid).ok_or(Error::DoesNotExist)?; + let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; target.raise_signal(signal); Ok(0) @@ -640,9 +647,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::GetProcessInfo => { let element = arg_user_mut::(args[0] as _)?; - run_with_io(process, |mut io| match element { + run_with_io(&process, |mut io| match element { ProcessInfoElement::Umask(mask) => { - *mask = io.ioctx().umask(); + *mask = io.ioctx_mut().umask(); Ok(0) } }) @@ -650,9 +657,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result SyscallFunction::SetProcessInfo => { let element = arg_user_ref::(args[0] as _)?; - run_with_io(process, |mut io| match element { + run_with_io(&process, |mut io| match element { &ProcessInfoElement::Umask(mask) => { - io.ioctx().set_umask(mask); + io.ioctx_mut().set_umask(mask); Ok(0) } }) @@ -672,7 +679,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result if let Some(ctty) = session_terminal { // Drop all FDs referring to the old session terminal - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { io.files.retain(|_, f| { f.node() .map(|node| !Arc::ptr_eq(node, &ctty)) @@ -692,9 +699,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result debugln!("WaitProcess #{}", pid); let status = arg_user_mut::(args[1] as _)?; - let target = Process::get(pid).ok_or(Error::DoesNotExist)?; + let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; - *status = block!(target.exit.wait_copy().await)?; + *status = block!(target.wait_for_exit().await)?; Ok(0) } @@ -704,7 +711,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let listen = arg_user_ref::(args[0] as usize)?; let ty = SocketType::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = match ty { SocketType::UdpPacket => { File::from_packet_socket(UdpSocket::bind((*listen).into())?) @@ -728,7 +735,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result // TODO connect on existing sockets? assert_eq!(socket_fd, RawFd::NONE); - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let (local, file) = match ty { SocketType::TcpStream => { let (local, socket) = TcpSocket::connect((*remote).into())?; @@ -746,7 +753,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let buffer = arg_buffer_ref(args[1] as usize, args[2] as usize)?; let recepient = arg_user_ref::>(args[3] as usize)?; - run_with_io(process, |io| { + run_with_io(&process, |io| { let file = io.files.file(socket_fd)?; file.send_to(buffer, recepient.map(Into::into)) @@ -758,7 +765,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let remote_result = arg_user_mut::>(args[3] as usize)?; - run_with_io(process, |io| { + run_with_io(&process, |io| { let file = io.files.file(socket_fd)?; let mut remote = MaybeUninit::uninit(); let len = file.receive_from(buffer, &mut remote)?; @@ -770,7 +777,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let socket_fd = RawFd::from(args[0] as u32); let option = arg_user_mut::(args[1] as usize)?; - run_with_io(process, |io| { + run_with_io(&process, |io| { let file = io.files.file(socket_fd)?; let socket = file.as_socket()?; socket.get_option(option)?; @@ -781,7 +788,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let socket_fd = RawFd::from(args[0] as u32); let option = arg_user_ref::(args[1] as usize)?; - run_with_io(process, |io| { + run_with_io(&process, |io| { let file = io.files.file(socket_fd)?; let socket = file.as_socket()?; socket.set_option(option)?; @@ -793,7 +800,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let remote_result = arg_user_mut::>(args[1] as usize)?; - run_with_io(process, |mut io| { + run_with_io(&process, |mut io| { let file = io.files.file(socket_fd)?; let mut remote = MaybeUninit::uninit(); let accepted_file = file.accept(&mut remote)?; diff --git a/src/task/context.rs b/src/task/context.rs deleted file mode 100644 index 382c915d..00000000 --- a/src/task/context.rs +++ /dev/null @@ -1 +0,0 @@ -//! Platform-specific task context manipulation interfaces diff --git a/src/task/mod.rs b/src/task/mod.rs index f97bd7c9..6e28c755 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -2,122 +2,25 @@ #![allow(dead_code)] -use core::ops::{Deref, DerefMut}; - use abi::error::Error; use alloc::{string::String, vec::Vec}; -use device_api::interrupt::IpiMessage; -use kernel_arch::{task::TaskContext, Architecture, ArchitectureImpl, CpuImpl, LocalCpuImpl}; +use kernel_arch::task::{Scheduler, TaskContext}; use libk::{ + arch::Cpu, runtime, thread::{TaskContextImpl, Termination}, }; -use libk_util::sync::{IrqGuard, SpinFence}; - -use crate::{ - arch::{Platform, PlatformImpl}, - task::{sched::CpuQueue, thread::Thread}, +use libk_thread::{ + sched::{init_queues, CpuQueue}, + thread::Thread, }; +use libk_util::sync::SpinFence; + +use crate::arch::{Platform, PlatformImpl}; + +use self::process::ProcessManagerImpl; -pub mod context; pub mod process; -pub mod sched; -pub mod sync; -pub mod thread; - -/// Kernel wrapper for local CPU info structure. See [kernel_arch::LocalCpuImpl]. -pub struct LocalCpu<'a>(LocalCpuImpl<'a, CpuQueue>); -/// Kernel wrapper for per-CPU info structure. See [kernel_arch::LocalCpuImpl]. -pub struct Cpu(CpuImpl); - -impl Cpu { - /// Initializes the local CPU - #[cfg(target_arch = "x86_64")] - pub unsafe fn init_local(id: u32, inner: ::PerCpuData) { - use alloc::boxed::Box; - - let cpu = Box::leak(Box::new(CpuImpl::::new(id, inner))); - cpu.this = cpu.deref_mut(); - - cpu.set_local(); - } - - /// Initializes the local CPU - #[cfg(target_arch = "aarch64")] - pub unsafe fn init_local(inner: ::PerCpuData) { - use aarch64_cpu::registers::MPIDR_EL1; - use alloc::boxed::Box; - use tock_registers::interfaces::Readable; - - let id = (MPIDR_EL1.get() & 0xFF) as u32; - let cpu = Box::leak(Box::new(CpuImpl::::new(id, inner))); - - cpu.set_local(); - } - - /// Returns local CPU reference - #[inline] - pub fn local<'a>() -> LocalCpu<'a> { - LocalCpu(CpuImpl::local()) - } - - /// Returns local CPU reference or None if it hasn't yet been initialized - #[inline] - pub fn try_local<'a>() -> Option> { - CpuImpl::try_local().map(LocalCpu) - } - - /// Pushes a message to the IPI queue - #[inline] - pub fn push_ipi_queue(cpu_id: u32, msg: IpiMessage) { - CpuImpl::::push_ipi_queue(cpu_id, msg) - } - - /// Initialize the IPI queues for all present CPUs - #[inline] - pub fn init_ipi_queues(cpu_count: usize) { - CpuImpl::::init_ipi_queues(cpu_count) - } -} - -impl Deref for Cpu { - type Target = CpuImpl; - - #[inline] - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for Cpu { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl<'a> LocalCpu<'a> { - /// Converts the local CPU handle into its IRQ guard - pub fn into_guard(self) -> IrqGuard { - self.0.into_guard() - } -} - -impl<'a> Deref for LocalCpu<'a> { - type Target = CpuImpl; - - #[inline] - fn deref(&self) -> &Self::Target { - self.0.deref() - } -} - -impl<'a> DerefMut for LocalCpu<'a> { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - self.0.deref_mut() - } -} /// Creates a new kernel-space process to execute a closure and queues it to some CPU pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send + 'static>( @@ -128,7 +31,7 @@ pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send name, TaskContextImpl::kernel_closure(move || { let result = f(); - Thread::current().exit(result.into_exit_code()); + Thread::current().exit::(result.into_exit_code()); })?, ); thread.enqueue(); @@ -141,7 +44,7 @@ pub fn init() -> Result<(), Error> { let cpu_count = PlatformImpl::cpu_count(); // Create a queue for each CPU - sched::init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); + init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); // Spawn async workers (0..cpu_count).for_each(|index| { diff --git a/src/task/process.rs b/src/task/process.rs index 0a83195e..e9015583 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,541 +1,45 @@ //! Process data structures -use core::{ - fmt, - mem::size_of, - sync::atomic::{AtomicU64, Ordering}, +use abi::process::Signal; +use alloc::{collections::BTreeMap, sync::Arc}; +use libk_thread::process::{Process, ProcessManager}; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; + +use crate::proc::io::ProcessIoImpl; + +pub use libk_thread::{ + process::ProcessImage, + types::{ProcessId, ProcessTlsInfo, ProcessTlsLayout}, }; -use abi::{ - error::{Error, SyscallResult}, - process::{ExitCode, Signal, ThreadSpawnOptions}, -}; -use alloc::{ - collections::BTreeMap, - string::String, - sync::{Arc, Weak}, - vec::Vec, -}; -use kernel_arch::task::TaskContext; -use libk::thread::{ForkFrame, TaskContextImpl}; -use libk_mm::process::ProcessAddressSpace; -use libk_util::{ - event::OneTimeEvent, - sync::{ - spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard}, - IrqSafeSpinlock, - }, -}; -use vfs::{IoContext, NodeRef}; +/// Process manager implementation +pub struct ProcessManagerImpl; -use crate::{ - proc::{self, io::ProcessIo}, - task::sched::CpuQueue, -}; +/// Alias type for [libk_thread::process::ProcessImpl] +pub type ProcessImpl = libk_thread::process::ProcessImpl; -use super::{ - sync::UserspaceMutex, - thread::{Thread, ThreadId}, -}; +static PROCESSES: IrqSafeRwLock>> = + IrqSafeRwLock::new(BTreeMap::new()); -/// Unique number assigned to each [Process] -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct ProcessId(u64); +impl ProcessManager for ProcessManagerImpl { + type Process = ProcessImpl; -// TLS layout (x86-64): -// | mem_size | uthread_size | -// | Data .......| self, ??? | -// -// TLS layout (aarch64): -// | uthread_size (0x10?) | mem_size | -// | ??? | Data .....| + fn register_process(process: Arc) { + PROCESSES.write().insert(process.id(), process); + } -/// Describes Thread-Local Storage of a process -#[derive(Clone, Debug)] -pub struct ProcessTlsInfo { - /// Location of the TLS master copy within the process's memory - pub master_copy_base: usize, - /// Layout of the TLS - pub layout: ProcessTlsLayout, -} + fn get(id: ProcessId) -> Option> { + PROCESSES.read().get(&id).cloned() + } -/// Describes TLS layout for a program image -#[derive(Clone, Debug)] -pub struct ProcessTlsLayout { - /// Data offset from the TLS base - pub data_offset: usize, - /// struct uthread offset from the TLS base - pub uthread_offset: usize, - /// Pointer offset from the TLS base. The pointer is passed to the userspace - pub ptr_offset: usize, - - /// Data size of the TLS segment - pub data_size: usize, - /// Memory size of the TLS segment (mem_size >= data_size) - pub mem_size: usize, - - /// Overall allocation size of the TLS data - pub full_size: usize, -} - -#[cfg(target_arch = "aarch64")] -impl ProcessTlsLayout { - /// Constructs a new thread-local storage layout info struct - pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { - debug_assert!(align.is_power_of_two()); - let tls_block0_offset = (size_of::() * 2 + align - 1) & !(align - 1); - - let full_size = (tls_block0_offset + mem_size + align - 1) & !(align - 1); - - Self { - data_offset: tls_block0_offset, - uthread_offset: 0, - ptr_offset: 0, - - data_size, - mem_size, - full_size, + fn for_each)>(f: F) { + for (k, v) in PROCESSES.read().iter() { + f(*k, v); } } } -#[cfg(target_arch = "x86_64")] -impl ProcessTlsLayout { - /// Constructs a new thread-local storage layout info struct - pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { - // The static TLS blocks are placed below TP - // TP points to the TCB - debug_assert!(align.is_power_of_two()); - let back_size = (mem_size + align - 1) & !(align - 1); - // Self-pointer - let forward_size = size_of::(); - - let full_size = back_size + forward_size; - - Self { - data_offset: 0, - uthread_offset: back_size, - ptr_offset: back_size, - - data_size, - mem_size, - full_size, - } - } -} - -/// Describes information about a program's image in memory -#[derive(Clone)] -pub struct ProcessImage { - /// Entry point address - pub entry: usize, - /// Thread-local storage information - pub tls: Option, -} - -struct ProcessInner { - session_id: ProcessId, - group_id: ProcessId, - - session_terminal: Option, - threads: Vec>, - mutexes: BTreeMap>, - - space: Option>, - image: Option, -} - -/// Describes a process within the system -pub struct Process { - name: String, - id: ProcessId, - parent: Option>, - - inner: IrqSafeRwLock, - - pub(crate) exit: OneTimeEvent, - - /// Process I/O information - pub io: IrqSafeSpinlock, -} - -static PROCESSES: IrqSafeSpinlock>> = - IrqSafeSpinlock::new(BTreeMap::new()); - -impl Process { - /// Creates a new process with given main thread - pub fn new_with_main>( - name: S, - parent: Option>, - space: Arc, - context: TaskContextImpl, - image: Option, - ) -> (Arc, Arc) { - let name = name.into(); - let id = ProcessId::next(); - - let process = Arc::new(Self { - name, - id, - parent, - - inner: IrqSafeRwLock::new(ProcessInner { - session_id: id, - group_id: id, - session_terminal: None, - threads: Vec::new(), - mutexes: BTreeMap::new(), - - image, - space: Some(space.clone()), - }), - - exit: OneTimeEvent::new(), - io: IrqSafeSpinlock::new(ProcessIo::new()), - }); - - // Create "main" thread - let thread = Thread::new_uthread(process.clone(), space, context); - process.inner.write().threads.push(thread.clone()); - - PROCESSES.lock().insert(id, process.clone()); - - (process, thread) - } - - /// Spawns a new child thread within the process - pub fn spawn_thread(self: &Arc, options: &ThreadSpawnOptions) -> Result { - debugln!( - "Spawn thread in {} with options: {:#x?}", - self.id(), - options - ); - let mut inner = self.inner.write(); - - let space = inner.space.clone().unwrap(); - - let tls_address = if let Some(image) = inner.image.as_ref() { - proc::elf::clone_tls(&space, image)? - } else { - 0 - }; - - let context = TaskContextImpl::user( - options.entry as _, - options.argument as _, - space.as_address_with_asid(), - options.stack_top, - tls_address, - )?; - let thread = Thread::new_uthread(self.clone(), space, context); - let id = thread.id; - - inner.threads.push(thread.clone()); - - thread.enqueue(); - - Ok(id) - } - - unsafe fn fork_inner>( - self: &Arc, - frame: &F, - ) -> Result { - let src_inner = self.inner.read(); - let new_space = src_inner.space.as_ref().unwrap().fork()?; - let new_context = frame.fork(new_space.as_address_with_asid())?; - - let (new_process, new_main) = Self::new_with_main( - self.name(), - Some(Arc::downgrade(self)), - Arc::new(new_space), - new_context, - src_inner.image.clone(), - ); - - { - let mut src_io = self.io.lock(); - - let new_ioctx = IoContext::inherit(src_io.ioctx()); - let mut new_io = new_process.io.lock(); - new_io.set_ioctx(new_ioctx); - - for (old_fd, old_file) in src_io.files.iter() { - let new_file = old_file.send()?; - new_io.files.set_file(*old_fd, new_file)?; - } - } - - new_process.inherit(self)?; - - infoln!("Process::fork -> {:?}", new_process.id()); - new_main.enqueue_to(CpuQueue::for_cpu(1)); - - Ok(new_process.id()) - } - - /// Performs a "fork" operation on the process, creating an identical copy of it, cloning - /// the register state from provided [ForkFrame]. - /// - /// # Safety - /// - /// Unsafe: frame must be a valid frame to be forked, the function is not yet stable and does - /// not yet properly fork all the necessary context details. - pub unsafe fn raw_fork>(frame: &mut F) { - let src_thread = Thread::current(); - let src_process = src_thread.process(); - - let rax = src_process - .fork_inner(frame) - .map(|ProcessId(p)| p as u32) - .into_syscall_result(); - - frame.set_return_value(rax as _); - } - - /// Replaces the process address space with a new one, loaded from the specified program - pub fn exec(&self, program: &str, argv: Vec, envp: Vec) -> Result { - if self.inner.read().threads.len() != 1 { - todo!(); - } - - let (context, image) = proc::exec::load_into(self, program, argv, envp)?; - let mut inner = self.inner.write(); - let main = &inner.threads[0]; - - let old_context = unsafe { main.replace_context(context) }; - let new_context = main.context.as_ptr(); - inner.image = Some(image); - - drop(inner); - - // TODO old context is leaked - unsafe { (*new_context).switch(&old_context) } - unreachable!() - } - - /// Returns the address space of the process - pub fn space(&self) -> Arc { - self.inner.read().space.clone().unwrap() - } - - /// Returns the [ProcessId] of this process - pub fn id(&self) -> ProcessId { - self.id - } - - /// Returns the process group ID of the process - pub fn group_id(&self) -> ProcessId { - self.inner.read().group_id - } - - /// Returns the process session ID of the process - pub fn session_id(&self) -> ProcessId { - self.inner.read().session_id - } - - /// Returns the process name - pub fn name(&self) -> &str { - self.name.as_ref() - } - - /// Changes the process's group ID - pub fn set_group_id(&self, id: ProcessId) { - self.inner.write().group_id = id; - } - - /// Changes the process's session ID - pub fn set_session_id(&self, id: ProcessId) { - self.inner.write().session_id = id; - } - - // Resources - - /// Returns the current session terminal of the process, if set - pub fn session_terminal(&self) -> Option { - self.inner.read().session_terminal.clone() - } - - /// Changes the current session terminal of the process - pub fn set_session_terminal(&self, node: NodeRef) { - self.inner.write().session_terminal.replace(node); - } - - /// Resets the current session terminal of the process - pub fn clear_session_terminal(&self) -> Option { - self.inner.write().session_terminal.take() - } - - /// Inherits the process information from the `parent` - pub fn inherit(&self, parent: &Process) -> Result<(), Error> { - let mut our_inner = self.inner.write(); - let their_inner = parent.inner.read(); - - our_inner.session_id = their_inner.session_id; - our_inner.group_id = their_inner.group_id; - our_inner.session_terminal = their_inner.session_terminal.clone(); - - Ok(()) - } - - // State - - /// Returns the [ExitCode] of the process, if it has exited - pub fn get_exit_status(&self) -> Option { - self.exit.try_read_copy() - } - - /// Returns `true` if the process has exited - pub fn has_exited(&self) -> bool { - self.exit.is_signalled() - } - - // /// Suspends the task until the process exits - // pub fn wait_for_exit(process: Arc) -> impl Future { - // // struct ProcessExitFuture { - // // process: Arc, - // // } - - // // impl Future for ProcessExitFuture { - // // type Output = ExitCode; - - // // fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // // let process = &self.process; - - // // process.exit_waker.register(cx.waker()); - - // // if let Some(exit_status) = process.get_exit_status() { - // // process.exit_waker.remove(cx.waker()); - // // Poll::Ready(exit_status) - // // } else { - // // Poll::Pending - // // } - // // } - // // } - - // // ProcessExitFuture { process } - // } - - /// Cleans up process resources - fn cleanup(&self, mut inner: IrqSafeRwLockWriteGuard) { - self.io.lock().handle_exit(); - inner.space = None; - } - - /// Handles exit of a single child thread - pub fn handle_thread_exit(&self, thread: ThreadId, code: ExitCode) { - debugln!("Thread {} of process {}: {:?}", thread, self.id, code); - let mut inner = self.inner.write(); - - // TODO make this cleaner - let old_len = inner.threads.len(); - inner.threads.retain(|t| t.id != thread); - assert_ne!(inner.threads.len(), old_len); - - let last_thread = inner.threads.is_empty(); - - if last_thread { - debugln!("Last thread of {} exited", self.id); - self.cleanup(inner); - self.exit.signal(code); - } - } - - /// Raises a signal for the specified process - pub fn raise_signal(self: &Arc, signal: Signal) { - let thread = self.inner.read().threads[0].clone(); - thread.raise_signal(signal); - } - - /// Raises a signal for the specified process group - pub fn signal_group(group_id: ProcessId, signal: Signal) { - let processes = PROCESSES.lock(); - for (_, proc) in processes.iter() { - let inner = proc.inner.read(); - if !proc.has_exited() && inner.group_id == group_id { - debugln!( - "Deliver group ({}) signal to {}: {:?}", - group_id, - proc.id(), - signal - ); - drop(inner); - proc.raise_signal(signal); - } - } - } - - /// Returns a [UserspaceMutex] associated with the `address`. If one does not exist, will - /// create it. - pub fn get_or_insert_mutex(&self, address: usize) -> Arc { - let mut inner = self.inner.write(); - inner - .mutexes - .entry(address) - .or_insert_with(|| Arc::new(UserspaceMutex::new(address))) - .clone() - } - - // Process list - - /// Returns the process with given [ProcessId], if it exists - pub fn get(id: ProcessId) -> Option> { - PROCESSES.lock().get(&id).cloned() - } - - /// Terminates all children of the process, `except` one - pub async fn terminate_others(&self, except: ThreadId) { - let mut inner = self.inner.write(); - - for thread in inner.threads.iter() { - if thread.id == except { - continue; - } - - infoln!("Terminate thread {}", thread.id); - thread.terminate().await; - } - - inner.threads.retain(|t| t.id == except); - } -} - -impl Drop for ProcessInner { - fn drop(&mut self) { - debugln!("ProcessInner dropped {:p}", self); - } -} - -impl fmt::Display for ProcessId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "", self.0) - } -} - -// XXX TODO Remove this -impl From for u32 { - fn from(value: ProcessId) -> Self { - value.0 as _ - } -} - -impl From for ProcessId { - fn from(value: u32) -> Self { - Self(value as _) - } -} - -impl ProcessId { - /// Generates a new [ProcessId] - pub fn next() -> Self { - static COUNTER: AtomicU64 = AtomicU64::new(1); - let id = COUNTER.fetch_add(1, Ordering::SeqCst); - Self(id) - } -} - #[no_mangle] fn __signal_process_group(group_id: u32, signal: Signal) { - Process::signal_group(ProcessId(group_id as _), signal) + ProcessImpl::signal_group(ProcessId(group_id as _), signal) } From 4f5d749298fe94f472cd80b0be8b2926b98f41a7 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 14 Feb 2024 11:04:29 +0200 Subject: [PATCH 190/211] libk: move binary loading to libk-thread --- arch/aarch64/src/lib.rs | 7 + arch/interface/src/lib.rs | 2 + arch/x86_64/src/lib.rs | 10 +- lib/vfs/Cargo.toml | 1 + lib/vfs/src/ioctx.rs | 26 +-- lib/vfs/src/traits.rs | 38 +---- libk-thread/Cargo.toml | 6 + libk-thread/src/binary/elf.rs | 306 ++++++++++++++++++++++++++++++++++ libk-thread/src/binary/mod.rs | 278 ++++++++++++++++++++++++++++++ libk-thread/src/lib.rs | 11 +- libk-util/src/io.rs | 62 +++++++ libk-util/src/lib.rs | 1 + libk/src/api.rs | 2 - libk/src/lib.rs | 4 +- src/arch/aarch64/boot/mod.rs | 6 +- src/arch/aarch64/gic/mod.rs | 4 +- src/arch/aarch64/mod.rs | 4 - src/arch/aarch64/smp.rs | 6 +- src/arch/mod.rs | 8 - src/arch/x86_64/acpi.rs | 3 +- src/arch/x86_64/apic/local.rs | 6 +- src/arch/x86_64/boot/mod.rs | 6 +- src/arch/x86_64/mod.rs | 5 - src/arch/x86_64/smp.rs | 16 +- src/init.rs | 2 +- src/main.rs | 14 +- src/panic.rs | 4 +- src/proc/elf.rs | 305 --------------------------------- src/proc/exec.rs | 244 --------------------------- src/proc/mod.rs | 21 ++- src/syscall/mod.rs | 2 +- src/task/mod.rs | 9 +- 32 files changed, 755 insertions(+), 664 deletions(-) create mode 100644 libk-thread/src/binary/elf.rs create mode 100644 libk-thread/src/binary/mod.rs create mode 100644 libk-util/src/io.rs diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index 768c92d8..e4d218b8 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -3,6 +3,8 @@ extern crate alloc; +use core::sync::atomic::{AtomicUsize, Ordering}; + use aarch64_cpu::registers::{DAIF, MPIDR_EL1, TPIDR_EL1}; use alloc::{boxed::Box, vec::Vec}; use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; @@ -29,6 +31,7 @@ pub struct PerCpuData { } static IPI_QUEUES: OneTimeInit>> = OneTimeInit::new(); +pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); #[naked] extern "C" fn idle_task(_: usize) -> ! { @@ -91,6 +94,10 @@ impl Architecture for ArchitectureImpl { idle_task } + fn cpu_count() -> usize { + CPU_COUNT.load(Ordering::Acquire) + } + fn local_interrupt_controller() -> &'static dyn LocalInterruptController { let local = Self::local_cpu_data().unwrap(); *local.gic.get() diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs index c324e2b5..2498437f 100644 --- a/arch/interface/src/lib.rs +++ b/arch/interface/src/lib.rs @@ -28,6 +28,8 @@ pub trait Architecture: Sized { fn idle_task() -> extern "C" fn(usize) -> !; + fn cpu_count() -> usize; + // Interrupt management fn interrupt_mask() -> bool; unsafe fn set_interrupt_mask(mask: bool) -> bool; diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index 16c1bd86..4f3a6289 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -3,7 +3,10 @@ extern crate alloc; -use core::ops::DerefMut; +use core::{ + ops::DerefMut, + sync::atomic::{AtomicUsize, Ordering}, +}; use alloc::vec::Vec; use device_api::interrupt::{LocalInterruptController, MessageInterruptController}; @@ -59,6 +62,7 @@ impl PerCpuData { } static IPI_QUEUES: OneTimeInit>> = OneTimeInit::new(); +pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); #[naked] extern "C" fn idle_task(_: usize) -> ! { @@ -116,6 +120,10 @@ impl Architecture for ArchitectureImpl { idle_task } + fn cpu_count() -> usize { + CPU_COUNT.load(Ordering::Acquire) + } + fn interrupt_mask() -> bool { let mut flags: u64; unsafe { diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index b7e91861..bef0188b 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -11,6 +11,7 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", fea libk = { path = "../../libk" } libk-mm = { path = "../../libk-mm" } libk-util = { path = "../../libk-util" } +libk-thread = { path = "../../libk-thread" } ygg_driver_block = { path = "../../driver/block/core" } diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index 65c76475..56893467 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -1,4 +1,5 @@ -use alloc::borrow::ToOwned; +use alloc::{borrow::ToOwned, sync::Arc}; +use libk_thread::binary::ProgramLoadSource; use yggdrasil_abi::{ error::Error, io::{FileMode, FileType, GroupId, OpenOptions, UserId}, @@ -7,7 +8,7 @@ use yggdrasil_abi::{ use crate::{ node::{AccessToken, CreateInfo, SymlinkData}, - FileRef, NodeRef, + File, FileRef, NodeRef, }; /// Describes a general filesystem access @@ -31,6 +32,16 @@ pub struct IoContext { root: NodeRef, } +impl ProgramLoadSource for IoContext { + type File = File; + + fn open_executable>(&mut self, path: P) -> Result, Error> { + let node = self.find(None, path, true, true)?; + let access = self.check_access(Action::Read, &node)?; + node.open(OpenOptions::READ, access) + } +} + impl IoContext { /// Constructs a new [IoContext] with given root node (which also becomes the cwd). By default, /// the root user/group is used. Default umask is 0o022. @@ -218,17 +229,6 @@ impl IoContext { node.open(opts, access) } - /// Opens a file at given path for execution - pub fn open_exec>( - &mut self, - at: Option, - path: P, - ) -> Result { - let node = self.find(at, path, true, true)?; - let access = self.check_access(Action::Read, &node)?; - node.open(OpenOptions::READ, access) - } - /// Creates a directory a given path pub fn create_directory>( &mut self, diff --git a/lib/vfs/src/traits.rs b/lib/vfs/src/traits.rs index be192eed..9b9e6850 100644 --- a/lib/vfs/src/traits.rs +++ b/lib/vfs/src/traits.rs @@ -1,42 +1,8 @@ use core::task::{Context, Poll}; -use yggdrasil_abi::{error::Error, io::SeekFrom}; +use yggdrasil_abi::error::Error; -/// Immutable read interface for VFS objects -pub trait Read { - /// Reads bytes into the given buffer - fn read(&self, buf: &mut [u8]) -> Result; - - /// Reads exactly `buf.len()` bytes into the given buffer, fails if such amount is not - /// available - fn read_exact(&self, buf: &mut [u8]) -> Result<(), Error> { - match self.read(buf) { - Ok(count) if count == buf.len() => Ok(()), - Ok(_) => Err(Error::InvalidFile), - Err(err) => Err(err), - } - } -} - -/// Immutable write interface for VFS objects -pub trait Write { - /// Writes bytes from the given buffer - fn write(&self, buf: &[u8]) -> Result; -} - -/// Immutable file positioning interface for VFS objects -pub trait Seek { - /// Changes position inside the file to a requested one. Fails if the file does not support - /// seeking. - fn seek(&self, from: SeekFrom) -> Result; - - /// Returns the position within the file, determined as an offset in bytes from the beginning - /// of the file. Fails if the file does not support seeking or the "offset" is not defined for - /// such type of nodes. - fn tell(&self) -> Result { - self.seek(SeekFrom::Current(0)) - } -} +pub use libk_util::io::{Read, Seek, Write}; /// Interface for polling files for events pub trait FileReadiness: Sync { diff --git a/libk-thread/Cargo.toml b/libk-thread/Cargo.toml index ae55d8c8..d001907f 100644 --- a/libk-thread/Cargo.toml +++ b/libk-thread/Cargo.toml @@ -16,3 +16,9 @@ log = "0.4.20" atomic_enum = "0.2.0" crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } + +[dependencies.elf] +version = "0.7.2" +git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" +default-features = false +features = ["no_std_stream"] diff --git a/libk-thread/src/binary/elf.rs b/libk-thread/src/binary/elf.rs new file mode 100644 index 00000000..0963e090 --- /dev/null +++ b/libk-thread/src/binary/elf.rs @@ -0,0 +1,306 @@ +//! ELF binary format support +use core::ops::DerefMut; + +use alloc::sync::Arc; +use elf::{ + abi::{PF_W, PF_X, PT_LOAD, PT_TLS}, + endian::AnyEndian, + segment::ProgramHeader, + ElfStream, ParseError, +}; +use libk_mm::{ + pointer::{PhysicalRef, PhysicalRefMut}, + process::{ProcessAddressSpace, VirtualRangeBacking}, + table::MapAttributes, +}; +use libk_util::io::{Read, Seek}; +use yggdrasil_abi::{error::Error, io::SeekFrom}; + +use crate::{ + process::ProcessImage, + types::{ProcessTlsInfo, ProcessTlsLayout}, +}; + +#[derive(Clone)] +struct FileReader { + file: F, +} + +impl elf::io_traits::InputStream for FileReader { + fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), elf::io_traits::StreamError> { + self.file.read_exact(buf).map_err(conv_stream_error) + } + + fn seek(&mut self, pos: elf::io_traits::SeekFrom) -> Result { + self.file + .seek(conv_seek_from(pos)) + .map_err(conv_stream_error) + } +} + +#[inline] +fn conv_stream_error(_v: Error) -> elf::io_traits::StreamError { + elf::io_traits::StreamError { + message: "Elf read error", + } +} + +#[inline] +fn conv_seek_from(v: elf::io_traits::SeekFrom) -> SeekFrom { + match v { + elf::io_traits::SeekFrom::End(off) => SeekFrom::End(off), + elf::io_traits::SeekFrom::Start(off) => SeekFrom::Start(off), + _ => todo!(), + } +} + +#[inline] +fn from_parse_error(v: ParseError) -> Error { + log::warn!("ELF loading error: {:?}", v); + Error::InvalidFile +} + +/// Creates a new copy of the TLS from given master image +pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result { + let Some(tls) = image.tls.as_ref() else { + // No TLS + return Ok(0); + }; + + assert_ne!(tls.master_copy_base, 0); + assert_ne!(tls.layout.mem_size, 0); + + let address = space.allocate( + None, + 0x1000, + VirtualRangeBacking::anonymous(), + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, + )?; + + // debugln!( + // "Clone TLS from {:#x} (data={:#x}) to {:#x} (data={:#x})", + // tls.master_copy_base, + // tls.master_copy_base + tls.layout.data_offset, + // address, + // address + tls.layout.data_offset + // ); + // debugln!("tls ptr = {:#x}", address + tls.layout.ptr_offset); + + let src_phys = space.translate(tls.master_copy_base)?; + let dst_phys = space.translate(address)?; + + // Copy data + unsafe { + let src = + PhysicalRef::::map_slice(src_phys.add(tls.layout.data_offset), tls.layout.mem_size); + let mut dst = + PhysicalRefMut::map_slice(dst_phys.add(tls.layout.data_offset), tls.layout.mem_size); + + dst.copy_from_slice(&src); + } + + // Setup self-pointer + unsafe { + let mut dst = PhysicalRefMut::::map(dst_phys.add(tls.layout.ptr_offset)); + + *dst = address + tls.layout.ptr_offset; + } + + Ok(address + tls.layout.ptr_offset) +} + +fn load_bytes( + space: &ProcessAddressSpace, + addr: usize, + mut src: F, + len: usize, +) -> Result<(), Error> +where + F: FnMut(usize, PhysicalRefMut<'_, [u8]>) -> Result<(), Error>, +{ + // TODO check for crazy addresses here + + let dst_page_off = addr & 0xFFF; + let dst_page_aligned = addr & !0xFFF; + let mut off = 0usize; + let mut rem = len; + + while rem != 0 { + let page_idx = (dst_page_off + off) / 0x1000; + let page_off = (dst_page_off + off) % 0x1000; + let count = core::cmp::min(rem, 0x1000 - page_off); + + let virt_page = dst_page_aligned + page_idx * 0x1000; + assert_eq!(virt_page & 0xFFF, 0); + + let phys_page = space.translate(virt_page)?; + let dst_slice = unsafe { PhysicalRefMut::map_slice(phys_page.add(page_off), count) }; + + src(off, dst_slice)?; + + rem -= count; + off += count; + } + + Ok(()) +} + +fn load_segment( + space: &ProcessAddressSpace, + phdr: &ProgramHeader, + file: &FileReader, +) -> Result<(), Error> { + if phdr.p_memsz == 0 { + return Ok(()); + } + + let attrs = match (phdr.p_flags & PF_W, phdr.p_flags & PF_X) { + (0, 0) => MapAttributes::USER_READ, + (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + (0, _) => MapAttributes::USER_READ, + (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, + } | MapAttributes::NON_GLOBAL; + + // Map the range + let aligned_start = (phdr.p_vaddr as usize) & !0xFFF; + let aligned_end = ((phdr.p_vaddr + phdr.p_memsz) as usize + 0xFFF) & !0xFFF; + + space.map( + aligned_start, + aligned_end - aligned_start, + VirtualRangeBacking::anonymous(), + attrs, + )?; + + if phdr.p_filesz > 0 { + load_bytes( + space, + phdr.p_vaddr as usize, + |off, mut dst| { + file.file + .seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + file.file.read_exact(dst.deref_mut()) + }, + phdr.p_filesz as usize, + )?; + } + + if phdr.p_memsz > phdr.p_filesz { + let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; + let len = (phdr.p_memsz - phdr.p_filesz) as usize; + + load_bytes( + space, + addr, + |_, mut dst| { + dst.fill(0); + Ok(()) + }, + len, + )?; + } + + Ok(()) +} + +fn tls_segment( + space: &ProcessAddressSpace, + phdr: &ProgramHeader, + file: &FileReader, +) -> Result { + assert_ne!(phdr.p_memsz, 0); + + if !phdr.p_align.is_power_of_two() { + return Err(Error::InvalidArgument); + } + + let layout = ProcessTlsLayout::new(phdr.p_align as _, phdr.p_filesz as _, phdr.p_memsz as _); + let data_offset = layout.data_offset; + let data_size = layout.data_size; + let mem_size = layout.mem_size; + let aligned_size = (layout.full_size + 0xFFF) & !0xFFF; + assert!(aligned_size <= 0x1000); + + let base_address = space.allocate( + None, + aligned_size, + VirtualRangeBacking::anonymous(), + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, + )?; + + // debugln!( + // "Allocated TLS master copy @ {:#x}, tls={:#x}, tls_data={:#x}", + // base_address, + // base_address + layout.ptr_offset, + // base_address + layout.data_offset + // ); + + let tls = ProcessTlsInfo { + master_copy_base: base_address, + layout, + }; + + if data_size > 0 { + load_bytes( + space, + base_address + data_offset, + |off, mut dst| { + file.file + .seek(SeekFrom::Start(phdr.p_offset + off as u64))?; + file.file.read_exact(dst.deref_mut()) + }, + data_size, + )?; + } + + if mem_size > data_size { + load_bytes( + space, + base_address + data_offset + data_size, + |_, mut dst| { + dst.fill(0); + Ok(()) + }, + mem_size - data_size, + )?; + } + + Ok(tls) +} + +/// Loads an ELF binary from `file` into the target address space +pub fn load_elf_from_file( + space: &ProcessAddressSpace, + file: Arc, +) -> Result { + let file = FileReader { file }; + let elf = ElfStream::::open_stream(file.clone()).map_err(from_parse_error)?; + let mut image_end = 0; + let mut tls = None; + + for phdr in elf.segments() { + match phdr.p_type { + PT_LOAD => { + if phdr.p_vaddr + phdr.p_memsz > image_end { + image_end = phdr.p_vaddr + phdr.p_memsz; + } + + load_segment(space, phdr, &file)?; + } + PT_TLS => { + assert!(tls.is_none()); + tls.replace(tls_segment(space, phdr, &file)?); + } + _ => (), + } + } + + // let image_end = (image_end as usize).page_align_up::(); + + // debugln!("Loaded image end: {:#x}", image_end); + + Ok(ProcessImage { + entry: elf.ehdr.e_entry as usize, + tls, + }) +} diff --git a/libk-thread/src/binary/mod.rs b/libk-thread/src/binary/mod.rs new file mode 100644 index 00000000..39a9d265 --- /dev/null +++ b/libk-thread/src/binary/mod.rs @@ -0,0 +1,278 @@ +use core::{alloc::Layout, ptr::NonNull}; + +use alloc::{ + borrow::ToOwned, + string::String, + sync::{Arc, Weak}, + vec::Vec, +}; +use kernel_arch::task::TaskContext; +use libk_mm::{ + pointer::PhysicalRefMut, + process::{ProcessAddressSpace, VirtualRangeBacking}, + table::MapAttributes, +}; +use libk_util::io::{Read, Seek}; +use yggdrasil_abi::{ + error::Error, + io::SeekFrom, + pass::{Place, Placer}, + path::Path, + process::ProgramArgumentInner, +}; + +use crate::{ + mem::ForeignPointer, + process::{ProcessImage, ProcessImpl, ProcessIo, ProcessManager}, + thread::Thread, + TaskContextImpl, +}; + +pub mod elf; + +pub trait ProgramLoadSource { + type File: Seek + Read; + + fn open_executable>(&mut self, path: P) -> Result, Error>; +} + +struct BufferPlacer<'a> { + buffer: &'a mut [u8], + virtual_offset: usize, + offset: usize, +} + +impl<'a> BufferPlacer<'a> { + pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self { + Self { + buffer, + virtual_offset, + offset: 0, + } + } + + unsafe fn alloc_layout( + &mut self, + layout: Layout, + ) -> Result<(NonNull, NonNull), Error> { + // TODO checks + let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1); + self.offset = aligned + layout.size(); + Ok(( + NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T), + NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T), + )) + } +} + +unsafe impl<'a> Placer for BufferPlacer<'a> { + fn place_ref(&mut self, r: &T) -> Result, Error> { + let layout = Layout::new::(); + unsafe { + let (kernel, user) = self.alloc_layout::(layout)?; + kernel.as_ptr().write(r.place(self)?); + Ok(user) + } + } + + fn place_slice(&mut self, r: &[T]) -> Result, Error> { + let layout = Layout::array::(r.len()).unwrap(); + unsafe { + let (kernel, user) = self.alloc_layout::(layout)?; + let kernel = NonNull::slice_from_raw_parts(kernel, r.len()); + let user = NonNull::slice_from_raw_parts(user, r.len()); + for (i, elem) in r.iter().enumerate() { + kernel + .get_unchecked_mut(i) + .as_ptr() + .write(elem.place(self)?); + } + Ok(user) + } + } +} + +// args, envs are passed as Vec to ensure kernel ownership +#[allow(clippy::ptr_arg)] +fn setup_program_env( + space: &ProcessAddressSpace, + virt: usize, + args: &Vec, + envs: &Vec, +) -> Result { + // TODO growing buffer + let phys_page = space.map_single( + virt, + VirtualRangeBacking::anonymous(), + MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, + )?; + let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; + let mut placer = BufferPlacer::new(virt, &mut buffer); + + let args = args.iter().map(String::as_ref).collect::>(); + let envs = envs.iter().map(String::as_ref).collect::>(); + + let in_kernel = ProgramArgumentInner { + args: &args, + env: &envs, + }; + let in_user = in_kernel.place_ref(&mut placer)?; + + Ok(in_user as *const _ as usize) +} + +fn setup_context( + space: &ProcessAddressSpace, + image: &ProcessImage, + args: &Vec, + envs: &Vec, +) -> Result { + const USER_STACK_PAGES: usize = 32; + + let virt_stack_base = 0x3000000; + // 0x1000 of guard page + let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; + + space.map( + virt_stack_base, + USER_STACK_PAGES * 0x1000, + VirtualRangeBacking::anonymous(), + MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, + )?; + + let arg = setup_program_env(space, virt_args_base, args, envs)?; + + let user_sp = + virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN; + + // Fill with some sentinel value to detect stack underflows + let ptr = user_sp as *mut u64; + + #[allow(clippy::reversed_empty_ranges)] + for i in 0..TaskContextImpl::USER_STACK_EXTRA_ALIGN / 8 { + unsafe { + ptr.add(i).write_foreign_volatile(space, 0xDEADC0DE); + } + } + + let tls_address = elf::clone_tls(space, image)?; + + TaskContext::user( + image.entry, + arg, + space.as_address_with_asid(), + user_sp, + tls_address, + ) +} + +fn setup_binary( + name: S, + parent: Option>>, + space: ProcessAddressSpace, + image: ProcessImage, + args: &Vec, + envs: &Vec, +) -> Result<(Arc>, Arc), Error> +where + S: Into, + PM: ProcessManager>, + IO: ProcessIo, +{ + let context = setup_context(&space, &image, args, envs)?; + let (process, main) = + ProcessImpl::new_with_main(name, parent, Arc::new(space), context, Some(image)); + Ok((process, main)) +} + +fn load_binary( + head: &[u8], + file: Arc, + space: &ProcessAddressSpace, +) -> Result { + if head.starts_with(b"\x7FELF") { + elf::load_elf_from_file(space, file) + } else { + Err(Error::UnrecognizedExecutable) + } +} + +fn xxx_load_program>( + space: &ProcessAddressSpace, + source: &mut PS, + path: P, + args: Vec, + envs: Vec, +) -> Result<(ProcessImage, Vec, Vec), Error> { + let mut head = [0; 256]; + let path = path.as_ref(); + let file = source.open_executable(path)?; + + file.seek(SeekFrom::Start(0))?; + let count = file.read(&mut head)?; + let head = &head[..count]; + + if let Some(shebang) = head.strip_prefix(b"#!") + && let Some((shebang, _)) = shebang.split_once(|&ch| ch == b'\n') + { + let shebang = core::str::from_utf8(shebang).map_err(|_| Error::InvalidFile)?; + let mut shebang_args = shebang.split(' ').map(|s| s.to_owned()).collect::>(); + if shebang_args.is_empty() || shebang_args.len() >= 8 { + return Err(Error::UnrecognizedExecutable); + } + shebang_args.extend_from_slice(&args); + + return xxx_load_program(space, source, shebang_args[0].clone(), shebang_args, envs); + } + + file.seek(SeekFrom::Start(0))?; + + let image = load_binary(head, file, space)?; + + Ok((image, args, envs)) +} + +/// Loads a program from given `path` +pub fn load( + source: &mut PS, + parent: Option>>, + path: P, + args: &[&str], + envs: &[&str], +) -> Result<(Arc>, Arc), Error> +where + PS: ProgramLoadSource, + P: AsRef, + PM: ProcessManager>, + IO: ProcessIo, +{ + let path = path.as_ref(); + let args = args.iter().map(|&s| s.to_owned()).collect(); + let envs = envs.iter().map(|&s| s.to_owned()).collect(); + + let space = ProcessAddressSpace::new()?; + let (image, args, envs) = xxx_load_program(&space, source, path, args, envs)?; + setup_binary(path.display(), parent, space, image, &args, &envs) +} + +pub fn load_into( + process: &ProcessImpl, + path: P, + args: Vec, + envs: Vec, +) -> Result<(TaskContextImpl, ProcessImage), Error> +where + P: AsRef, + PM: ProcessManager>, + IO: ProcessIo + ProgramLoadSource, +{ + let mut io = process.io.lock(); + // Have to make the Path owned, going to drop the address space from which it came + let path = path.as_ref().to_owned(); + let space = process.space(); + space.clear()?; + let (image, args, envs) = xxx_load_program(&space, &mut *io, &path, args, envs)?; + let context = setup_context(&space, &image, &args, &envs)?; + + Ok((context, image)) +} diff --git a/libk-thread/src/lib.rs b/libk-thread/src/lib.rs index aa2fb686..ad12502e 100644 --- a/libk-thread/src/lib.rs +++ b/libk-thread/src/lib.rs @@ -1,5 +1,13 @@ #![no_std] -#![feature(trait_alias, never_type, inline_const, arbitrary_self_types)] +#![feature( + trait_alias, + never_type, + inline_const, + arbitrary_self_types, + slice_ptr_get, + slice_split_once, + let_chains +)] extern crate alloc; @@ -11,6 +19,7 @@ pub(crate) mod api; #[macro_use] pub mod runtime; +pub mod binary; pub mod futex; pub mod mem; pub mod process; diff --git a/libk-util/src/io.rs b/libk-util/src/io.rs new file mode 100644 index 00000000..6fad6b2a --- /dev/null +++ b/libk-util/src/io.rs @@ -0,0 +1,62 @@ +use alloc::sync::Arc; +use yggdrasil_abi::{error::Error, io::SeekFrom}; + +/// Immutable read interface for VFS objects +pub trait Read { + /// Reads bytes into the given buffer + fn read(&self, buf: &mut [u8]) -> Result; + + /// Reads exactly `buf.len()` bytes into the given buffer, fails if such amount is not + /// available + fn read_exact(&self, buf: &mut [u8]) -> Result<(), Error> { + match self.read(buf) { + Ok(count) if count == buf.len() => Ok(()), + Ok(_) => Err(Error::InvalidFile), + Err(err) => Err(err), + } + } +} + +/// Immutable write interface for VFS objects +pub trait Write { + /// Writes bytes from the given buffer + fn write(&self, buf: &[u8]) -> Result; +} + +/// Immutable file positioning interface for VFS objects +pub trait Seek { + /// Changes position inside the file to a requested one. Fails if the file does not support + /// seeking. + fn seek(&self, from: SeekFrom) -> Result; + + /// Returns the position within the file, determined as an offset in bytes from the beginning + /// of the file. Fails if the file does not support seeking or the "offset" is not defined for + /// such type of nodes. + fn tell(&self) -> Result { + self.seek(SeekFrom::Current(0)) + } +} + +impl Read for Arc { + #[inline] + fn read(&self, buf: &mut [u8]) -> Result { + self.as_ref().read(buf) + } + + #[inline] + fn read_exact(&self, buf: &mut [u8]) -> Result<(), Error> { + self.as_ref().read_exact(buf) + } +} + +impl Seek for Arc { + #[inline] + fn seek(&self, from: SeekFrom) -> Result { + self.as_ref().seek(from) + } + + #[inline] + fn tell(&self) -> Result { + self.as_ref().tell() + } +} diff --git a/libk-util/src/lib.rs b/libk-util/src/lib.rs index f7d5495d..6f8e7c25 100644 --- a/libk-util/src/lib.rs +++ b/libk-util/src/lib.rs @@ -10,6 +10,7 @@ use core::{ }; pub mod event; +pub mod io; pub mod queue; pub mod ring; pub mod sync; diff --git a/libk/src/api.rs b/libk/src/api.rs index 657f9567..b3c24ad6 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -3,8 +3,6 @@ use core::time::Duration; use yggdrasil_abi::{error::Error, process::Signal}; extern "Rust" { - pub fn __cpu_count() -> usize; - pub fn __monotonic_timestamp() -> Result; pub fn __signal_process_group(group_id: u32, signal: Signal); diff --git a/libk/src/lib.rs b/libk/src/lib.rs index a8b97f51..09d99c46 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -19,7 +19,7 @@ use core::time::Duration; use arch::Cpu; -use kernel_arch::CpuImpl; +use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl}; use yggdrasil_abi::{error::Error, process::Signal}; extern crate alloc; @@ -35,7 +35,7 @@ pub mod thread; #[inline] pub fn cpu_count() -> usize { - unsafe { api::__cpu_count() } + ArchitectureImpl::cpu_count() } /// Returns local CPU index diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index a8a75853..77be9ded 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -5,8 +5,8 @@ use aarch64_cpu::{ asm::barrier, registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; -use kernel_arch::absolute_address; -use kernel_arch::{Architecture, ArchitectureImpl}; +use kernel_arch::{absolute_address, Architecture, ArchitectureImpl}; +use kernel_arch_aarch64::CPU_COUNT; use kernel_fs::devfs; use libk::runtime; use libk_mm::{ @@ -16,7 +16,7 @@ use libk_mm::{ }; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use super::{exception, smp::CPU_COUNT, BootStack, BOOT_STACK_SIZE, PLATFORM}; +use super::{exception, BootStack, BOOT_STACK_SIZE, PLATFORM}; use crate::{arch::L3, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET}; unsafe fn pre_init_mmu() { diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index e9f275dc..db543a0b 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -14,7 +14,7 @@ use device_api::{ Device, }; use device_tree::{device_tree_driver, dt::DevTreeIndexPropExt}; -use kernel_arch_aarch64::GicInterface; +use kernel_arch_aarch64::{GicInterface, CPU_COUNT}; use libk::{arch::Cpu, cpu_index, device::register_external_interrupt_controller}; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, @@ -24,7 +24,7 @@ use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use self::{gicc::Gicc, gicd::Gicd}; -use super::{smp::CPU_COUNT, AArch64}; +use super::AArch64; const MAX_IRQ: usize = 300; const IPI_VECTOR: u64 = 1; diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index f564b08c..d3e26187 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -80,10 +80,6 @@ impl Platform for AArch64 { } } - fn cpu_count() -> usize { - smp::CPU_COUNT.load(Ordering::Acquire) - } - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { *self.mtimer.get() } diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index 829793e4..cf49dfe2 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -1,9 +1,10 @@ //! Simultaneous multiprocessing support for aarch64 -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::Ordering; use abi::error::Error; use device_api::CpuBringupDevice; use device_tree::dt::{DevTreeIndexNodePropGet, DeviceTree}; +use kernel_arch_aarch64::CPU_COUNT; use crate::arch::PLATFORM; use crate::mem::KERNEL_VIRT_OFFSET; @@ -65,9 +66,6 @@ impl CpuEnableMethod { } } -/// Number of online CPUs, initially set to 1 (BSP processor is up) -pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); - // TODO can be made smaller #[link_section = ".bss"] static AP_TRAMPOLINE_STACK: BootStack = BootStack { diff --git a/src/arch/mod.rs b/src/arch/mod.rs index f165380a..a461ad4b 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -50,9 +50,6 @@ pub trait Platform { // Architecture intrinsics - /// Returns the count of present CPUs, including the BSP - fn cpu_count() -> usize; - /// Adds a monotonic timer to the system fn register_monotonic_timer( &self, @@ -101,11 +98,6 @@ pub trait Platform { // External API for architecture specifics -#[no_mangle] -fn __cpu_count() -> usize { - PlatformImpl::cpu_count() -} - #[no_mangle] fn __monotonic_timestamp() -> Result { PLATFORM.monotonic_timer().monotonic_timestamp() diff --git a/src/arch/x86_64/acpi.rs b/src/arch/x86_64/acpi.rs index 614d057e..fa2c5390 100644 --- a/src/arch/x86_64/acpi.rs +++ b/src/arch/x86_64/acpi.rs @@ -15,6 +15,7 @@ use device_api::{ interrupt::{InterruptHandler, IpiDeliveryTarget, IpiMessage, Irq}, Device, }; +use kernel_arch_x86_64::CPU_COUNT; use libk::device::external_interrupt_controller; use libk_mm::{ address::{FromRaw, PhysicalAddress, Virtualize}, @@ -25,7 +26,7 @@ use yggdrasil_abi::error::Error; use crate::{ arch::{ - x86_64::{apic::ioapic::ISA_IRQ_OFFSET, smp::CPU_COUNT, SHUTDOWN_FENCE}, + x86_64::{apic::ioapic::ISA_IRQ_OFFSET, SHUTDOWN_FENCE}, Platform, PLATFORM, }, mem::{heap::GLOBAL_HEAP, read_memory, write_memory}, diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index 56e51007..ddc93fd4 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -10,7 +10,9 @@ use device_api::{ }, Device, }; -use kernel_arch_x86_64::{mem::table::L3, registers::MSR_IA32_APIC_BASE, LocalApicInterface}; +use kernel_arch_x86_64::{ + mem::table::L3, registers::MSR_IA32_APIC_BASE, LocalApicInterface, CPU_COUNT, +}; use libk::arch::Cpu; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress}, @@ -27,7 +29,7 @@ use tock_registers::{ registers::{ReadOnly, ReadWrite, WriteOnly}, }; -use crate::arch::x86_64::{apic::APIC_MSI_OFFSET, smp::CPU_COUNT}; +use crate::arch::x86_64::apic::APIC_MSI_OFFSET; use super::{ APIC_IPI_VECTOR, APIC_LINT0_VECTOR, APIC_LINT1_VECTOR, APIC_SPURIOUS_VECTOR, APIC_TIMER_VECTOR, diff --git a/src/arch/x86_64/boot/mod.rs b/src/arch/x86_64/boot/mod.rs index f1358a21..e1d0ef21 100644 --- a/src/arch/x86_64/boot/mod.rs +++ b/src/arch/x86_64/boot/mod.rs @@ -1,7 +1,7 @@ //! x86-64 boot and entry functions use core::{arch::global_asm, sync::atomic::Ordering}; -use kernel_arch_x86_64::registers::MSR_IA32_KERNEL_GS_BASE; +use kernel_arch_x86_64::{registers::MSR_IA32_KERNEL_GS_BASE, CPU_COUNT}; use kernel_fs::devfs; use libk::runtime; use tock_registers::interfaces::Writeable; @@ -10,9 +10,7 @@ use yboot_proto::{ LoadProtocolHeader, LoadProtocolV1, KERNEL_MAGIC, LOADER_MAGIC, PROTOCOL_VERSION_1, }; -use crate::{ - arch::x86_64::smp::CPU_COUNT, kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET, -}; +use crate::{kernel_main, kernel_secondary_main, mem::KERNEL_VIRT_OFFSET}; use super::{cpuid::init_cpuid, exception, PLATFORM}; diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 06f13397..452967f6 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -60,7 +60,6 @@ use self::{ boot::BootData, cpuid::{ProcessorFeatures, PROCESSOR_FEATURES}, peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort}, - smp::CPU_COUNT, }; use super::{IpiMessage, Platform}; @@ -108,10 +107,6 @@ impl Platform for X86_64 { } } - fn cpu_count() -> usize { - CPU_COUNT.load(Ordering::Acquire) - } - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { self.timer.get() } diff --git a/src/arch/x86_64/smp.rs b/src/arch/x86_64/smp.rs index f4fa1197..d38a5729 100644 --- a/src/arch/x86_64/smp.rs +++ b/src/arch/x86_64/smp.rs @@ -1,12 +1,15 @@ //! x86-64 multiprocessing implementation -use core::sync::atomic::{AtomicUsize, Ordering}; +use core::sync::atomic::Ordering; use acpi_lib::platform::{ProcessorInfo, ProcessorState}; use kernel_arch::{Architecture, ArchitectureImpl}; -use kernel_arch_x86_64::mem::{ - flush_tlb_entry, - table::{PageAttributes, PageEntry, PageTable, L1, L2}, - KERNEL_TABLES, +use kernel_arch_x86_64::{ + mem::{ + flush_tlb_entry, + table::{PageAttributes, PageEntry, PageTable, L1, L2}, + KERNEL_TABLES, + }, + CPU_COUNT, }; use libk::arch::Cpu; use libk_mm::{ @@ -20,9 +23,6 @@ use crate::arch::x86_64::boot::__x86_64_ap_entry; use super::acpi::AcpiAllocator; -/// The number of CPUs present in the system -pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); - static AP_BOOTSTRAP_BIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/__x86_64_ap_boot.bin")); const AP_STACK_PAGES: usize = 8; diff --git a/src/init.rs b/src/init.rs index 4423cd9c..2855261d 100644 --- a/src/init.rs +++ b/src/init.rs @@ -51,7 +51,7 @@ pub fn kinit() -> Result<(), Error> { { let (user_init, user_init_main) = - proc::exec::load(&mut ioctx, None, "/init", &["/init", "xxx"], &[])?; + proc::load_binary(&mut ioctx, None, "/init", &["/init", "xxx"], &[])?; let mut io = user_init.io.lock(); io.set_ioctx(ioctx); diff --git a/src/main.rs b/src/main.rs index d848477d..23818470 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,15 +39,11 @@ #![no_main] use arch::Platform; +use kernel_arch::{Architecture, ArchitectureImpl}; use libk::arch::Cpu; use libk_util::sync::SpinFence; -use crate::{ - arch::{PlatformImpl, PLATFORM}, - fs::sysfs, - mem::heap, - task::spawn_kernel_closure, -}; +use crate::{arch::PLATFORM, fs::sysfs, mem::heap, task::spawn_kernel_closure}; extern crate yggdrasil_abi as abi; @@ -75,7 +71,7 @@ static CPU_INIT_FENCE: SpinFence = SpinFence::new(); pub fn kernel_secondary_main() -> ! { // Synchronize the CPUs to this point CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(PlatformImpl::cpu_count()); + CPU_INIT_FENCE.wait_all(ArchitectureImpl::cpu_count()); unsafe { task::enter(); @@ -104,11 +100,11 @@ pub fn kernel_main() -> ! { PLATFORM.start_application_processors(); } - Cpu::init_ipi_queues(PlatformImpl::cpu_count()); + Cpu::init_ipi_queues(ArchitectureImpl::cpu_count()); // Wait until all APs initialize CPU_INIT_FENCE.signal(); - CPU_INIT_FENCE.wait_all(PlatformImpl::cpu_count()); + CPU_INIT_FENCE.wait_all(ArchitectureImpl::cpu_count()); task::init().expect("Failed to initialize the scheduler"); diff --git a/src/panic.rs b/src/panic.rs index 0d433ee8..43dea3c7 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -7,7 +7,7 @@ use libk::arch::Cpu; use libk_util::sync::{hack_locks, SpinFence}; use crate::{ - arch::{Platform, PlatformImpl, PLATFORM}, + arch::{Platform, PLATFORM}, debug::{debug_internal, LogLevel}, device::display::console::flush_consoles, }; @@ -59,7 +59,7 @@ fn panic_handler(pi: &core::panic::PanicInfo) -> ! { .ok(); } - let ap_count = PlatformImpl::cpu_count() - 1; + let ap_count = ArchitectureImpl::cpu_count() - 1; PANIC_HANDLED_FENCE.wait_all(ap_count); unsafe { diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 4d3066c5..e69de29b 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -1,305 +0,0 @@ -//! ELF binary format support -use core::ops::DerefMut; - -use elf::{ - abi::{PF_W, PF_X, PT_LOAD, PT_TLS}, - endian::AnyEndian, - segment::ProgramHeader, - ElfStream, ParseError, -}; -use libk_mm::{ - pointer::{PhysicalRef, PhysicalRefMut}, - process::{ProcessAddressSpace, VirtualRangeBacking}, - table::{EntryLevelExt, MapAttributes}, -}; -use vfs::{FileRef, Read, Seek}; -use yggdrasil_abi::{error::Error, io::SeekFrom}; - -use crate::{ - arch::L3, - task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, -}; - -#[derive(Clone, Copy)] -struct FileReader<'a> { - file: &'a FileRef, -} - -impl elf::io_traits::InputStream for FileReader<'_> { - fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), elf::io_traits::StreamError> { - self.file.read_exact(buf).map_err(conv_stream_error) - } - - fn seek(&mut self, pos: elf::io_traits::SeekFrom) -> Result { - self.file - .seek(conv_seek_from(pos)) - .map_err(conv_stream_error) - } -} - -#[inline] -fn conv_stream_error(_v: Error) -> elf::io_traits::StreamError { - elf::io_traits::StreamError { - message: "Elf read error", - } -} - -#[inline] -fn conv_seek_from(v: elf::io_traits::SeekFrom) -> SeekFrom { - match v { - elf::io_traits::SeekFrom::End(off) => SeekFrom::End(off), - elf::io_traits::SeekFrom::Start(off) => SeekFrom::Start(off), - _ => todo!(), - } -} - -#[inline] -fn from_parse_error(v: ParseError) -> Error { - warnln!("ELF loading error: {:?}", v); - Error::InvalidFile -} - -/// Creates a new copy of the TLS from given master image -pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result { - let Some(tls) = image.tls.as_ref() else { - // No TLS - return Ok(0); - }; - - assert_ne!(tls.master_copy_base, 0); - assert_ne!(tls.layout.mem_size, 0); - - let address = space.allocate( - None, - 0x1000, - VirtualRangeBacking::anonymous(), - MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, - )?; - - debugln!( - "Clone TLS from {:#x} (data={:#x}) to {:#x} (data={:#x})", - tls.master_copy_base, - tls.master_copy_base + tls.layout.data_offset, - address, - address + tls.layout.data_offset - ); - debugln!("tls ptr = {:#x}", address + tls.layout.ptr_offset); - - let src_phys = space.translate(tls.master_copy_base)?; - let dst_phys = space.translate(address)?; - - // Copy data - unsafe { - let src = - PhysicalRef::::map_slice(src_phys.add(tls.layout.data_offset), tls.layout.mem_size); - let mut dst = - PhysicalRefMut::map_slice(dst_phys.add(tls.layout.data_offset), tls.layout.mem_size); - - dst.copy_from_slice(&src); - } - - // Setup self-pointer - unsafe { - let mut dst = PhysicalRefMut::::map(dst_phys.add(tls.layout.ptr_offset)); - - *dst = address + tls.layout.ptr_offset; - } - - Ok(address + tls.layout.ptr_offset) -} - -fn load_bytes( - space: &ProcessAddressSpace, - addr: usize, - mut src: F, - len: usize, -) -> Result<(), Error> -where - F: FnMut(usize, PhysicalRefMut<'_, [u8]>) -> Result<(), Error>, -{ - // TODO check for crazy addresses here - - let dst_page_off = addr & 0xFFF; - let dst_page_aligned = addr & !0xFFF; - let mut off = 0usize; - let mut rem = len; - - while rem != 0 { - let page_idx = (dst_page_off + off) / 0x1000; - let page_off = (dst_page_off + off) % 0x1000; - let count = core::cmp::min(rem, 0x1000 - page_off); - - let virt_page = dst_page_aligned + page_idx * 0x1000; - assert_eq!(virt_page & 0xFFF, 0); - - let phys_page = space.translate(virt_page)?; - let dst_slice = unsafe { PhysicalRefMut::map_slice(phys_page.add(page_off), count) }; - - src(off, dst_slice)?; - - rem -= count; - off += count; - } - - Ok(()) -} - -fn load_segment( - space: &ProcessAddressSpace, - phdr: &ProgramHeader, - file: &FileReader, -) -> Result<(), Error> { - if phdr.p_memsz == 0 { - return Ok(()); - } - - let attrs = match (phdr.p_flags & PF_W, phdr.p_flags & PF_X) { - (0, 0) => MapAttributes::USER_READ, - (_, 0) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, - (0, _) => MapAttributes::USER_READ, - (_, _) => MapAttributes::USER_WRITE | MapAttributes::USER_READ, - } | MapAttributes::NON_GLOBAL; - - // Map the range - let aligned_start = (phdr.p_vaddr as usize) & !0xFFF; - let aligned_end = ((phdr.p_vaddr + phdr.p_memsz) as usize + 0xFFF) & !0xFFF; - - space.map( - aligned_start, - aligned_end - aligned_start, - VirtualRangeBacking::anonymous(), - attrs, - )?; - - if phdr.p_filesz > 0 { - load_bytes( - space, - phdr.p_vaddr as usize, - |off, mut dst| { - file.file - .seek(SeekFrom::Start(phdr.p_offset + off as u64))?; - file.file.read_exact(dst.deref_mut()) - }, - phdr.p_filesz as usize, - )?; - } - - if phdr.p_memsz > phdr.p_filesz { - let addr = (phdr.p_vaddr + phdr.p_filesz) as usize; - let len = (phdr.p_memsz - phdr.p_filesz) as usize; - - load_bytes( - space, - addr, - |_, mut dst| { - dst.fill(0); - Ok(()) - }, - len, - )?; - } - - Ok(()) -} - -fn tls_segment( - space: &ProcessAddressSpace, - phdr: &ProgramHeader, - file: &FileReader, -) -> Result { - assert_ne!(phdr.p_memsz, 0); - - if !phdr.p_align.is_power_of_two() { - return Err(Error::InvalidArgument); - } - - let layout = ProcessTlsLayout::new(phdr.p_align as _, phdr.p_filesz as _, phdr.p_memsz as _); - let data_offset = layout.data_offset; - let data_size = layout.data_size; - let mem_size = layout.mem_size; - let aligned_size = (layout.full_size + 0xFFF) & !0xFFF; - assert!(aligned_size <= 0x1000); - - let base_address = space.allocate( - None, - aligned_size, - VirtualRangeBacking::anonymous(), - MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, - )?; - - debugln!( - "Allocated TLS master copy @ {:#x}, tls={:#x}, tls_data={:#x}", - base_address, - base_address + layout.ptr_offset, - base_address + layout.data_offset - ); - - let tls = ProcessTlsInfo { - master_copy_base: base_address, - layout, - }; - - if data_size > 0 { - load_bytes( - space, - base_address + data_offset, - |off, mut dst| { - file.file - .seek(SeekFrom::Start(phdr.p_offset + off as u64))?; - file.file.read_exact(dst.deref_mut()) - }, - data_size, - )?; - } - - if mem_size > data_size { - load_bytes( - space, - base_address + data_offset + data_size, - |_, mut dst| { - dst.fill(0); - Ok(()) - }, - mem_size - data_size, - )?; - } - - Ok(tls) -} - -/// Loads an ELF binary from `file` into the target address space -pub fn load_elf_from_file( - space: &ProcessAddressSpace, - file: FileRef, -) -> Result { - let file = FileReader { file: &file }; - let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; - let mut image_end = 0; - let mut tls = None; - - for phdr in elf.segments() { - match phdr.p_type { - PT_LOAD => { - if phdr.p_vaddr + phdr.p_memsz > image_end { - image_end = phdr.p_vaddr + phdr.p_memsz; - } - - load_segment(space, phdr, &file)?; - } - PT_TLS => { - assert!(tls.is_none()); - tls.replace(tls_segment(space, phdr, &file)?); - } - _ => (), - } - } - - let image_end = (image_end as usize).page_align_up::(); - - debugln!("Loaded image end: {:#x}", image_end); - - Ok(ProcessImage { - entry: elf.ehdr.e_entry as usize, - tls, - }) -} diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 4cddf3a4..0e12beee 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -28,247 +28,3 @@ use crate::{ proc, task::process::{ProcessImage, ProcessImpl}, }; - -struct BufferPlacer<'a> { - buffer: &'a mut [u8], - virtual_offset: usize, - offset: usize, -} - -impl<'a> BufferPlacer<'a> { - pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self { - Self { - buffer, - virtual_offset, - offset: 0, - } - } - - unsafe fn alloc_layout( - &mut self, - layout: Layout, - ) -> Result<(NonNull, NonNull), Error> { - // TODO checks - let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1); - self.offset = aligned + layout.size(); - Ok(( - NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T), - NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T), - )) - } -} - -unsafe impl<'a> Placer for BufferPlacer<'a> { - fn place_ref(&mut self, r: &T) -> Result, Error> { - let layout = Layout::new::(); - unsafe { - let (kernel, user) = self.alloc_layout::(layout)?; - kernel.as_ptr().write(r.place(self)?); - Ok(user) - } - } - - fn place_slice(&mut self, r: &[T]) -> Result, Error> { - let layout = Layout::array::(r.len()).unwrap(); - unsafe { - let (kernel, user) = self.alloc_layout::(layout)?; - let kernel = NonNull::slice_from_raw_parts(kernel, r.len()); - let user = NonNull::slice_from_raw_parts(user, r.len()); - for (i, elem) in r.iter().enumerate() { - kernel - .get_unchecked_mut(i) - .as_ptr() - .write(elem.place(self)?); - } - Ok(user) - } - } -} - -// args, envs are passed as Vec to ensure kernel ownership -#[allow(clippy::ptr_arg)] -fn setup_program_env( - space: &ProcessAddressSpace, - virt: usize, - args: &Vec, - envs: &Vec, -) -> Result { - // TODO growing buffer - let phys_page = space.map_single( - virt, - VirtualRangeBacking::anonymous(), - MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, - )?; - let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; - let mut placer = BufferPlacer::new(virt, &mut buffer); - - let args = args.iter().map(String::as_ref).collect::>(); - let envs = envs.iter().map(String::as_ref).collect::>(); - - let in_kernel = ProgramArgumentInner { - args: &args, - env: &envs, - }; - let in_user = in_kernel.place_ref(&mut placer)?; - - Ok(in_user as *const _ as usize) -} - -fn setup_context( - space: &ProcessAddressSpace, - image: &ProcessImage, - args: &Vec, - envs: &Vec, -) -> Result { - const USER_STACK_PAGES: usize = 32; - - let virt_stack_base = 0x3000000; - // 0x1000 of guard page - let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000; - - space.map( - virt_stack_base, - USER_STACK_PAGES * 0x1000, - VirtualRangeBacking::anonymous(), - MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - )?; - - let arg = setup_program_env(space, virt_args_base, args, envs)?; - - debugln!( - "Entry: {:#x}, Stack: {:#x}..{:#x}, Args: {:#x}", - image.entry, - virt_stack_base, - virt_stack_base + USER_STACK_PAGES * 0x1000, - virt_args_base - ); - - let user_sp = - virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN; - - // Fill with some sentinel value to detect stack underflows - let ptr = user_sp as *mut u64; - - #[allow(clippy::reversed_empty_ranges)] - for i in 0..TaskContextImpl::USER_STACK_EXTRA_ALIGN / 8 { - unsafe { - ptr.add(i).write_foreign_volatile(space, 0xDEADC0DE); - } - } - - let tls_address = proc::elf::clone_tls(space, image)?; - - debugln!( - "Create User TaskContext: entry = {:#x}, rsp3 = {:#x}, stack: {:#x?}", - image.entry, - user_sp, - virt_stack_base..virt_stack_base + USER_STACK_PAGES * 0x1000 - ); - - TaskContext::user( - image.entry, - arg, - space.as_address_with_asid(), - user_sp, - tls_address, - ) -} - -fn setup_binary>( - name: S, - parent: Option>, - space: ProcessAddressSpace, - image: ProcessImage, - args: &Vec, - envs: &Vec, -) -> Result<(Arc, Arc), Error> { - let context = setup_context(&space, &image, args, envs)?; - let (process, main) = - ProcessImpl::new_with_main(name, parent, Arc::new(space), context, Some(image)); - infoln!("exec::setup_binary -> {:?}", process.id()); - Ok((process, main)) -} - -fn load_binary( - head: &[u8], - file: FileRef, - space: &ProcessAddressSpace, -) -> Result { - if head.starts_with(b"\x7FELF") { - proc::elf::load_elf_from_file(space, file) - } else { - Err(Error::UnrecognizedExecutable) - } -} - -fn xxx_load_program>( - space: &ProcessAddressSpace, - ioctx: &mut IoContext, - path: P, - args: Vec, - envs: Vec, -) -> Result<(ProcessImage, Vec, Vec), Error> { - let mut head = [0; 256]; - let path = path.as_ref(); - let file = ioctx.open_exec(None, path)?; - - infoln!("exec::load({:?}, {:?})", path, args); - - file.seek(SeekFrom::Start(0))?; - let count = file.read(&mut head)?; - let head = &head[..count]; - - if let Some(shebang) = head.strip_prefix(b"#!") - && let Some((shebang, _)) = shebang.split_once(|&ch| ch == b'\n') - { - let shebang = core::str::from_utf8(shebang).map_err(|_| Error::InvalidFile)?; - let mut shebang_args = shebang.split(' ').map(|s| s.to_owned()).collect::>(); - if shebang_args.is_empty() || shebang_args.len() >= 8 { - return Err(Error::UnrecognizedExecutable); - } - shebang_args.extend_from_slice(&args); - - return xxx_load_program(space, ioctx, shebang_args[0].clone(), shebang_args, envs); - } - - file.seek(SeekFrom::Start(0))?; - - let image = load_binary(head, file, space)?; - - Ok((image, args, envs)) -} - -/// Loads a program from given `path` -pub fn load>( - ioctx: &mut IoContext, - parent: Option>, - path: P, - args: &[&str], - envs: &[&str], -) -> Result<(Arc, Arc), Error> { - let path = path.as_ref(); - let args = args.iter().map(|&s| s.to_owned()).collect(); - let envs = envs.iter().map(|&s| s.to_owned()).collect(); - - let space = ProcessAddressSpace::new()?; - let (image, args, envs) = xxx_load_program(&space, ioctx, path, args, envs)?; - setup_binary(path.display(), parent, space, image, &args, &envs) -} - -/// XXX -pub fn load_into>( - process: &ProcessImpl, - path: P, - args: Vec, - envs: Vec, -) -> Result<(TaskContextImpl, ProcessImage), Error> { - let mut io = process.io.lock(); - // Have to make the Path owned, going to drop the address space from which it came - let path = path.as_ref().to_owned(); - let space = process.space(); - space.clear()?; - let (image, args, envs) = xxx_load_program(&space, io.ioctx_mut(), &path, args, envs)?; - let context = setup_context(&space, &image, &args, &envs)?; - - Ok((context, image)) -} diff --git a/src/proc/mod.rs b/src/proc/mod.rs index 3e1e2bf1..daf65ad4 100644 --- a/src/proc/mod.rs +++ b/src/proc/mod.rs @@ -1,6 +1,23 @@ //! Internal management for processes -pub mod elf; -pub mod exec; +use abi::{error::Error, path::Path}; +use alloc::sync::{Arc, Weak}; +use libk_thread::thread::Thread; +use vfs::IoContext; + +use crate::task::process::ProcessImpl; + pub mod io; pub mod random; + +/// Loads a binary and creates a process for it. See [libk_thread::binary::load]. +#[inline] +pub fn load_binary>( + ioctx: &mut IoContext, + parent: Option>, + path: P, + args: &[&str], + envs: &[&str], +) -> Result<(Arc, Arc), Error> { + libk_thread::binary::load(ioctx, parent, path, args, envs) +} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index b6f7ce09..fc8a3e51 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -527,7 +527,7 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result run_with_io(&process, |mut io| { // Setup a new process from the file - let (child_process, child_main) = proc::exec::load( + let (child_process, child_main) = proc::load_binary( io.ioctx_mut(), Some(Arc::downgrade(&process)), options.program, diff --git a/src/task/mod.rs b/src/task/mod.rs index 6e28c755..6abc6b9a 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -4,7 +4,10 @@ use abi::error::Error; use alloc::{string::String, vec::Vec}; -use kernel_arch::task::{Scheduler, TaskContext}; +use kernel_arch::{ + task::{Scheduler, TaskContext}, + Architecture, ArchitectureImpl, +}; use libk::{ arch::Cpu, runtime, @@ -16,8 +19,6 @@ use libk_thread::{ }; use libk_util::sync::SpinFence; -use crate::arch::{Platform, PlatformImpl}; - use self::process::ProcessManagerImpl; pub mod process; @@ -41,7 +42,7 @@ pub fn spawn_kernel_closure, T: Termination, F: Fn() -> T + Send /// Sets up CPU queues and gives them some processes to run pub fn init() -> Result<(), Error> { - let cpu_count = PlatformImpl::cpu_count(); + let cpu_count = ArchitectureImpl::cpu_count(); // Create a queue for each CPU init_queues(Vec::from_iter((0..cpu_count).map(CpuQueue::new))); From a5724eb93d12775c129d2b93c2b9ed87e81e550d Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 14 Feb 2024 14:45:18 +0200 Subject: [PATCH 191/211] libk: move devices to libk-device --- Cargo.toml | 1 + driver/net/core/src/socket.rs | 3 ++- libk-device/Cargo.toml | 13 +++++++++++ libk/src/device.rs => libk-device/src/lib.rs | 24 +++++++++++++++++--- libk-thread/Cargo.toml | 1 + libk-thread/src/api.rs | 7 ------ libk-thread/src/lib.rs | 2 -- libk-thread/src/runtime/timer.rs | 6 ++--- libk/Cargo.toml | 1 + libk/src/api.rs | 6 +---- libk/src/lib.rs | 14 ++++-------- src/arch/aarch64/mod.rs | 17 -------------- src/arch/aarch64/timer.rs | 10 ++++---- src/arch/mod.rs | 23 ------------------- src/arch/x86_64/mod.rs | 15 ++++-------- src/proc/random.rs | 8 ++----- 16 files changed, 59 insertions(+), 92 deletions(-) create mode 100644 libk-device/Cargo.toml rename libk/src/device.rs => libk-device/src/lib.rs (68%) delete mode 100644 libk-thread/src/api.rs diff --git a/Cargo.toml b/Cargo.toml index 0e1e8e28..c2a84d31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ libk = { path = "libk" } libk-util = { path = "libk-util" } libk-mm = { path = "libk-mm" } libk-thread = { path = "libk-thread" } +libk-device = { path = "libk-device" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 08ed82bf..07aeaee1 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -8,7 +8,8 @@ use core::{ use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use libk::{ - block, monotonic_timestamp, + block, + device::monotonic_timestamp, runtime::{run_with_timeout, FutureTimeout}, }; use libk_mm::PageBox; diff --git a/libk-device/Cargo.toml b/libk-device/Cargo.toml new file mode 100644 index 00000000..b3c7b74c --- /dev/null +++ b/libk-device/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "libk-device" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +libk-util = { path = "../libk-util" } +kernel-arch = { path = "../arch" } + +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../lib/device-api", features = ["derive"] } diff --git a/libk/src/device.rs b/libk-device/src/lib.rs similarity index 68% rename from libk/src/device.rs rename to libk-device/src/lib.rs index eeef2e45..7feca539 100644 --- a/libk/src/device.rs +++ b/libk-device/src/lib.rs @@ -1,6 +1,13 @@ -use device_api::interrupt::{ - ExternalInterruptController, InterruptHandler, Irq, IrqOptions, LocalInterruptController, - MessageInterruptController, +#![no_std] + +use core::time::Duration; + +use device_api::{ + interrupt::{ + ExternalInterruptController, InterruptHandler, Irq, IrqOptions, LocalInterruptController, + MessageInterruptController, + }, + timer::MonotonicTimestampProviderDevice, }; use kernel_arch::{Architecture, ArchitectureImpl}; use libk_util::OneTimeInit; @@ -27,6 +34,13 @@ register_get!( &'static dyn ExternalInterruptController ); +register_get!( + register_monotonic_timestamp_provider, + monotonic_timestamp_provider, + MONOTONIC_TIMER, + &'static dyn MonotonicTimestampProviderDevice +); + pub fn local_interrupt_controller() -> &'static dyn LocalInterruptController { ArchitectureImpl::local_interrupt_controller() } @@ -35,6 +49,10 @@ pub fn message_interrupt_controller() -> &'static dyn MessageInterruptController ArchitectureImpl::message_interrupt_controller() } +pub fn monotonic_timestamp() -> Result { + monotonic_timestamp_provider().monotonic_timestamp() +} + #[inline] pub fn register_global_interrupt( irq: u32, diff --git a/libk-thread/Cargo.toml b/libk-thread/Cargo.toml index d001907f..83afaf16 100644 --- a/libk-thread/Cargo.toml +++ b/libk-thread/Cargo.toml @@ -8,6 +8,7 @@ edition = "2021" [dependencies] libk-util = { path = "../libk-util" } libk-mm = { path = "../libk-mm" } +libk-device = { path = "../libk-device" } kernel-arch = { path = "../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/libk-thread/src/api.rs b/libk-thread/src/api.rs deleted file mode 100644 index 82d9549d..00000000 --- a/libk-thread/src/api.rs +++ /dev/null @@ -1,7 +0,0 @@ -use core::time::Duration; - -use yggdrasil_abi::error::Error; - -extern "Rust" { - pub fn __monotonic_timestamp() -> Result; -} diff --git a/libk-thread/src/lib.rs b/libk-thread/src/lib.rs index ad12502e..9dfec935 100644 --- a/libk-thread/src/lib.rs +++ b/libk-thread/src/lib.rs @@ -14,8 +14,6 @@ extern crate alloc; use kernel_arch::KernelTableManagerImpl; use libk_mm::phys::GlobalPhysicalAllocator; -pub(crate) mod api; - #[macro_use] pub mod runtime; diff --git a/libk-thread/src/runtime/timer.rs b/libk-thread/src/runtime/timer.rs index f573296a..0150ee6e 100644 --- a/libk-thread/src/runtime/timer.rs +++ b/libk-thread/src/runtime/timer.rs @@ -8,8 +8,6 @@ use alloc::vec::Vec; use futures_util::{future::BoxFuture, Future, FutureExt}; use libk_util::sync::IrqSafeSpinlock; -use crate::api; - // 1..32ms, tick every 1ms static SHORT_TERM_SLEEPS: IrqSafeSpinlock> = IrqSafeSpinlock::new(TimerWheel::new(0, 1)); @@ -84,7 +82,7 @@ impl Future for SleepFuture { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + let now = libk_device::monotonic_timestamp().unwrap(); match self.deadline.checked_sub(now) { // Pending Some(duration) if !duration.is_zero() => { @@ -99,7 +97,7 @@ impl Future for SleepFuture { /// Suspends the task until given duration passes pub fn sleep(duration: Duration) -> SleepFuture { - let now = unsafe { api::__monotonic_timestamp() }.unwrap(); + let now = libk_device::monotonic_timestamp().unwrap(); let deadline = now + duration; SleepFuture { deadline } diff --git a/libk/Cargo.toml b/libk/Cargo.toml index 35073b8f..bb3b597d 100644 --- a/libk/Cargo.toml +++ b/libk/Cargo.toml @@ -10,6 +10,7 @@ authors = ["Mark Poliakov "] libk-mm = { path = "../libk-mm" } libk-util = { path = "../libk-util" } libk-thread = { path = "../libk-thread" } +libk-device = { path = "../libk-device" } kernel-arch = { path = "../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/libk/src/api.rs b/libk/src/api.rs index b3c24ad6..254032b9 100644 --- a/libk/src/api.rs +++ b/libk/src/api.rs @@ -1,9 +1,5 @@ -use core::time::Duration; - -use yggdrasil_abi::{error::Error, process::Signal}; +use yggdrasil_abi::process::Signal; extern "Rust" { - pub fn __monotonic_timestamp() -> Result; - pub fn __signal_process_group(group_id: u32, signal: Signal); } diff --git a/libk/src/lib.rs b/libk/src/lib.rs index 09d99c46..52ab1455 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -16,11 +16,9 @@ trait_alias )] -use core::time::Duration; - use arch::Cpu; use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl}; -use yggdrasil_abi::{error::Error, process::Signal}; +use yggdrasil_abi::process::Signal; extern crate alloc; @@ -29,10 +27,13 @@ pub use libk_thread::{block, runtime}; pub(crate) mod api; pub mod arch; -pub mod device; pub mod sync; pub mod thread; +pub mod device { + pub use libk_device::*; +} + #[inline] pub fn cpu_count() -> usize { ArchitectureImpl::cpu_count() @@ -54,8 +55,3 @@ pub struct AlignedTo { pub align: [Align; 0], pub bytes: Bytes, } - -#[inline] -pub fn monotonic_timestamp() -> Result { - unsafe { api::__monotonic_timestamp() } -} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index d3e26187..ded07213 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -6,7 +6,6 @@ use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0}; use abi::error::Error; use device_api::{ interrupt::{IpiDeliveryTarget, IpiMessage, Irq, LocalInterruptController}, - timer::MonotonicTimestampProviderDevice, ResetDevice, }; use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}; @@ -63,8 +62,6 @@ pub struct AArch64 { pub psci: OneTimeInit<&'static Psci>, reset: OneTimeInit<&'static dyn ResetDevice>, - mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, - initrd: OneTimeInit>, } @@ -80,18 +77,6 @@ impl Platform for AArch64 { } } - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { - *self.mtimer.get() - } - - fn register_monotonic_timer( - &self, - timer: &'static dyn MonotonicTimestampProviderDevice, - ) -> Result<(), Error> { - self.mtimer.init(timer); - Ok(()) - } - unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: IpiMessage) -> Result<(), Error> { Ok(()) // XXX @@ -374,6 +359,4 @@ pub static PLATFORM: AArch64 = AArch64 { psci: OneTimeInit::new(), reset: OneTimeInit::new(), - - mtimer: OneTimeInit::new(), }; diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index 84b9ade0..9579cc7e 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -12,11 +12,13 @@ use device_api::{ }; use device_tree::device_tree_driver; use kernel_arch::task::Scheduler; -use libk::{arch::Cpu, device::external_interrupt_controller, runtime}; +use libk::{ + arch::Cpu, + device::{external_interrupt_controller, register_monotonic_timestamp_provider}, + runtime, +}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use crate::arch::{Platform, PLATFORM}; - /// ARM Generic Timer driver pub struct ArmTimer { irq: Irq, @@ -56,7 +58,7 @@ impl Device for ArmTimer { unsafe fn init(&'static self) -> Result<(), Error> { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); - PLATFORM.register_monotonic_timer(self)?; + register_monotonic_timestamp_provider(self); Ok(()) } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index a461ad4b..c742d66b 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -1,13 +1,10 @@ //! Provides architecture/platform-specific implementation details -use core::time::Duration; - use abi::error::Error; use cfg_if::cfg_if; use device_api::{ interrupt::{IpiDeliveryTarget, IpiMessage}, - timer::MonotonicTimestampProviderDevice, ResetDevice, }; use kernel_arch::{Architecture, ArchitectureImpl}; @@ -50,24 +47,11 @@ pub trait Platform { // Architecture intrinsics - /// Adds a monotonic timer to the system - fn register_monotonic_timer( - &self, - timer: &'static dyn MonotonicTimestampProviderDevice, - ) -> Result<(), Error> { - Err(Error::NotImplemented) - } - /// Adds a reset device to the system fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { Err(Error::NotImplemented) } - /// Returns the monotonic timer - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { - unimplemented!() - } - /// Sends a message to the requested set of CPUs through an interprocessor interrupt. /// /// # Note @@ -95,10 +79,3 @@ pub trait Platform { } } } - -// External API for architecture specifics - -#[no_mangle] -fn __monotonic_timestamp() -> Result { - PLATFORM.monotonic_timer().monotonic_timestamp() -} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 452967f6..673a195d 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -6,7 +6,6 @@ use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; use device_api::{ interrupt::{Irq, MessageInterruptController}, - timer::MonotonicTimestampProviderDevice, Device, }; use git_version::git_version; @@ -20,6 +19,7 @@ use kernel_arch_x86_64::{ }; use kernel_fs::devfs; use libk::{arch::Cpu, device::register_external_interrupt_controller}; +use libk_device::register_monotonic_timestamp_provider; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress, Virtualize}, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, @@ -72,8 +72,6 @@ pub struct X86_64 { // Display framebuffer: OneTimeInit, fbconsole: OneTimeInit, - - timer: OneTimeInit, } static SHUTDOWN_FENCE: SpinFence = SpinFence::new(); @@ -85,8 +83,6 @@ pub static PLATFORM: X86_64 = X86_64 { framebuffer: OneTimeInit::new(), fbconsole: OneTimeInit::new(), - - timer: OneTimeInit::new(), }; impl Platform for X86_64 { @@ -106,10 +102,6 @@ impl Platform for X86_64 { smp::start_ap_cores(&pinfo); } } - - fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { - self.timer.get() - } } impl X86_64 { @@ -331,7 +323,8 @@ impl X86_64 { Self::disable_8259(); - self.timer.init(I8253::new()); + let timer = Box::leak(Box::new(I8253::new())); + register_monotonic_timestamp_provider(timer); let com1_3 = Box::leak(Box::new(ComPort::new( 0x3F8, @@ -362,7 +355,7 @@ impl X86_64 { self.init_platform_from_acpi(acpi)?; } - self.timer.get().init_irq()?; + timer.init_irq()?; // ps2.connect(self.tty.get()); ps2.init_irq()?; diff --git a/src/proc/random.rs b/src/proc/random.rs index c9340947..cbf2873c 100644 --- a/src/proc/random.rs +++ b/src/proc/random.rs @@ -1,9 +1,8 @@ //! Random generation utilities +use libk::device::monotonic_timestamp; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; -use crate::arch::{Platform, PLATFORM}; - const BUFFER_SIZE: usize = 1024; struct RandomState { @@ -67,10 +66,7 @@ pub fn read(buf: &mut [u8]) { /// Initializes the random generator state pub fn init() { - let now = PLATFORM - .monotonic_timer() - .monotonic_timestamp() - .unwrap(); + let now = monotonic_timestamp().unwrap(); let random_seed = now.subsec_millis(); let mut state = RandomState::new(random_seed); From dbebc56ba8eeead01dd8ab002c7dcf717e0169dd Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 14 Feb 2024 17:20:42 +0200 Subject: [PATCH 192/211] libk: move libk sub-libs inside libk dir --- Cargo.toml | 8 ++++---- arch/aarch64/Cargo.toml | 2 +- arch/x86_64/Cargo.toml | 2 +- driver/block/ahci/Cargo.toml | 6 +++--- driver/block/ahci/src/lib.rs | 2 +- driver/block/core/Cargo.toml | 5 ++--- driver/block/nvme/Cargo.toml | 5 ++--- driver/bus/pci/Cargo.toml | 6 +++--- driver/bus/pci/src/device.rs | 2 +- driver/fs/kernel-fs/Cargo.toml | 2 +- driver/fs/memfs/Cargo.toml | 3 +-- driver/net/core/Cargo.toml | 7 ++++--- driver/net/core/src/l3/arp.rs | 2 +- driver/net/core/src/lib.rs | 2 +- driver/net/core/src/socket.rs | 6 +++--- driver/net/loopback/Cargo.toml | 4 ++-- driver/virtio/core/Cargo.toml | 2 +- driver/virtio/net/Cargo.toml | 4 ++-- lib/device-tree/Cargo.toml | 3 +-- lib/vfs/Cargo.toml | 7 +++---- lib/vfs/src/channel.rs | 2 +- lib/vfs/src/file/pipe.rs | 2 +- lib/vfs/src/poll.rs | 2 +- lib/vfs/src/pty.rs | 2 +- lib/vfs/src/timer.rs | 4 ++-- libk/Cargo.toml | 8 ++++---- {libk-device => libk/libk-device}/Cargo.toml | 4 ++-- {libk-device => libk/libk-device}/src/lib.rs | 0 {libk-mm => libk/libk-mm}/Cargo.toml | 4 ++-- {libk-mm => libk/libk-mm}/interface/Cargo.toml | 2 +- .../libk-mm}/interface/src/address.rs | 0 {libk-mm => libk/libk-mm}/interface/src/lib.rs | 0 .../libk-mm}/interface/src/pointer.rs | 0 .../libk-mm}/interface/src/process.rs | 0 {libk-mm => libk/libk-mm}/interface/src/table.rs | 0 {libk-mm => libk/libk-mm}/src/address.rs | 0 {libk-mm => libk/libk-mm}/src/device.rs | 0 {libk-mm => libk/libk-mm}/src/lib.rs | 0 {libk-mm => libk/libk-mm}/src/phys/manager.rs | 0 {libk-mm => libk/libk-mm}/src/phys/mod.rs | 0 {libk-mm => libk/libk-mm}/src/phys/reserved.rs | 0 {libk-mm => libk/libk-mm}/src/pointer.rs | 0 {libk-mm => libk/libk-mm}/src/process.rs | 0 {libk-thread => libk/libk-thread}/Cargo.toml | 2 +- .../libk-thread}/src/binary/elf.rs | 0 .../libk-thread}/src/binary/mod.rs | 0 {libk-thread => libk/libk-thread}/src/futex.rs | 0 {libk-thread => libk/libk-thread}/src/lib.rs | 16 ++++++++++++++++ {libk-thread => libk/libk-thread}/src/mem.rs | 0 {libk-thread => libk/libk-thread}/src/process.rs | 0 .../libk-thread}/src/runtime/executor.rs | 0 .../libk-thread}/src/runtime/macros.rs | 0 .../libk-thread}/src/runtime/mod.rs | 0 .../libk-thread}/src/runtime/task.rs | 0 .../libk-thread}/src/runtime/task_queue.rs | 0 .../libk-thread}/src/runtime/timer.rs | 0 {libk-thread => libk/libk-thread}/src/sched.rs | 0 .../sync/mutex.rs => libk-thread/src/sync.rs} | 3 ++- {libk-thread => libk/libk-thread}/src/thread.rs | 0 {libk-thread => libk/libk-thread}/src/types.rs | 0 {libk-util => libk/libk-util}/Cargo.toml | 2 +- {libk-util => libk/libk-util}/src/event.rs | 0 {libk-util => libk/libk-util}/src/io.rs | 0 {libk-util => libk/libk-util}/src/lib.rs | 0 {libk-util => libk/libk-util}/src/queue.rs | 0 {libk-util => libk/libk-util}/src/ring.rs | 0 {libk-util => libk/libk-util}/src/sync/fence.rs | 0 {libk-util => libk/libk-util}/src/sync/mod.rs | 0 .../libk-util}/src/sync/spin_rwlock.rs | 0 .../libk-util}/src/sync/spinlock.rs | 0 {libk-util => libk/libk-util}/src/waker.rs | 0 libk/src/api.rs | 5 ----- libk/src/lib.rs | 9 --------- libk/src/sync/mod.rs | 2 -- 74 files changed, 73 insertions(+), 76 deletions(-) rename {libk-device => libk/libk-device}/Cargo.toml (73%) rename {libk-device => libk/libk-device}/src/lib.rs (100%) rename {libk-mm => libk/libk-mm}/Cargo.toml (81%) rename {libk-mm => libk/libk-mm}/interface/Cargo.toml (82%) rename {libk-mm => libk/libk-mm}/interface/src/address.rs (100%) rename {libk-mm => libk/libk-mm}/interface/src/lib.rs (100%) rename {libk-mm => libk/libk-mm}/interface/src/pointer.rs (100%) rename {libk-mm => libk/libk-mm}/interface/src/process.rs (100%) rename {libk-mm => libk/libk-mm}/interface/src/table.rs (100%) rename {libk-mm => libk/libk-mm}/src/address.rs (100%) rename {libk-mm => libk/libk-mm}/src/device.rs (100%) rename {libk-mm => libk/libk-mm}/src/lib.rs (100%) rename {libk-mm => libk/libk-mm}/src/phys/manager.rs (100%) rename {libk-mm => libk/libk-mm}/src/phys/mod.rs (100%) rename {libk-mm => libk/libk-mm}/src/phys/reserved.rs (100%) rename {libk-mm => libk/libk-mm}/src/pointer.rs (100%) rename {libk-mm => libk/libk-mm}/src/process.rs (100%) rename {libk-thread => libk/libk-thread}/Cargo.toml (95%) rename {libk-thread => libk/libk-thread}/src/binary/elf.rs (100%) rename {libk-thread => libk/libk-thread}/src/binary/mod.rs (100%) rename {libk-thread => libk/libk-thread}/src/futex.rs (100%) rename {libk-thread => libk/libk-thread}/src/lib.rs (61%) rename {libk-thread => libk/libk-thread}/src/mem.rs (100%) rename {libk-thread => libk/libk-thread}/src/process.rs (100%) rename {libk-thread => libk/libk-thread}/src/runtime/executor.rs (100%) rename {libk-thread => libk/libk-thread}/src/runtime/macros.rs (100%) rename {libk-thread => libk/libk-thread}/src/runtime/mod.rs (100%) rename {libk-thread => libk/libk-thread}/src/runtime/task.rs (100%) rename {libk-thread => libk/libk-thread}/src/runtime/task_queue.rs (100%) rename {libk-thread => libk/libk-thread}/src/runtime/timer.rs (100%) rename {libk-thread => libk/libk-thread}/src/sched.rs (100%) rename libk/{src/sync/mutex.rs => libk-thread/src/sync.rs} (98%) rename {libk-thread => libk/libk-thread}/src/thread.rs (100%) rename {libk-thread => libk/libk-thread}/src/types.rs (100%) rename {libk-util => libk/libk-util}/Cargo.toml (92%) rename {libk-util => libk/libk-util}/src/event.rs (100%) rename {libk-util => libk/libk-util}/src/io.rs (100%) rename {libk-util => libk/libk-util}/src/lib.rs (100%) rename {libk-util => libk/libk-util}/src/queue.rs (100%) rename {libk-util => libk/libk-util}/src/ring.rs (100%) rename {libk-util => libk/libk-util}/src/sync/fence.rs (100%) rename {libk-util => libk/libk-util}/src/sync/mod.rs (100%) rename {libk-util => libk/libk-util}/src/sync/spin_rwlock.rs (100%) rename {libk-util => libk/libk-util}/src/sync/spinlock.rs (100%) rename {libk-util => libk/libk-util}/src/waker.rs (100%) delete mode 100644 libk/src/api.rs delete mode 100644 libk/src/sync/mod.rs diff --git a/Cargo.toml b/Cargo.toml index c2a84d31..4941a6c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } device-api = { path = "lib/device-api", features = ["derive"] } libk = { path = "libk" } -libk-util = { path = "libk-util" } -libk-mm = { path = "libk-mm" } -libk-thread = { path = "libk-thread" } -libk-device = { path = "libk-device" } +libk-util = { path = "libk/libk-util" } +libk-mm = { path = "libk/libk-mm" } +libk-thread = { path = "libk/libk-thread" } +libk-device = { path = "libk/libk-device" } memtables = { path = "lib/memtables" } vmalloc = { path = "lib/vmalloc" } device-api-macros = { path = "lib/device-api/macros" } diff --git a/arch/aarch64/Cargo.toml b/arch/aarch64/Cargo.toml index 882600f0..839d6ff0 100644 --- a/arch/aarch64/Cargo.toml +++ b/arch/aarch64/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../interface" } -libk-mm-interface = { path = "../../libk-mm/interface" } +libk-mm-interface = { path = "../../libk/libk-mm/interface" } memtables = { path = "../../lib/memtables" } device-api = { path = "../../lib/device-api", features = ["derive"] } diff --git a/arch/x86_64/Cargo.toml b/arch/x86_64/Cargo.toml index f513dc5d..3971debc 100644 --- a/arch/x86_64/Cargo.toml +++ b/arch/x86_64/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../interface" } -libk-mm-interface = { path = "../../libk-mm/interface" } +libk-mm-interface = { path = "../../libk/libk-mm/interface" } memtables = { path = "../../lib/memtables" } device-api = { path = "../../lib/device-api", features = ["derive"] } diff --git a/driver/block/ahci/Cargo.toml b/driver/block/ahci/Cargo.toml index b962c397..45ae22f2 100644 --- a/driver/block/ahci/Cargo.toml +++ b/driver/block/ahci/Cargo.toml @@ -8,9 +8,9 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } -libk-mm = { path = "../../../libk-mm" } -libk-util = { path = "../../../libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } +libk-thread = { path = "../../../libk/libk-thread" } +libk-util = { path = "../../../libk/libk-util" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index e06dae56..8f1aa9ad 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -13,8 +13,8 @@ use device_api::{ }; use error::AhciError; use kernel_fs::devfs; -use libk::runtime; use libk_mm::{address::AsPhysicalAddress, device::DeviceMemoryIo, PageBox}; +use libk_thread::runtime; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use port::AhciPort; use regs::{PortRegs, Regs}; diff --git a/driver/block/core/Cargo.toml b/driver/block/core/Cargo.toml index e9c521d3..ade50366 100644 --- a/driver/block/core/Cargo.toml +++ b/driver/block/core/Cargo.toml @@ -8,9 +8,8 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } -libk-util = { path = "../../../libk-util" } -libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk/libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml index e3eb4638..b6c60cd4 100644 --- a/driver/block/nvme/Cargo.toml +++ b/driver/block/nvme/Cargo.toml @@ -8,9 +8,8 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } -libk-util = { path = "../../../libk-util" } -libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk/libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/bus/pci/Cargo.toml b/driver/bus/pci/Cargo.toml index 0f3231b3..ed51f994 100644 --- a/driver/bus/pci/Cargo.toml +++ b/driver/bus/pci/Cargo.toml @@ -9,9 +9,9 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../../../lib/device-api", features = ["derive"] } -libk = { path = "../../../libk" } -libk-mm = { path = "../../../libk-mm" } -libk-util = { path = "../../../libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } +libk-device = { path = "../../../libk/libk-device" } +libk-util = { path = "../../../libk/libk-util" } log = "0.4.20" bitflags = "2.3.3" diff --git a/driver/bus/pci/src/device.rs b/driver/bus/pci/src/device.rs index 748da52c..8afe3c85 100644 --- a/driver/bus/pci/src/device.rs +++ b/driver/bus/pci/src/device.rs @@ -5,7 +5,7 @@ use device_api::{ interrupt::{InterruptAffinity, InterruptHandler, IrqOptions, MsiInfo}, Device, }; -use libk::device::{message_interrupt_controller, register_global_interrupt}; +use libk_device::{message_interrupt_controller, register_global_interrupt}; use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; use yggdrasil_abi::error::Error; diff --git a/driver/fs/kernel-fs/Cargo.toml b/driver/fs/kernel-fs/Cargo.toml index 3428724d..e6c25931 100644 --- a/driver/fs/kernel-fs/Cargo.toml +++ b/driver/fs/kernel-fs/Cargo.toml @@ -9,7 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "../../../lib/vfs" } -libk-util = { path = "../../../libk-util" } +libk-util = { path = "../../../libk/libk-util" } ygg_driver_block = { path = "../../block/core" } diff --git a/driver/fs/memfs/Cargo.toml b/driver/fs/memfs/Cargo.toml index 4dd636cc..71f5250a 100644 --- a/driver/fs/memfs/Cargo.toml +++ b/driver/fs/memfs/Cargo.toml @@ -8,8 +8,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk = { path = "../../../libk" } -libk-util = { path = "../../../libk-util" } +libk-util = { path = "../../../libk/libk-util" } vfs = { path = "../../../lib/vfs" } static_assertions = "1.1.0" diff --git a/driver/net/core/Cargo.toml b/driver/net/core/Cargo.toml index eb7ab6c0..bfe925c6 100644 --- a/driver/net/core/Cargo.toml +++ b/driver/net/core/Cargo.toml @@ -7,9 +7,10 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["serde_kernel", "bytemuck"] } -libk = { path = "../../../libk" } -libk-mm = { path = "../../../libk-mm" } -libk-util = { path = "../../../libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } +libk-util = { path = "../../../libk/libk-util" } +libk-thread = { path = "../../../libk/libk-thread" } +libk-device = { path = "../../../libk/libk-device" } vfs = { path = "../../../lib/vfs" } kernel-fs = { path = "../../fs/kernel-fs" } diff --git a/driver/net/core/src/l3/arp.rs b/driver/net/core/src/l3/arp.rs index 18cdc3b7..cf4dbb6f 100644 --- a/driver/net/core/src/l3/arp.rs +++ b/driver/net/core/src/l3/arp.rs @@ -7,7 +7,7 @@ use core::{ }; use alloc::{boxed::Box, collections::BTreeMap}; -use libk::runtime; +use libk_thread::runtime; use libk_util::{sync::spin_rwlock::IrqSafeRwLock, waker::QueueWaker}; use yggdrasil_abi::{ error::Error, diff --git a/driver/net/core/src/lib.rs b/driver/net/core/src/lib.rs index 91fd89e4..f3d50deb 100644 --- a/driver/net/core/src/lib.rs +++ b/driver/net/core/src/lib.rs @@ -10,8 +10,8 @@ use alloc::sync::Arc; use bytemuck::Pod; use ethernet::L2Packet; use l3::L3Packet; -use libk::runtime; use libk_mm::PageBox; +use libk_thread::runtime; use libk_util::queue::UnboundedMpmcQueue; use yggdrasil_abi::{error::Error, net::protocols::EthernetFrame}; diff --git a/driver/net/core/src/socket.rs b/driver/net/core/src/socket.rs index 07aeaee1..23416657 100644 --- a/driver/net/core/src/socket.rs +++ b/driver/net/core/src/socket.rs @@ -7,12 +7,12 @@ use core::{ }; use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; -use libk::{ +use libk_device::monotonic_timestamp; +use libk_mm::PageBox; +use libk_thread::{ block, - device::monotonic_timestamp, runtime::{run_with_timeout, FutureTimeout}, }; -use libk_mm::PageBox; use libk_util::{ queue::BoundedMpmcQueue, sync::{ diff --git a/driver/net/loopback/Cargo.toml b/driver/net/loopback/Cargo.toml index 4dce1a70..d7e37a79 100644 --- a/driver/net/loopback/Cargo.toml +++ b/driver/net/loopback/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk-util = { path = "../../../libk-util" } -libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk/libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } ygg_driver_net_core = { path = "../../net/core" } diff --git a/driver/virtio/core/Cargo.toml b/driver/virtio/core/Cargo.toml index b61400da..68d512f3 100644 --- a/driver/virtio/core/Cargo.toml +++ b/driver/virtio/core/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk-mm = { path = "../../../libk-mm" } +libk-mm = { path = "../../../libk/libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } ygg_driver_pci = { path = "../../bus/pci", optional = true } diff --git a/driver/virtio/net/Cargo.toml b/driver/virtio/net/Cargo.toml index 0ae88914..fcb6869b 100644 --- a/driver/virtio/net/Cargo.toml +++ b/driver/virtio/net/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -libk-util = { path = "../../../libk-util" } -libk-mm = { path = "../../../libk-mm" } +libk-util = { path = "../../../libk/libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } ygg_driver_virtio_core = { path = "../core" } diff --git a/lib/device-tree/Cargo.toml b/lib/device-tree/Cargo.toml index 3363fbb9..0d343613 100644 --- a/lib/device-tree/Cargo.toml +++ b/lib/device-tree/Cargo.toml @@ -8,8 +8,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } device-api = { path = "../device-api", features = ["derive"] } -libk = { path = "../../libk" } -libk-mm = { path = "../../libk-mm" } +libk-mm = { path = "../../libk/libk-mm" } fdt-rs = { version = "0.4.3", default-features = false } log = "0.4.20" diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index bef0188b..066d2505 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -8,10 +8,9 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git", features = ["alloc"] } -libk = { path = "../../libk" } -libk-mm = { path = "../../libk-mm" } -libk-util = { path = "../../libk-util" } -libk-thread = { path = "../../libk-thread" } +libk-mm = { path = "../../libk/libk-mm" } +libk-util = { path = "../../libk/libk-util" } +libk-thread = { path = "../../libk/libk-thread" } ygg_driver_block = { path = "../../driver/block/core" } diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index 71ac68dc..28b19f49 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -11,7 +11,7 @@ use alloc::{ sync::Arc, }; use futures_util::{task::AtomicWaker, Future}; -use libk::{block, sync::mutex::Mutex}; +use libk_thread::{block, sync::Mutex}; use libk_util::sync::{IrqSafeSpinlock, LockMethod}; use yggdrasil_abi::{error::Error, io::MessageDestination}; diff --git a/lib/vfs/src/file/pipe.rs b/lib/vfs/src/file/pipe.rs index 1c4da47f..de9190d3 100644 --- a/lib/vfs/src/file/pipe.rs +++ b/lib/vfs/src/file/pipe.rs @@ -6,7 +6,7 @@ use core::{ use alloc::{sync::Arc, vec, vec::Vec}; use futures_util::{task::AtomicWaker, Future}; -use libk::block; +use libk_thread::block; use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::error::Error; diff --git a/lib/vfs/src/poll.rs b/lib/vfs/src/poll.rs index 30b22846..d575c8d8 100644 --- a/lib/vfs/src/poll.rs +++ b/lib/vfs/src/poll.rs @@ -7,7 +7,7 @@ use core::{ use alloc::collections::BTreeMap; use futures_util::{future::BoxFuture, FutureExt}; -use libk::{runtime, sync::mutex::Mutex}; +use libk_thread::{runtime, sync::Mutex}; use libk_util::sync::LockMethod; use yggdrasil_abi::{error::Error, io::RawFd}; diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 550b2e10..97bf1a42 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -8,7 +8,7 @@ use core::{ }; use alloc::{boxed::Box, sync::Arc}; -use libk::{block, signal_process_group}; +use libk_thread::{block, signal_process_group}; use libk_util::{ ring::{LossyRingQueue, RingBuffer}, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, diff --git a/lib/vfs/src/timer.rs b/lib/vfs/src/timer.rs index b44ae92d..b59e8d08 100644 --- a/lib/vfs/src/timer.rs +++ b/lib/vfs/src/timer.rs @@ -7,9 +7,9 @@ use core::{ use alloc::boxed::Box; use futures_util::FutureExt; -use libk::{ +use libk_thread::{ runtime::{self, SleepFuture}, - sync::mutex::Mutex, + sync::Mutex, }; use libk_util::sync::LockMethod; use yggdrasil_abi::error::Error; diff --git a/libk/Cargo.toml b/libk/Cargo.toml index bb3b597d..d21f44a3 100644 --- a/libk/Cargo.toml +++ b/libk/Cargo.toml @@ -7,10 +7,10 @@ authors = ["Mark Poliakov "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libk-mm = { path = "../libk-mm" } -libk-util = { path = "../libk-util" } -libk-thread = { path = "../libk-thread" } -libk-device = { path = "../libk-device" } +libk-mm = { path = "libk-mm" } +libk-util = { path = "libk-util" } +libk-thread = { path = "libk-thread" } +libk-device = { path = "libk-device" } kernel-arch = { path = "../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/libk-device/Cargo.toml b/libk/libk-device/Cargo.toml similarity index 73% rename from libk-device/Cargo.toml rename to libk/libk-device/Cargo.toml index b3c7b74c..88be578e 100644 --- a/libk-device/Cargo.toml +++ b/libk/libk-device/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] libk-util = { path = "../libk-util" } -kernel-arch = { path = "../arch" } +kernel-arch = { path = "../../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -device-api = { path = "../lib/device-api", features = ["derive"] } +device-api = { path = "../../lib/device-api", features = ["derive"] } diff --git a/libk-device/src/lib.rs b/libk/libk-device/src/lib.rs similarity index 100% rename from libk-device/src/lib.rs rename to libk/libk-device/src/lib.rs diff --git a/libk-mm/Cargo.toml b/libk/libk-mm/Cargo.toml similarity index 81% rename from libk-mm/Cargo.toml rename to libk/libk-mm/Cargo.toml index b0f9a1c4..5536289a 100644 --- a/libk-mm/Cargo.toml +++ b/libk/libk-mm/Cargo.toml @@ -7,9 +7,9 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-arch = { path = "../arch" } +kernel-arch = { path = "../../arch" } libk-util = { path = "../libk-util" } libk-mm-interface = { path = "interface" } -vmalloc = { path = "../lib/vmalloc" } +vmalloc = { path = "../../lib/vmalloc" } log = "0.4.20" diff --git a/libk-mm/interface/Cargo.toml b/libk/libk-mm/interface/Cargo.toml similarity index 82% rename from libk-mm/interface/Cargo.toml rename to libk/libk-mm/interface/Cargo.toml index 2efbb6a6..9eb6d23d 100644 --- a/libk-mm/interface/Cargo.toml +++ b/libk/libk-mm/interface/Cargo.toml @@ -8,6 +8,6 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-arch-interface = { path = "../../arch/interface" } +kernel-arch-interface = { path = "../../../arch/interface" } bitflags = "2.3.3" diff --git a/libk-mm/interface/src/address.rs b/libk/libk-mm/interface/src/address.rs similarity index 100% rename from libk-mm/interface/src/address.rs rename to libk/libk-mm/interface/src/address.rs diff --git a/libk-mm/interface/src/lib.rs b/libk/libk-mm/interface/src/lib.rs similarity index 100% rename from libk-mm/interface/src/lib.rs rename to libk/libk-mm/interface/src/lib.rs diff --git a/libk-mm/interface/src/pointer.rs b/libk/libk-mm/interface/src/pointer.rs similarity index 100% rename from libk-mm/interface/src/pointer.rs rename to libk/libk-mm/interface/src/pointer.rs diff --git a/libk-mm/interface/src/process.rs b/libk/libk-mm/interface/src/process.rs similarity index 100% rename from libk-mm/interface/src/process.rs rename to libk/libk-mm/interface/src/process.rs diff --git a/libk-mm/interface/src/table.rs b/libk/libk-mm/interface/src/table.rs similarity index 100% rename from libk-mm/interface/src/table.rs rename to libk/libk-mm/interface/src/table.rs diff --git a/libk-mm/src/address.rs b/libk/libk-mm/src/address.rs similarity index 100% rename from libk-mm/src/address.rs rename to libk/libk-mm/src/address.rs diff --git a/libk-mm/src/device.rs b/libk/libk-mm/src/device.rs similarity index 100% rename from libk-mm/src/device.rs rename to libk/libk-mm/src/device.rs diff --git a/libk-mm/src/lib.rs b/libk/libk-mm/src/lib.rs similarity index 100% rename from libk-mm/src/lib.rs rename to libk/libk-mm/src/lib.rs diff --git a/libk-mm/src/phys/manager.rs b/libk/libk-mm/src/phys/manager.rs similarity index 100% rename from libk-mm/src/phys/manager.rs rename to libk/libk-mm/src/phys/manager.rs diff --git a/libk-mm/src/phys/mod.rs b/libk/libk-mm/src/phys/mod.rs similarity index 100% rename from libk-mm/src/phys/mod.rs rename to libk/libk-mm/src/phys/mod.rs diff --git a/libk-mm/src/phys/reserved.rs b/libk/libk-mm/src/phys/reserved.rs similarity index 100% rename from libk-mm/src/phys/reserved.rs rename to libk/libk-mm/src/phys/reserved.rs diff --git a/libk-mm/src/pointer.rs b/libk/libk-mm/src/pointer.rs similarity index 100% rename from libk-mm/src/pointer.rs rename to libk/libk-mm/src/pointer.rs diff --git a/libk-mm/src/process.rs b/libk/libk-mm/src/process.rs similarity index 100% rename from libk-mm/src/process.rs rename to libk/libk-mm/src/process.rs diff --git a/libk-thread/Cargo.toml b/libk/libk-thread/Cargo.toml similarity index 95% rename from libk-thread/Cargo.toml rename to libk/libk-thread/Cargo.toml index 83afaf16..03634422 100644 --- a/libk-thread/Cargo.toml +++ b/libk/libk-thread/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" libk-util = { path = "../libk-util" } libk-mm = { path = "../libk-mm" } libk-device = { path = "../libk-device" } -kernel-arch = { path = "../arch" } +kernel-arch = { path = "../../arch" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/libk-thread/src/binary/elf.rs b/libk/libk-thread/src/binary/elf.rs similarity index 100% rename from libk-thread/src/binary/elf.rs rename to libk/libk-thread/src/binary/elf.rs diff --git a/libk-thread/src/binary/mod.rs b/libk/libk-thread/src/binary/mod.rs similarity index 100% rename from libk-thread/src/binary/mod.rs rename to libk/libk-thread/src/binary/mod.rs diff --git a/libk-thread/src/futex.rs b/libk/libk-thread/src/futex.rs similarity index 100% rename from libk-thread/src/futex.rs rename to libk/libk-thread/src/futex.rs diff --git a/libk-thread/src/lib.rs b/libk/libk-thread/src/lib.rs similarity index 61% rename from libk-thread/src/lib.rs rename to libk/libk-thread/src/lib.rs index 9dfec935..ec92f2e8 100644 --- a/libk-thread/src/lib.rs +++ b/libk/libk-thread/src/lib.rs @@ -11,9 +11,18 @@ extern crate alloc; +use api::__signal_process_group; use kernel_arch::KernelTableManagerImpl; use libk_mm::phys::GlobalPhysicalAllocator; +pub(crate) mod api { + use yggdrasil_abi::process::Signal; + + extern "Rust" { + pub fn __signal_process_group(group_id: u32, signal: Signal); + } +} + #[macro_use] pub mod runtime; @@ -22,6 +31,7 @@ pub mod futex; pub mod mem; pub mod process; pub mod sched; +pub mod sync; pub mod thread; pub mod types; @@ -29,3 +39,9 @@ pub type TaskContextImpl = kernel_arch::TaskContextImpl; pub use types::{AtomicThreadState, ThreadAffinity, ThreadId, ThreadState}; + +use yggdrasil_abi::process::Signal; + +pub fn signal_process_group(group_id: u32, signal: Signal) { + unsafe { __signal_process_group(group_id, signal) } +} diff --git a/libk-thread/src/mem.rs b/libk/libk-thread/src/mem.rs similarity index 100% rename from libk-thread/src/mem.rs rename to libk/libk-thread/src/mem.rs diff --git a/libk-thread/src/process.rs b/libk/libk-thread/src/process.rs similarity index 100% rename from libk-thread/src/process.rs rename to libk/libk-thread/src/process.rs diff --git a/libk-thread/src/runtime/executor.rs b/libk/libk-thread/src/runtime/executor.rs similarity index 100% rename from libk-thread/src/runtime/executor.rs rename to libk/libk-thread/src/runtime/executor.rs diff --git a/libk-thread/src/runtime/macros.rs b/libk/libk-thread/src/runtime/macros.rs similarity index 100% rename from libk-thread/src/runtime/macros.rs rename to libk/libk-thread/src/runtime/macros.rs diff --git a/libk-thread/src/runtime/mod.rs b/libk/libk-thread/src/runtime/mod.rs similarity index 100% rename from libk-thread/src/runtime/mod.rs rename to libk/libk-thread/src/runtime/mod.rs diff --git a/libk-thread/src/runtime/task.rs b/libk/libk-thread/src/runtime/task.rs similarity index 100% rename from libk-thread/src/runtime/task.rs rename to libk/libk-thread/src/runtime/task.rs diff --git a/libk-thread/src/runtime/task_queue.rs b/libk/libk-thread/src/runtime/task_queue.rs similarity index 100% rename from libk-thread/src/runtime/task_queue.rs rename to libk/libk-thread/src/runtime/task_queue.rs diff --git a/libk-thread/src/runtime/timer.rs b/libk/libk-thread/src/runtime/timer.rs similarity index 100% rename from libk-thread/src/runtime/timer.rs rename to libk/libk-thread/src/runtime/timer.rs diff --git a/libk-thread/src/sched.rs b/libk/libk-thread/src/sched.rs similarity index 100% rename from libk-thread/src/sched.rs rename to libk/libk-thread/src/sched.rs diff --git a/libk/src/sync/mutex.rs b/libk/libk-thread/src/sync.rs similarity index 98% rename from libk/src/sync/mutex.rs rename to libk/libk-thread/src/sync.rs index c0ec9003..dc85dec3 100644 --- a/libk/src/sync/mutex.rs +++ b/libk/libk-thread/src/sync.rs @@ -7,10 +7,11 @@ use core::{ use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; use kernel_arch::task::Scheduler; -use libk_thread::{sched::CpuQueue, thread::Thread}; use libk_util::sync::LockMethod; use yggdrasil_abi::error::Error; +use crate::{sched::CpuQueue, thread::Thread}; + struct ThreadedMutexInner { queue: ArrayQueue>, lock: AtomicU32, diff --git a/libk-thread/src/thread.rs b/libk/libk-thread/src/thread.rs similarity index 100% rename from libk-thread/src/thread.rs rename to libk/libk-thread/src/thread.rs diff --git a/libk-thread/src/types.rs b/libk/libk-thread/src/types.rs similarity index 100% rename from libk-thread/src/types.rs rename to libk/libk-thread/src/types.rs diff --git a/libk-util/Cargo.toml b/libk/libk-util/Cargo.toml similarity index 92% rename from libk-util/Cargo.toml rename to libk/libk-util/Cargo.toml index fe3e89b2..7aeb87e3 100644 --- a/libk-util/Cargo.toml +++ b/libk/libk-util/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } -kernel-arch = { path = "../arch" } +kernel-arch = { path = "../../arch" } log = "0.4.20" crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } diff --git a/libk-util/src/event.rs b/libk/libk-util/src/event.rs similarity index 100% rename from libk-util/src/event.rs rename to libk/libk-util/src/event.rs diff --git a/libk-util/src/io.rs b/libk/libk-util/src/io.rs similarity index 100% rename from libk-util/src/io.rs rename to libk/libk-util/src/io.rs diff --git a/libk-util/src/lib.rs b/libk/libk-util/src/lib.rs similarity index 100% rename from libk-util/src/lib.rs rename to libk/libk-util/src/lib.rs diff --git a/libk-util/src/queue.rs b/libk/libk-util/src/queue.rs similarity index 100% rename from libk-util/src/queue.rs rename to libk/libk-util/src/queue.rs diff --git a/libk-util/src/ring.rs b/libk/libk-util/src/ring.rs similarity index 100% rename from libk-util/src/ring.rs rename to libk/libk-util/src/ring.rs diff --git a/libk-util/src/sync/fence.rs b/libk/libk-util/src/sync/fence.rs similarity index 100% rename from libk-util/src/sync/fence.rs rename to libk/libk-util/src/sync/fence.rs diff --git a/libk-util/src/sync/mod.rs b/libk/libk-util/src/sync/mod.rs similarity index 100% rename from libk-util/src/sync/mod.rs rename to libk/libk-util/src/sync/mod.rs diff --git a/libk-util/src/sync/spin_rwlock.rs b/libk/libk-util/src/sync/spin_rwlock.rs similarity index 100% rename from libk-util/src/sync/spin_rwlock.rs rename to libk/libk-util/src/sync/spin_rwlock.rs diff --git a/libk-util/src/sync/spinlock.rs b/libk/libk-util/src/sync/spinlock.rs similarity index 100% rename from libk-util/src/sync/spinlock.rs rename to libk/libk-util/src/sync/spinlock.rs diff --git a/libk-util/src/waker.rs b/libk/libk-util/src/waker.rs similarity index 100% rename from libk-util/src/waker.rs rename to libk/libk-util/src/waker.rs diff --git a/libk/src/api.rs b/libk/src/api.rs deleted file mode 100644 index 254032b9..00000000 --- a/libk/src/api.rs +++ /dev/null @@ -1,5 +0,0 @@ -use yggdrasil_abi::process::Signal; - -extern "Rust" { - pub fn __signal_process_group(group_id: u32, signal: Signal); -} diff --git a/libk/src/lib.rs b/libk/src/lib.rs index 52ab1455..4e39d17b 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -18,16 +18,12 @@ use arch::Cpu; use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl}; -use yggdrasil_abi::process::Signal; extern crate alloc; pub use libk_thread::{block, runtime}; -pub(crate) mod api; - pub mod arch; -pub mod sync; pub mod thread; pub mod device { @@ -45,11 +41,6 @@ pub fn cpu_index() -> u32 { Cpu::try_local().as_deref().map(CpuImpl::id).unwrap_or(0) } -#[inline] -pub fn signal_process_group(group_id: u32, signal: Signal) { - unsafe { api::__signal_process_group(group_id, signal) } -} - #[repr(C)] pub struct AlignedTo { pub align: [Align; 0], diff --git a/libk/src/sync/mod.rs b/libk/src/sync/mod.rs deleted file mode 100644 index 5e260883..00000000 --- a/libk/src/sync/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -//! Synchronization primitives -pub mod mutex; From c4be544a9a71ecf4d5d99f950b17a359309a206c Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 15 Feb 2024 11:32:53 +0200 Subject: [PATCH 193/211] aarch64: better handling of user exceptions --- libk/src/lib.rs | 1 - libk/src/thread.rs | 8 -------- src/arch/aarch64/exception.rs | 8 +++----- src/task/mod.rs | 9 +++------ 4 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 libk/src/thread.rs diff --git a/libk/src/lib.rs b/libk/src/lib.rs index 4e39d17b..c035bb55 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -24,7 +24,6 @@ extern crate alloc; pub use libk_thread::{block, runtime}; pub mod arch; -pub mod thread; pub mod device { pub use libk_device::*; diff --git a/libk/src/thread.rs b/libk/src/thread.rs deleted file mode 100644 index 2d66b87f..00000000 --- a/libk/src/thread.rs +++ /dev/null @@ -1,8 +0,0 @@ -use kernel_arch::KernelTableManagerImpl; -use libk_mm::phys::GlobalPhysicalAllocator; - -pub use kernel_arch::task::{TaskContext, TaskFrame, Termination}; - -pub type TaskContextImpl = - kernel_arch::TaskContextImpl; -pub trait ForkFrame = kernel_arch::task::ForkFrame; diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index eb8fe2a6..1c5c8ba6 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -194,6 +194,7 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { thread.raise_signal(Signal::Aborted); } _ => { + let thread = Thread::current(); let iss = esr_el1 & 0x1FFFFFF; if ec == 0b100100 { // Data abort from lower level @@ -205,14 +206,11 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { ELR_EL1.get(), FAR_EL1.get() ); - - thread.raise_signal(Signal::MemoryAccessViolation); - return; } - dump_irrecoverable_exception(frame, ec, iss); - panic!("Irrecoverable exception"); + thread.raise_signal(Signal::MemoryAccessViolation); + return; } } } diff --git a/src/task/mod.rs b/src/task/mod.rs index 6abc6b9a..ccce3e8b 100644 --- a/src/task/mod.rs +++ b/src/task/mod.rs @@ -5,17 +5,14 @@ use abi::error::Error; use alloc::{string::String, vec::Vec}; use kernel_arch::{ - task::{Scheduler, TaskContext}, + task::{Scheduler, TaskContext, Termination}, Architecture, ArchitectureImpl, }; -use libk::{ - arch::Cpu, - runtime, - thread::{TaskContextImpl, Termination}, -}; +use libk::{arch::Cpu, runtime}; use libk_thread::{ sched::{init_queues, CpuQueue}, thread::Thread, + TaskContextImpl, }; use libk_util::sync::SpinFence; From 012eb46cb97322c420946f52f55a65eee69603c0 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 26 Feb 2024 23:04:51 +0200 Subject: [PATCH 194/211] bus/usb: basic xHCI implementation --- Cargo.toml | 5 +- arch/interface/src/lib.rs | 1 + arch/x86_64/src/lib.rs | 12 +- driver/block/ahci/src/lib.rs | 12 +- driver/block/nvme/Cargo.toml | 1 + driver/block/nvme/src/drive.rs | 2 +- driver/block/nvme/src/lib.rs | 2 +- driver/bus/usb/Cargo.toml | 18 + driver/bus/usb/src/bus.rs | 51 ++ driver/bus/usb/src/class_driver/mod.rs | 203 ++++++ driver/bus/usb/src/communication.rs | 92 +++ driver/bus/usb/src/descriptor.rs | 146 ++++ driver/bus/usb/src/device.rs | 191 +++++ driver/bus/usb/src/info.rs | 84 +++ driver/bus/usb/src/lib.rs | 19 + driver/bus/usb/src/pipe/control.rs | 323 +++++++++ driver/bus/usb/src/pipe/interrupt.rs | 32 + driver/bus/usb/src/pipe/mod.rs | 12 + driver/input/Cargo.toml | 11 + .../input.rs => driver/input/src/lib.rs | 13 +- driver/usb/xhci/Cargo.toml | 22 + driver/usb/xhci/src/lib.rs | 484 +++++++++++++ driver/usb/xhci/src/pipe.rs | 88 +++ driver/usb/xhci/src/regs.rs | 240 +++++++ driver/usb/xhci/src/ring.rs | 656 ++++++++++++++++++ lib/vfs/src/lib.rs | 9 +- libk/libk-mm/src/lib.rs | 60 +- libk/libk-thread/src/lib.rs | 13 +- libk/libk-thread/src/sync.rs | 84 ++- libk/src/lib.rs | 16 +- src/arch/x86_64/apic/ioapic.rs | 8 +- src/arch/x86_64/apic/local.rs | 4 +- src/arch/x86_64/mod.rs | 17 +- src/arch/x86_64/peripherals/ps2/mod.rs | 11 +- src/device/mod.rs | 1 - src/init.rs | 5 +- tools/gentables/src/main.rs | 2 - 37 files changed, 2881 insertions(+), 69 deletions(-) create mode 100644 driver/bus/usb/Cargo.toml create mode 100644 driver/bus/usb/src/bus.rs create mode 100644 driver/bus/usb/src/class_driver/mod.rs create mode 100644 driver/bus/usb/src/communication.rs create mode 100644 driver/bus/usb/src/descriptor.rs create mode 100644 driver/bus/usb/src/device.rs create mode 100644 driver/bus/usb/src/info.rs create mode 100644 driver/bus/usb/src/lib.rs create mode 100644 driver/bus/usb/src/pipe/control.rs create mode 100644 driver/bus/usb/src/pipe/interrupt.rs create mode 100644 driver/bus/usb/src/pipe/mod.rs create mode 100644 driver/input/Cargo.toml rename src/device/input.rs => driver/input/src/lib.rs (93%) create mode 100644 driver/usb/xhci/Cargo.toml create mode 100644 driver/usb/xhci/src/lib.rs create mode 100644 driver/usb/xhci/src/pipe.rs create mode 100644 driver/usb/xhci/src/regs.rs create mode 100644 driver/usb/xhci/src/ring.rs diff --git a/Cargo.toml b/Cargo.toml index 4941a6c0..08935458 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,11 +26,14 @@ kernel-arch = { path = "arch" } # Drivers ygg_driver_pci = { path = "driver/bus/pci" } +ygg_driver_usb = { path = "driver/bus/usb" } ygg_driver_block = { path = "driver/block/core" } ygg_driver_net_core = { path = "driver/net/core" } ygg_driver_net_loopback = { path = "driver/net/loopback" } ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] } ygg_driver_ahci = { path = "driver/block/ahci" } +ygg_driver_usb_xhci = { path = "driver/usb/xhci" } +ygg_driver_input = { path = "driver/input" } kernel-fs = { path = "driver/fs/kernel-fs" } memfs = { path = "driver/fs/memfs" } @@ -64,8 +67,6 @@ yboot-proto = { git = "https://git.alnyan.me/yggdrasil/yboot-proto.git" } aml = { git = "https://github.com/alnyan/acpi.git", branch = "acpi-system" } acpi_lib = { git = "https://github.com/alnyan/acpi.git", package = "acpi", branch = "acpi-system" } acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } -# TODO currently only supported here -xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } ygg_driver_nvme = { path = "driver/block/nvme" } kernel-arch-x86_64 = { path = "arch/x86_64" } diff --git a/arch/interface/src/lib.rs b/arch/interface/src/lib.rs index 2498437f..2815fed1 100644 --- a/arch/interface/src/lib.rs +++ b/arch/interface/src/lib.rs @@ -29,6 +29,7 @@ pub trait Architecture: Sized { fn idle_task() -> extern "C" fn(usize) -> !; fn cpu_count() -> usize; + fn cpu_index() -> u32; // Interrupt management fn interrupt_mask() -> bool; diff --git a/arch/x86_64/src/lib.rs b/arch/x86_64/src/lib.rs index 4f3a6289..86158492 100644 --- a/arch/x86_64/src/lib.rs +++ b/arch/x86_64/src/lib.rs @@ -1,5 +1,11 @@ #![no_std] -#![feature(effects, strict_provenance, asm_const, naked_functions)] +#![feature( + effects, + strict_provenance, + asm_const, + naked_functions, + trait_upcasting +)] extern crate alloc; @@ -124,6 +130,10 @@ impl Architecture for ArchitectureImpl { CPU_COUNT.load(Ordering::Acquire) } + fn cpu_index() -> u32 { + CpuImpl::::local().id() + } + fn interrupt_mask() -> bool { let mut flags: u64; unsafe { diff --git a/driver/block/ahci/src/lib.rs b/driver/block/ahci/src/lib.rs index 8f1aa9ad..b3f74369 100644 --- a/driver/block/ahci/src/lib.rs +++ b/driver/block/ahci/src/lib.rs @@ -182,12 +182,14 @@ impl InterruptHandler for AhciController { let is = regs.IS.get(); if is != 0 { - // Clear global interrupt status - regs.IS.set(u32::MAX); + if let Some(ports) = self.ports.try_get() { + // Clear global interrupt status + regs.IS.set(u32::MAX); - for &port in self.ports.get() { - if is & (1 << port.index) != 0 { - port.handle_pending_interrupts(); + for &port in ports { + if is & (1 << port.index) != 0 { + port.handle_pending_interrupts(); + } } } } diff --git a/driver/block/nvme/Cargo.toml b/driver/block/nvme/Cargo.toml index b6c60cd4..611d52a3 100644 --- a/driver/block/nvme/Cargo.toml +++ b/driver/block/nvme/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } libk-util = { path = "../../../libk/libk-util" } +libk-thread = { path = "../../../libk/libk-thread" } libk-mm = { path = "../../../libk/libk-mm" } device-api = { path = "../../../lib/device-api", features = ["derive"] } vfs = { path = "../../../lib/vfs" } diff --git a/driver/block/nvme/src/drive.rs b/driver/block/nvme/src/drive.rs index c353f2b5..49bb6443 100644 --- a/driver/block/nvme/src/drive.rs +++ b/driver/block/nvme/src/drive.rs @@ -2,8 +2,8 @@ use core::task::Poll; use alloc::{boxed::Box, format}; use kernel_fs::devfs; -use libk::cpu_index; use libk_mm::address::AsPhysicalAddress; +use libk_thread::cpu_index; use libk_util::waker::QueueWaker; use ygg_driver_block::{ probe_partitions, IoOperation, IoRequest, IoSubmissionId, NgBlockDevice, NgBlockDeviceWrapper, diff --git a/driver/block/nvme/src/lib.rs b/driver/block/nvme/src/lib.rs index 08b1e231..56467f78 100644 --- a/driver/block/nvme/src/lib.rs +++ b/driver/block/nvme/src/lib.rs @@ -17,11 +17,11 @@ use device_api::{ Device, }; use drive::NvmeDrive; -use libk::{cpu_count, cpu_index, runtime}; use libk_mm::{ address::{IntoRaw, PhysicalAddress}, device::DeviceMemoryIo, }; +use libk_thread::{cpu_count, cpu_index, runtime}; use libk_util::{ sync::{IrqGuard, IrqSafeSpinlock}, OneTimeInit, diff --git a/driver/bus/usb/Cargo.toml b/driver/bus/usb/Cargo.toml new file mode 100644 index 00000000..b6b70c3a --- /dev/null +++ b/driver/bus/usb/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ygg_driver_usb" +version = "0.1.0" +edition = "2021" +authors = ["Mark Poliakov "] + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } +ygg_driver_input = { path = "../../input" } + +libk-util = { path = "../../../libk/libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } +libk-thread = { path = "../../../libk/libk-thread" } + +log = "0.4.20" +bytemuck = { version = "1.14.0", features = ["derive"] } +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/driver/bus/usb/src/bus.rs b/driver/bus/usb/src/bus.rs new file mode 100644 index 00000000..e99b69ad --- /dev/null +++ b/driver/bus/usb/src/bus.rs @@ -0,0 +1,51 @@ +use core::sync::atomic::{AtomicU16, Ordering}; + +use alloc::{collections::BTreeMap, sync::Arc}; +use libk_util::{queue::UnboundedMpmcQueue, sync::spin_rwlock::IrqSafeRwLock}; + +use crate::{ + class_driver, + device::{UsbBusAddress, UsbDeviceAccess}, + UsbHostController, +}; + +pub struct UsbBusManager { + busses: IrqSafeRwLock>, + devices: IrqSafeRwLock>>, + + last_bus_address: AtomicU16, +} + +impl UsbBusManager { + pub fn register_bus(hc: &'static dyn UsbHostController) -> u16 { + let i = BUS_MANAGER.last_bus_address.fetch_add(1, Ordering::AcqRel); + BUS_MANAGER.busses.write().insert(i, hc); + i + } + + pub fn register_device(device: Arc) { + BUS_MANAGER + .devices + .write() + .insert(device.bus_address(), device.clone()); + + QUEUE.push_back(device); + } +} + +pub async fn bus_handler() { + loop { + let new_device = QUEUE.pop_front().await; + log::debug!("New device connected: {}", new_device.bus_address()); + + class_driver::spawn_driver(new_device).await.ok(); + } +} + +static BUS_MANAGER: UsbBusManager = UsbBusManager { + busses: IrqSafeRwLock::new(BTreeMap::new()), + devices: IrqSafeRwLock::new(BTreeMap::new()), + + last_bus_address: AtomicU16::new(0), +}; +static QUEUE: UnboundedMpmcQueue> = UnboundedMpmcQueue::new(); diff --git a/driver/bus/usb/src/class_driver/mod.rs b/driver/bus/usb/src/class_driver/mod.rs new file mode 100644 index 00000000..21823a05 --- /dev/null +++ b/driver/bus/usb/src/class_driver/mod.rs @@ -0,0 +1,203 @@ +use core::mem::MaybeUninit; + +use alloc::sync::Arc; +use libk_mm::PageBox; +use libk_thread::runtime; +use yggdrasil_abi::{ + error::Error, + io::{KeyboardKey, KeyboardKeyEvent}, +}; + +use crate::{ + device::UsbDeviceAccess, + info::{UsbDeviceClass, UsbDeviceProtocol}, + UsbDirection, +}; + +pub async fn spawn_driver(device: Arc) -> Result<(), Error> { + // TODO query all configurations? + let device_info = &device.info; + let config_info = device.query_configuration_info(0).await?; + + // Select device class/subclass/protocol based on device/interface + if config_info.interfaces.len() == 1 { + let if_info = &config_info.interfaces[0]; + + let class = if device_info.device_class == UsbDeviceClass::FromInterface { + if_info.interface_class + } else { + device_info.device_class + }; + let subclass = if device_info.device_subclass == 0 { + if_info.interface_subclass + } else { + device_info.device_subclass + }; + let protocol = if device_info.device_protocol == UsbDeviceProtocol::FromInterface { + if_info.interface_protocol + } else { + device_info.device_protocol + }; + + match (class, subclass, protocol) { + (UsbDeviceClass::Hid, 0x01, _) => runtime::spawn(keyboard_driver(device)), + (_, _, _) => Ok(()), + } + } else { + Ok(()) + } +} + +pub async fn keyboard_driver(device: Arc) -> Result<(), Error> { + const MODIFIER_MAP: &[KeyboardKey] = &[ + KeyboardKey::LControl, + KeyboardKey::LShift, + KeyboardKey::LAlt, + KeyboardKey::Unknown, + KeyboardKey::RControl, + KeyboardKey::RShift, + KeyboardKey::RAlt, + KeyboardKey::Unknown, + ]; + + #[derive(Default)] + struct KeyboardState { + state: [u64; 4], + mods: u8, + } + + impl KeyboardState { + pub fn new() -> Self { + Self::default() + } + + pub fn translate_key(k: u8) -> KeyboardKey { + match k { + 4..=29 => KeyboardKey::Char(k - 4 + b'a'), + 30..=38 => KeyboardKey::Char(k - 30 + b'1'), + 39 => KeyboardKey::Char(b'0'), + + 40 => KeyboardKey::Enter, + 41 => KeyboardKey::Escape, + 42 => KeyboardKey::Backspace, + 43 => KeyboardKey::Tab, + 44 => KeyboardKey::Char(b' '), + 45 => KeyboardKey::Char(b'-'), + 46 => KeyboardKey::Char(b'='), + 47 => KeyboardKey::Char(b'['), + 48 => KeyboardKey::Char(b']'), + 49 => KeyboardKey::Char(b'\\'), + 51 => KeyboardKey::Char(b';'), + 52 => KeyboardKey::Char(b'\''), + 53 => KeyboardKey::Char(b'`'), + 54 => KeyboardKey::Char(b','), + 55 => KeyboardKey::Char(b'.'), + 56 => KeyboardKey::Char(b'/'), + + 58..=69 => KeyboardKey::F(k - 58), + + _ => { + log::debug!("Unknown key: {}", k); + KeyboardKey::Unknown + } + } + } + + pub fn retain_modifiers( + &mut self, + m: u8, + events: &mut [MaybeUninit], + ) -> usize { + let mut count = 0; + let released = self.mods & !m; + for i in 0..8 { + if released & (1 << i) != 0 { + events[count].write(KeyboardKeyEvent::Released(MODIFIER_MAP[i])); + count += 1; + } + } + self.mods &= m; + count + } + + pub fn press_modifiers( + &mut self, + m: u8, + events: &mut [MaybeUninit], + ) -> usize { + let mut count = 0; + let pressed = m & !self.mods; + for i in 0..8 { + if pressed & (1 << i) != 0 { + events[count].write(KeyboardKeyEvent::Pressed(MODIFIER_MAP[i])); + count += 1; + } + } + self.mods = m; + count + } + + pub fn retain( + &mut self, + keys: &[u8], + events: &mut [MaybeUninit], + ) -> usize { + let mut count = 0; + for i in 1..256 { + if self.state[i / 64] & (1 << (i % 64)) != 0 { + if !keys.contains(&(i as u8)) { + events[count] + .write(KeyboardKeyEvent::Released(Self::translate_key(i as u8))); + self.state[i / 64] &= !(1 << (i % 64)); + count += 1; + } + } + } + count + } + + pub fn press( + &mut self, + keys: &[u8], + events: &mut [MaybeUninit], + ) -> usize { + let mut count = 0; + for &k in keys { + let index = (k as usize) / 64; + if self.state[index] & (1 << (k % 64)) == 0 { + self.state[index] |= 1 << (k % 64); + events[count].write(KeyboardKeyEvent::Pressed(Self::translate_key(k))); + count += 1; + } + } + count + } + } + + // TODO not sure whether to use boot protocol (easy) or GetReport + let config = device.select_configuration(|_| true).await?.unwrap(); + assert_eq!(config.endpoints.len(), 1); + + let pipe = device.open_interrupt_pipe(1, UsbDirection::In).await?; + + let mut buffer = PageBox::new_slice(0, 8)?; + let mut state = KeyboardState::new(); + let mut events = [MaybeUninit::uninit(); 16]; + + loop { + let mut event_count = 0; + + let data = pipe.listen(&mut buffer).await?; + + event_count += state.retain_modifiers(data[0], &mut events); + event_count += state.press_modifiers(data[0], &mut events[event_count..]); + event_count += state.retain(&data[2..], &mut events[event_count..]); + event_count += state.press(&data[2..], &mut events[event_count..]); + + let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) }; + + for &event in events { + ygg_driver_input::send_event(event); + } + } +} diff --git a/driver/bus/usb/src/communication.rs b/driver/bus/usb/src/communication.rs new file mode 100644 index 00000000..849e7819 --- /dev/null +++ b/driver/bus/usb/src/communication.rs @@ -0,0 +1,92 @@ +use core::{ + future::poll_fn, + sync::atomic::{AtomicU32, Ordering}, + task::{Context, Poll}, +}; + +use alloc::{sync::Arc, vec::Vec}; +use futures_util::task::AtomicWaker; +use libk_mm::address::PhysicalAddress; +use yggdrasil_abi::error::Error; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum UsbDirection { + Out, + In, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct UsbTransferToken(pub u64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct UsbTransferResult(pub u32); + +pub struct UsbTransferStatus { + pub result: AtomicU32, + pub notify: AtomicWaker, +} + +pub struct UsbTransfer { + pub id: UsbTransferToken, + pub length: usize, + pub direction: UsbDirection, + pub elements: Vec, + pub status: Arc, +} + +// TODO this is xHCI-specific +impl UsbTransferResult { + pub fn is_success(&self) -> bool { + (self.0 >> 24) & 0xFF == 1 + } + + pub fn sub_length(&self) -> usize { + (self.0 & 0xFFFFFF) as _ + } +} + +impl UsbTransfer { + pub async fn wait(&self) -> Result { + let sub_length = self.status.wait().await?; + Ok(self.length.saturating_sub(sub_length)) + } +} + +impl UsbTransferStatus { + pub fn new() -> Self { + Self { + result: AtomicU32::new(0), + notify: AtomicWaker::new(), + } + } + + pub(crate) async fn wait(&self) -> Result { + poll_fn(|cx| { + self.poll(cx).map(|v| { + if v.is_success() { + Ok(v.sub_length()) + } else { + Err(Error::InvalidOperation) + } + }) + }) + .await + } + + pub fn signal(&self, status: u32) { + self.result.store(status, Ordering::Release); + self.notify.wake(); + } + + pub fn poll(&self, cx: &mut Context<'_>) -> Poll { + self.notify.register(cx.waker()); + let value = self.result.load(Ordering::Acquire); + if value != 0 { + Poll::Ready(UsbTransferResult(value)) + } else { + Poll::Pending + } + } +} diff --git a/driver/bus/usb/src/descriptor.rs b/driver/bus/usb/src/descriptor.rs new file mode 100644 index 00000000..6cc4e276 --- /dev/null +++ b/driver/bus/usb/src/descriptor.rs @@ -0,0 +1,146 @@ +use bytemuck::{Pod, Zeroable}; +use yggdrasil_abi::error::Error; + +use crate::{ + info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType}, + UsbDirection, +}; + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C, packed)] +pub struct UsbDeviceDescriptor { + pub length: u8, + pub ty: u8, + pub bcd_usb: u16, + pub device_class: u8, + pub device_subclass: u8, + pub device_protocol: u8, + pub max_packet_size_0: u8, + pub id_vendor: u16, + pub id_product: u16, + pub bcd_device: u16, + pub manufacturer_str: u8, + pub product_str: u8, + pub serial_number_str: u8, + pub num_configurations: u8, +} + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C, packed)] +pub struct UsbConfigurationDescriptor { + pub length: u8, + pub ty: u8, + pub total_length: u16, + pub num_interfaces: u8, + pub config_val: u8, + pub config_str: u8, + pub attributes: u8, + pub max_power: u8, +} + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C, packed)] +pub struct UsbInterfaceDescriptor { + pub length: u8, + pub ty: u8, + pub interface_number: u8, + pub alternate_setting: u8, + pub num_endpoints: u8, + pub interface_class: u8, + pub interface_subclass: u8, + pub interface_protocol: u8, + pub interface_str: u8, +} + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C, packed)] +pub struct UsbEndpointDescriptor { + pub length: u8, + pub ty: u8, + pub endpoint_address: u8, + pub attributes: u8, + pub max_packet_size: u16, + pub interval: u8, +} + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C, packed)] +pub struct UsbDeviceQualifier { + pub length: u8, + pub ty: u8, + pub bcd_usb: u16, + pub device_class: u8, + pub device_subclass: u8, + pub device_protocol: u8, + pub max_packet_size_0: u8, + pub num_configurations: u8, + pub _reserved: u8, +} + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C, packed)] +pub struct UsbOtherSpeedConfiguration { + pub length: u8, + pub ty: u8, + pub total_length: u16, + pub num_interfaces: u8, + pub config_val: u8, + pub config_str: u8, + pub attributes: u8, + pub max_power: u8, +} + +impl UsbInterfaceDescriptor { + pub fn class(&self) -> UsbDeviceClass { + UsbDeviceClass::try_from(self.interface_class).unwrap_or(UsbDeviceClass::Unknown) + } + + pub fn protocol(&self) -> UsbDeviceProtocol { + UsbDeviceProtocol::try_from(self.interface_protocol).unwrap_or(UsbDeviceProtocol::Unknown) + } +} + +impl UsbEndpointDescriptor { + pub fn direction(&self) -> UsbDirection { + match self.endpoint_address >> 7 { + 1 => UsbDirection::In, + 0 => UsbDirection::Out, + _ => unreachable!(), + } + } + + pub fn number(&self) -> u8 { + assert_ne!(self.endpoint_address & 0xF, 0); + self.endpoint_address & 0xF + } + + pub fn transfer_type(&self) -> UsbEndpointType { + match self.attributes & 0x3 { + 0 => UsbEndpointType::Control, + 1 => UsbEndpointType::Isochronous, + 2 => UsbEndpointType::Bulk, + 3 => UsbEndpointType::Interrupt, + _ => unreachable!(), + } + } +} + +impl UsbDeviceDescriptor { + pub fn class(&self) -> UsbDeviceClass { + UsbDeviceClass::try_from(self.device_class).unwrap_or(UsbDeviceClass::Unknown) + } + + pub fn protocol(&self) -> UsbDeviceProtocol { + UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown) + } + + pub fn max_packet_size(&self) -> Result { + match self.max_packet_size_0 { + 8 => Ok(8), + 16 => Ok(16), + 32 => Ok(32), + 64 => Ok(64), + _ => Err(Error::InvalidArgument), + } + } +} diff --git a/driver/bus/usb/src/device.rs b/driver/bus/usb/src/device.rs new file mode 100644 index 00000000..d738d4bc --- /dev/null +++ b/driver/bus/usb/src/device.rs @@ -0,0 +1,191 @@ +use core::{fmt, ops::Deref}; + +use alloc::{boxed::Box, vec::Vec}; +use futures_util::future::BoxFuture; +use libk_mm::PageBox; +use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}; +use yggdrasil_abi::error::Error; + +use crate::{ + info::{UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbInterfaceInfo}, + pipe::{ + control::{ConfigurationDescriptorEntry, UsbControlPipeAccess}, + interrupt::UsbInterruptPipeAccess, + }, + UsbDirection, UsbHostController, +}; + +// High-level structures for info provided through descriptors + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct UsbBusAddress { + pub bus: u16, + pub device: u8, +} + +pub struct UsbDeviceAccess { + pub device: Box, + pub info: UsbDeviceInfo, + pub num_configurations: u8, + pub current_configuration: IrqSafeRwLock>, +} + +#[allow(unused)] +pub trait UsbDevice: Send + Sync { + // Endpoint "0" + fn control_pipe(&self) -> Option<&UsbControlPipeAccess>; + + fn open_interrupt_pipe<'a>( + &'a self, + number: u8, + direction: UsbDirection, + ) -> BoxFuture> { + unimplemented!() + } + + fn port_number(&self) -> u8; + fn bus_address(&self) -> UsbBusAddress; + fn controller(&self) -> &'static dyn UsbHostController; + + fn debug(&self) {} +} + +impl UsbDeviceAccess { + /// Expected device state: + /// + /// * Link-layer stuff has been reset and established properly by the HCD + /// * Device is not yet configured + /// * Control pipe for the device has been properly set up + /// * Device has been assigned a bus address + pub async fn setup(raw: Box) -> Result { + let control = raw.control_pipe().ok_or(Error::InvalidOperation)?; + let mut string_buffer = PageBox::new_uninit()?; + + let device_desc = control.query_device_descriptor().await?; + + let manufacturer = control + .query_string(device_desc.manufacturer_str, &mut string_buffer) + .await?; + let product = control + .query_string(device_desc.product_str, &mut string_buffer) + .await?; + + let info = UsbDeviceInfo { + manufacturer, + product, + + id_vendor: device_desc.id_vendor, + id_product: device_desc.id_product, + + device_class: device_desc.class(), + device_subclass: device_desc.device_subclass, + device_protocol: device_desc.protocol(), + + max_packet_size: device_desc.max_packet_size()?, + }; + + Ok(Self { + device: raw, + info, + num_configurations: device_desc.num_configurations, + current_configuration: IrqSafeRwLock::new(None), + }) + } + + pub fn read_current_configuration( + &self, + ) -> IrqSafeRwLockReadGuard> { + self.current_configuration.read() + } + + pub async fn select_configuration bool>( + &self, + predicate: F, + ) -> Result, Error> { + let mut current_config = self.current_configuration.write(); + let control_pipe = self.control_pipe().ok_or(Error::InvalidOperation)?; + + for i in 0..self.num_configurations { + let info = self.query_configuration_info(i).await?; + + if predicate(&info) { + log::debug!("Selected configuration: {:#?}", info); + let config = current_config.insert(info); + + control_pipe + .set_configuration(config.config_value as _) + .await?; + + return Ok(Some(config.clone())); + } + } + + Ok(None) + } + + pub async fn query_configuration_info(&self, index: u8) -> Result { + if index >= self.num_configurations { + return Err(Error::InvalidArgument); + } + let mut string_buffer = PageBox::new_uninit()?; + let control_pipe = self.control_pipe().ok_or(Error::InvalidOperation)?; + let query = control_pipe.query_configuration_descriptor(index).await?; + + let configuration_name = control_pipe + .query_string(query.configuration().config_str, &mut string_buffer) + .await?; + + let mut endpoints = Vec::new(); + let mut interfaces = Vec::new(); + + for desc in query.descriptors() { + match desc { + ConfigurationDescriptorEntry::Endpoint(ep) => { + endpoints.push(UsbEndpointInfo { + number: ep.number(), + direction: ep.direction(), + max_packet_size: ep.max_packet_size as _, + ty: ep.transfer_type(), + }); + } + ConfigurationDescriptorEntry::Interface(iface) => { + let name = control_pipe + .query_string(iface.interface_str, &mut string_buffer) + .await?; + interfaces.push(UsbInterfaceInfo { + name, + number: iface.interface_number, + + interface_class: iface.class(), + interface_subclass: iface.interface_subclass, + interface_protocol: iface.protocol(), + }); + } + _ => (), + } + } + + let info = UsbConfigurationInfo { + name: configuration_name, + config_value: query.configuration().config_val, + interfaces, + endpoints, + }; + + Ok(info) + } +} + +impl Deref for UsbDeviceAccess { + type Target = dyn UsbDevice; + + fn deref(&self) -> &Self::Target { + &*self.device + } +} + +impl fmt::Display for UsbBusAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}", self.bus, self.device) + } +} diff --git a/driver/bus/usb/src/info.rs b/driver/bus/usb/src/info.rs new file mode 100644 index 00000000..8e079bb3 --- /dev/null +++ b/driver/bus/usb/src/info.rs @@ -0,0 +1,84 @@ +use alloc::{string::String, vec::Vec}; +use yggdrasil_abi::primitive_enum; + +use crate::UsbDirection; + +#[derive(Debug, Clone, Copy)] +pub enum UsbEndpointType { + Control, + Isochronous, + Bulk, + Interrupt, +} + +#[derive(Debug, Clone, Copy)] +pub enum UsbSyncType { + NoSync, + Async, + Adaptive, + Sync, +} + +#[derive(Debug)] +pub enum UsbUsageType { + Data, + Feedback, + ImplicitFeedbackData, + Reserved, +} + +primitive_enum! { + pub enum UsbDeviceClass: u8 { + FromInterface = 0x00, + Hid = 0x03, + Unknown = 0xFF, + } +} + +primitive_enum! { + pub enum UsbDeviceProtocol: u8 { + FromInterface = 0x00, + Unknown = 0xFF, + } +} + +#[derive(Debug, Clone)] +pub struct UsbInterfaceInfo { + pub name: String, + pub number: u8, + + pub interface_class: UsbDeviceClass, + pub interface_subclass: u8, + pub interface_protocol: UsbDeviceProtocol, +} + +#[derive(Debug, Clone)] +pub struct UsbEndpointInfo { + pub number: u8, + pub direction: UsbDirection, + pub max_packet_size: usize, + pub ty: UsbEndpointType, +} + +#[derive(Debug, Clone)] +pub struct UsbConfigurationInfo { + pub name: String, + pub config_value: u8, + pub interfaces: Vec, + pub endpoints: Vec, +} + +#[derive(Debug, Clone)] +pub struct UsbDeviceInfo { + pub manufacturer: String, + pub product: String, + + pub id_vendor: u16, + pub id_product: u16, + + pub device_class: UsbDeviceClass, + pub device_subclass: u8, + pub device_protocol: UsbDeviceProtocol, + + pub max_packet_size: usize, +} diff --git a/driver/bus/usb/src/lib.rs b/driver/bus/usb/src/lib.rs new file mode 100644 index 00000000..e51ef978 --- /dev/null +++ b/driver/bus/usb/src/lib.rs @@ -0,0 +1,19 @@ +#![no_std] +#![feature(iter_array_chunks, maybe_uninit_slice)] + +extern crate alloc; + +pub mod bus; +pub mod communication; +pub mod descriptor; +pub mod device; +pub mod info; +pub mod pipe; + +pub mod class_driver; + +pub use communication::{UsbDirection, UsbTransfer, UsbTransferStatus, UsbTransferToken}; + +pub trait UsbEndpoint {} + +pub trait UsbHostController {} diff --git a/driver/bus/usb/src/pipe/control.rs b/driver/bus/usb/src/pipe/control.rs new file mode 100644 index 00000000..2f8a17a5 --- /dev/null +++ b/driver/bus/usb/src/pipe/control.rs @@ -0,0 +1,323 @@ +use core::{ + cmp::Ordering, + mem::{size_of, MaybeUninit}, + ops::Deref, +}; + +use alloc::{boxed::Box, string::String}; +use bytemuck::{Pod, Zeroable}; +use libk_mm::{ + address::{AsPhysicalAddress, PhysicalAddress}, + PageBox, +}; +use yggdrasil_abi::error::Error; + +use crate::{ + descriptor::{ + UsbConfigurationDescriptor, UsbDeviceDescriptor, UsbDeviceQualifier, UsbEndpointDescriptor, + UsbInterfaceDescriptor, UsbOtherSpeedConfiguration, + }, + UsbDirection, UsbTransfer, +}; + +use super::UsbGenericPipe; + +#[derive(Debug)] +pub struct ControlTransferSetup { + pub bm_request_type: u8, + pub b_request: u8, + pub w_value: u16, + pub w_index: u16, + pub w_length: u16, +} + +#[derive(Clone, Copy, Debug, Default, Pod, Zeroable)] +#[repr(C)] +pub struct SetConfiguration; + +pub trait UsbDeviceRequest: Sized + Pod { + const BM_REQUEST_TYPE: u8; + const B_REQUEST: u8; +} + +pub trait UsbDescriptorRequest: UsbDeviceRequest { + const DESCRIPTOR_TYPE: u8; +} + +impl UsbDescriptorRequest for UsbDeviceDescriptor { + const DESCRIPTOR_TYPE: u8 = 1; +} + +impl UsbDescriptorRequest for UsbConfigurationDescriptor { + const DESCRIPTOR_TYPE: u8 = 2; +} + +impl UsbDescriptorRequest for UsbInterfaceDescriptor { + const DESCRIPTOR_TYPE: u8 = 4; +} + +impl UsbDeviceRequest for SetConfiguration { + const BM_REQUEST_TYPE: u8 = 0; + const B_REQUEST: u8 = 0x09; +} + +impl UsbDeviceRequest for U { + const BM_REQUEST_TYPE: u8 = 0b10000000; + const B_REQUEST: u8 = 0x06; +} + +fn decode_usb_string(bytes: &[u8]) -> Result { + if bytes.len() % 2 != 0 { + return Err(Error::InvalidArgument); + } + + char::decode_utf16( + bytes + .into_iter() + .array_chunks::<2>() + .map(|[&a, &b]| u16::from_le_bytes([a, b])), + ) + .collect::>() + .map_err(|_| Error::InvalidArgument) +} + +// Pipe impl + +pub trait UsbControlPipe: UsbGenericPipe + Send + Sync { + fn start_transfer( + &self, + setup: ControlTransferSetup, + data: Option<(PhysicalAddress, usize, UsbDirection)>, + ) -> Result; +} + +pub struct UsbControlPipeAccess(pub Box); + +fn input_buffer( + data: &mut PageBox>, +) -> (PhysicalAddress, usize, UsbDirection) { + ( + unsafe { data.as_physical_address() }, + size_of::(), + UsbDirection::In, + ) +} + +#[derive(Debug)] +pub enum ConfigurationDescriptorEntry<'a> { + Configuration(&'a UsbConfigurationDescriptor), + Interface(&'a UsbInterfaceDescriptor), + Endpoint(&'a UsbEndpointDescriptor), + DeviceQualifier(&'a UsbDeviceQualifier), + OtherSpeed(&'a UsbOtherSpeedConfiguration), + Other, +} + +pub struct ConfigurationDescriptorIter<'a> { + buffer: &'a PageBox<[u8]>, + offset: usize, +} + +pub struct ConfigurationDescriptorQuery { + buffer: PageBox<[u8]>, +} + +impl<'a> Iterator for ConfigurationDescriptorIter<'a> { + type Item = ConfigurationDescriptorEntry<'a>; + + fn next(&mut self) -> Option { + if self.offset + 2 >= self.buffer.len() { + return None; + } + + let desc_len = self.buffer[self.offset] as usize; + let desc_ty = self.buffer[self.offset + 1]; + + if desc_len == 0 { + return None; + } + + let entry = match desc_ty { + 0x02 if desc_len == size_of::() => { + ConfigurationDescriptorEntry::Configuration(bytemuck::from_bytes( + &self.buffer[self.offset..self.offset + desc_len], + )) + } + 0x04 if desc_len == size_of::() => { + ConfigurationDescriptorEntry::Interface(bytemuck::from_bytes( + &self.buffer[self.offset..self.offset + desc_len], + )) + } + 0x05 if desc_len == size_of::() => { + ConfigurationDescriptorEntry::Endpoint(bytemuck::from_bytes( + &self.buffer[self.offset..self.offset + desc_len], + )) + } + 0x07 if desc_len == size_of::() => { + ConfigurationDescriptorEntry::OtherSpeed(bytemuck::from_bytes( + &self.buffer[self.offset..self.offset + desc_len], + )) + } + _ => ConfigurationDescriptorEntry::Other, + }; + + self.offset += desc_len; + + Some(entry) + } +} + +impl ConfigurationDescriptorQuery { + pub fn configuration(&self) -> &UsbConfigurationDescriptor { + bytemuck::from_bytes(&self.buffer[..size_of::()]) + } + + pub fn descriptors(&self) -> ConfigurationDescriptorIter<'_> { + ConfigurationDescriptorIter { + buffer: &self.buffer, + offset: 0, + } + } +} + +impl UsbControlPipeAccess { + pub async fn perform_value_control( + &self, + setup: ControlTransferSetup, + buffer: Option<(PhysicalAddress, usize, UsbDirection)>, + ) -> Result<(), Error> { + let transfer = self.start_transfer(setup, buffer)?; + transfer.status.wait().await?; + self.complete_transfer(transfer); + Ok(()) + } + + async fn fill_configuation_descriptor( + &self, + index: u8, + buffer: &mut PageBox<[MaybeUninit]>, + ) -> Result<(), Error> { + self.perform_value_control( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0x06, + w_value: 0x200 | (index as u16), + w_index: 0, + w_length: buffer.len().try_into().unwrap(), + }, + Some(( + unsafe { buffer.as_physical_address() }, + buffer.len(), + UsbDirection::In, + )), + ) + .await + } + + pub async fn query_configuration_descriptor( + &self, + index: u8, + ) -> Result { + // First, query the real length of the descriptor + let mut buffer = PageBox::new_uninit_slice(size_of::())?; + self.fill_configuation_descriptor(index, &mut buffer) + .await?; + let buffer = unsafe { PageBox::assume_init_slice(buffer) }; + + let desc: &UsbConfigurationDescriptor = bytemuck::from_bytes(&buffer); + let total_len = desc.total_length as usize; + + // Return if everything's ready at this point + match total_len.cmp(&size_of::()) { + Ordering::Less => todo!(), + Ordering::Equal => return Ok(ConfigurationDescriptorQuery { buffer }), + _ => (), + } + + // Otherwise, query the rest of the data + let mut buffer = PageBox::new_uninit_slice(total_len)?; + self.fill_configuation_descriptor(index, &mut buffer) + .await?; + let buffer = unsafe { PageBox::assume_init_slice(buffer) }; + + let desc: &UsbConfigurationDescriptor = + bytemuck::from_bytes(&buffer[..size_of::()]); + let total_len = desc.total_length as usize; + + if total_len != buffer.len() { + todo!(); + } + + Ok(ConfigurationDescriptorQuery { buffer }) + } + + pub async fn query_device_descriptor(&self) -> Result, Error> { + let mut output = PageBox::new_uninit()?; + self.perform_value_control( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0x06, + w_value: 0x100, + w_index: 0, + w_length: size_of::() as _, + }, + Some(input_buffer(&mut output)), + ) + .await?; + + Ok(unsafe { output.assume_init() }) + } + + pub async fn query_string( + &self, + index: u8, + buffer: &mut PageBox>, + ) -> Result { + self.perform_value_control( + ControlTransferSetup { + bm_request_type: 0b10000000, + b_request: 0x06, + w_value: 0x300 | (index as u16), + w_index: 0, + w_length: 4096, + }, + Some(input_buffer(buffer)), + ) + .await?; + let data = unsafe { buffer.assume_init_ref() }; + + let len = data[0] as usize; + + decode_usb_string(&data[2..len]) + } + + pub async fn perform_action( + &self, + w_value: u16, + w_index: u16, + ) -> Result<(), Error> { + self.perform_value_control( + ControlTransferSetup { + bm_request_type: D::BM_REQUEST_TYPE, + b_request: D::B_REQUEST, + w_value, + w_index, + w_length: 0, + }, + None, + ) + .await + } + + pub async fn set_configuration(&self, value: u16) -> Result<(), Error> { + self.perform_action::(value, 0).await + } +} + +impl Deref for UsbControlPipeAccess { + type Target = dyn UsbControlPipe; + + fn deref(&self) -> &Self::Target { + &*self.0 + } +} diff --git a/driver/bus/usb/src/pipe/interrupt.rs b/driver/bus/usb/src/pipe/interrupt.rs new file mode 100644 index 00000000..a42d6d36 --- /dev/null +++ b/driver/bus/usb/src/pipe/interrupt.rs @@ -0,0 +1,32 @@ +use core::ops::Deref; + +use alloc::boxed::Box; +use libk_mm::PageBox; +use yggdrasil_abi::error::Error; + +use crate::UsbTransfer; + +use super::UsbGenericPipe; + +pub trait UsbInterruptPipe: UsbGenericPipe + Send + Sync { + fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result; +} + +pub struct UsbInterruptPipeAccess(pub Box); + +impl UsbInterruptPipeAccess { + // TODO timeout + pub async fn listen<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], Error> { + let transfer = self.start_read(buffer)?; + let len = transfer.wait().await?; + Ok(&buffer[..len]) + } +} + +impl Deref for UsbInterruptPipeAccess { + type Target = dyn UsbInterruptPipe; + + fn deref(&self) -> &Self::Target { + &*self.0 + } +} diff --git a/driver/bus/usb/src/pipe/mod.rs b/driver/bus/usb/src/pipe/mod.rs new file mode 100644 index 00000000..45fab67b --- /dev/null +++ b/driver/bus/usb/src/pipe/mod.rs @@ -0,0 +1,12 @@ +use super::UsbTransfer; + +pub mod control; +pub mod interrupt; + +pub trait UsbGenericPipe { + fn complete_transfer(&self, transfer: UsbTransfer); +} + +pub enum UsbPipe { + Control(control::UsbControlPipeAccess), +} diff --git a/driver/input/Cargo.toml b/driver/input/Cargo.toml new file mode 100644 index 00000000..abfeffd1 --- /dev/null +++ b/driver/input/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ygg_driver_input" +version = "0.1.0" +edition = "2021" + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +libk-util = { path = "../../libk/libk-util" } +libk-thread = { path = "../../libk/libk-thread" } +libk-mm = { path = "../../libk/libk-mm" } +vfs = { path = "../../lib/vfs" } diff --git a/src/device/input.rs b/driver/input/src/lib.rs similarity index 93% rename from src/device/input.rs rename to driver/input/src/lib.rs index a2493b5b..87242944 100644 --- a/src/device/input.rs +++ b/driver/input/src/lib.rs @@ -1,15 +1,16 @@ -// TODO -#![allow(missing_docs)] +#![no_std] + +extern crate alloc; use core::task::{Context, Poll}; -use abi::{ +use libk_thread::block; +use libk_util::ring::LossyRingQueue; +use vfs::{CharDevice, FileReadiness}; +use yggdrasil_abi::{ error::Error, io::{DeviceRequest, KeyboardKeyEvent}, }; -use libk::block; -use libk_util::ring::LossyRingQueue; -use vfs::{CharDevice, FileReadiness}; pub struct KeyboardDevice; diff --git a/driver/usb/xhci/Cargo.toml b/driver/usb/xhci/Cargo.toml new file mode 100644 index 00000000..9c4359f9 --- /dev/null +++ b/driver/usb/xhci/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "ygg_driver_usb_xhci" +version = "0.1.0" +edition = "2021" +authors = ["Mark Poliakov "] + +[dependencies] +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +device-api = { path = "../../../lib/device-api", features = ["derive"] } +ygg_driver_pci = { path = "../../bus/pci" } +ygg_driver_usb = { path = "../../bus/usb" } + +libk-util = { path = "../../../libk/libk-util" } +libk-mm = { path = "../../../libk/libk-mm" } +libk-thread = { path = "../../../libk/libk-thread" } + +xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } + +log = "0.4.20" +tock-registers = "0.9.0" +bytemuck = { version = "1.14.0", features = ["derive"] } +futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } diff --git a/driver/usb/xhci/src/lib.rs b/driver/usb/xhci/src/lib.rs new file mode 100644 index 00000000..5b778e9f --- /dev/null +++ b/driver/usb/xhci/src/lib.rs @@ -0,0 +1,484 @@ +#![no_std] +#![feature(iter_array_chunks)] + +extern crate alloc; + +use core::{ + future::poll_fn, + sync::atomic::{AtomicBool, Ordering}, + task::{Context, Poll}, + time::Duration, +}; + +use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec::Vec}; +use device_api::{ + interrupt::{InterruptAffinity, InterruptHandler}, + Device, +}; +use futures_util::{future::BoxFuture, FutureExt}; +use libk_mm::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + PageBox, +}; +use libk_thread::runtime::{self, FutureTimeout}; +use libk_util::{sync::spin_rwlock::IrqSafeRwLock, waker::QueueWaker, OneTimeInit}; +use pipe::ControlPipe; +use regs::{Mapper, Regs}; +use ring::{CommandExecutor, CommandRing, EventRing}; +use xhci_lib::context::{self, InputHandler}; +use ygg_driver_pci::{ + device::{PciDeviceInfo, PreferredInterruptMode}, + PciCommandRegister, PciConfigurationSpace, +}; +use ygg_driver_usb::{ + bus::UsbBusManager, + device::{UsbBusAddress, UsbDevice, UsbDeviceAccess}, + info::UsbEndpointType, + pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptPipeAccess}, + UsbDirection, UsbHostController, +}; +use yggdrasil_abi::error::Error; + +use crate::{ + pipe::InterruptPipe, + ring::{Event, EventRingSegmentTable, TransferRing}, +}; + +mod pipe; +mod regs; +mod ring; + +pub struct XhciContext { + pub(crate) input: IrqSafeRwLock>>, + pub(crate) output: PageBox>, +} + +// TODO device context information +pub struct XhciBusDevice { + port_id: u8, + slot_id: u8, + bus_address: UsbBusAddress, + + xhci: &'static Xhci, + + context: Arc>, + control_pipe: UsbControlPipeAccess, +} + +pub struct Xhci { + regs: Regs, + + bus_address: OneTimeInit, + + port_count: usize, + // TODO use to allocate proper contexts + #[allow(unused)] + context_size: usize, + + dcbaa: IrqSafeRwLock>, + endpoints: IrqSafeRwLock>>, + event_ring: EventRing, + command_ring: CommandRing, + + spawned_ports: Vec, + port_reset_notify: QueueWaker, +} + +impl XhciBusDevice { + fn dci(ty: UsbEndpointType, dir: UsbDirection, number: u8) -> usize { + match ty { + UsbEndpointType::Control => (number as usize) * 2 + 1, + _ => { + let off = match dir { + UsbDirection::Out => 0, + UsbDirection::In => 1, + }; + + (number as usize * 2) + off + } + } + } + + async fn setup_endpoint_inner( + &self, + number: u8, + ty: UsbEndpointType, + direction: UsbDirection, + ) -> Result, Error> { + log::debug!("Setup endpoint #{}: {:?} {:?}", number, ty, direction); + + let mut input = self.context.input.write(); + + let ep_type = match (ty, direction) { + (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn, + _ => todo!(), + }; + // number = 1 + // dci = (number * 2) + IN = 1 * 2 + 1 = 3 + let dci = Self::dci(ty, direction, number); + + let ring = Arc::new(TransferRing::new(self.slot_id, dci as _, 128)?); + + { + let control = input.control_mut(); + + control.set_add_context_flag(0); + control.clear_add_context_flag(1); + control.set_add_context_flag(dci); + } + + { + let slot = input.device_mut().slot_mut(); + + slot.set_context_entries(31); + } + + { + let ep_cx = input.device_mut().endpoint_mut(dci); + + ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); + ep_cx.set_dequeue_cycle_state(); + ep_cx.set_endpoint_type(ep_type); + ep_cx.set_error_count(3); + + // TODO get from endpoint info + ep_cx.set_max_packet_size(8); + } + + self.xhci + .command_ring + .configure_endpoint(self.xhci, self.slot_id, &mut input) + .await?; + + self.xhci + .register_endpoint(self.slot_id, dci as _, ring.clone()); + + Ok(ring) + } +} + +impl UsbDevice for XhciBusDevice { + fn control_pipe(&self) -> Option<&UsbControlPipeAccess> { + Some(&self.control_pipe) + } + + fn port_number(&self) -> u8 { + self.port_id + } + + fn bus_address(&self) -> UsbBusAddress { + self.bus_address + } + + fn controller(&self) -> &'static dyn UsbHostController { + self.xhci + } + + fn open_interrupt_pipe<'a>( + &'a self, + number: u8, + direction: UsbDirection, + ) -> BoxFuture> { + async move { + let ring = self + .setup_endpoint_inner(number, UsbEndpointType::Interrupt, direction) + .await?; + + let dci = Self::dci(UsbEndpointType::Interrupt, direction, number) + 1; + let pipe = InterruptPipe::input(self.xhci, self.slot_id, number, dci as _, ring); + + Ok(UsbInterruptPipeAccess(Box::new(pipe))) + } + .boxed() + } + + fn debug(&self) {} +} + +impl UsbHostController for Xhci {} + +impl XhciContext<8> { + pub fn new_32byte() -> Result { + let input = PageBox::new(context::Input::new_32byte())?; + let output = PageBox::new(context::Device::new_32byte())?; + + Ok(Self { + input: IrqSafeRwLock::new(input), + output, + }) + } +} + +impl Xhci { + pub fn new(regs: xhci_lib::Registers) -> Result { + let event_ring = EventRing::new(128)?; + let command_ring = CommandRing::new(128)?; + + let regs = Regs::from(regs); + + let port_count = regs.port_count(); + let slot_count = regs.max_slot_count(); + let context_size = regs.context_size(); + + Ok(Self { + regs, + + bus_address: OneTimeInit::new(), + + port_count, + context_size, + + event_ring, + command_ring, + + dcbaa: IrqSafeRwLock::new(PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1)?), + endpoints: IrqSafeRwLock::new(BTreeMap::new()), + + spawned_ports: Vec::from_iter((0..port_count).map(|_| AtomicBool::new(false))), + port_reset_notify: QueueWaker::new(), + }) + } + + pub fn register_device_context(&self, slot_id: u8, context: PhysicalAddress) { + self.dcbaa.write()[slot_id as usize] = context; + } + + pub fn register_endpoint(&self, slot_id: u8, endpoint_id: u8, ring: Arc) { + self.endpoints.write().insert((slot_id, endpoint_id), ring); + } + + pub fn notify_transfer( + &self, + slot_id: u8, + endpoint_id: u8, + address: PhysicalAddress, + status: u32, + ) { + if let Some(ep) = self.endpoints.read().get(&(slot_id, endpoint_id)) { + ep.notify(address, status); + } + } + + async fn assign_device( + &'static self, + slot_id: u8, + root_hub_port_number: u8, + ) -> Result, Error> { + let address = 1; + let ring = Arc::new(TransferRing::new(slot_id, 1, 128)?); + + let context = XhciContext::new_32byte()?; + let mut input = context.input.write(); + + // Setup input context + { + let control = input.control_mut(); + + control.set_add_context_flag(0); + control.set_add_context_flag(1); + } + + { + let slot = input.device_mut().slot_mut(); + + slot.set_context_entries(1); + slot.set_interrupter_target(0); + slot.set_usb_device_address(address); + slot.set_root_hub_port_number(root_hub_port_number); + } + + { + let ep0 = input.device_mut().endpoint_mut(1); + + ep0.set_endpoint_type(context::EndpointType::Control); + ep0.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); + ep0.set_dequeue_cycle_state(); + ep0.set_error_count(3); + } + + self.register_device_context(slot_id, unsafe { context.output.as_physical_address() }); + + self.command_ring + .address_device(self, slot_id, &mut input) + .await?; + + self.register_endpoint(slot_id, 1, ring.clone()); + + let pipe = ControlPipe::new(self, slot_id, ring); + + drop(input); + + let bus_address = UsbBusAddress { + bus: *self.bus_address.get(), + device: address, + }; + + let device = XhciBusDevice { + xhci: self, + slot_id, + port_id: root_hub_port_number, + bus_address, + context: Arc::new(context), + control_pipe: UsbControlPipeAccess(Box::new(pipe)), + }; + + Ok(Box::new(device)) + } + + async fn port_task(&'static self, index: usize) -> Result<(), Error> { + log::debug!("Task spawned for port {}", index); + + self.reset_port(index).await?; + + let slot_id = self.command_ring.enable_slot(self).await?; + log::debug!("Enabled slot: {}", slot_id); + + let device = self.assign_device(slot_id, (index + 1) as _).await?; + let device = UsbDeviceAccess::setup(device).await?; + + UsbBusManager::register_device(Arc::new(device)); + + Ok(()) + } + + fn handle_device_attached(&'static self, port: usize) -> Result<(), Error> { + if !self.spawned_ports[port].swap(true, Ordering::Release) { + // Port has not yet been spawned + runtime::spawn(async move { self.port_task(port).await })?; + } + Ok(()) + } + + fn poll_port_reset(&self, cx: &mut Context<'_>, port: usize) -> Poll<()> { + self.port_reset_notify.register(cx.waker()); + if self.regs.ports.read(port).portsc.port_reset_change() { + self.port_reset_notify.remove(cx.waker()); + Poll::Ready(()) + } else { + Poll::Pending + } + } + + async fn reset_port(&self, port: usize) -> Result<(), Error> { + log::debug!("Reset port {}", port); + + self.regs.ports.update(port, |u| { + u.portsc.set_port_reset(); + }); + + // Wait for port reset + let status = runtime::run_with_timeout( + Duration::from_secs(1), + poll_fn(|cx| self.poll_port_reset(cx, port)), + ) + .await; + + match status { + FutureTimeout::Ok(()) => Ok(()), + FutureTimeout::Timeout => Err(Error::TimedOut), + } + } + + fn handle_event(&self) { + while let Some(event) = self.event_ring.try_dequeue() { + match event { + Event::PortChange(_) => { + self.port_reset_notify.wake_one(); + } + Event::CommandCompletion { address, reply } => { + self.command_ring.notify(address, reply); + } + Event::Transfer { + address, + slot_id, + endpoint_id, + status, + } => { + self.notify_transfer(slot_id, endpoint_id, address, status); + } + } + } + + self.regs + .set_interrupter_0_dequeue_pointer(self.event_ring.dequeue_pointer()); + } +} + +impl CommandExecutor for Xhci { + fn ring_doorbell(&self, index: usize, target: u8) { + self.regs.ring_doorbell(index, target); + } +} + +impl Device for Xhci { + unsafe fn init(&'static self) -> Result<(), Error> { + log::info!("Init USB xHCI"); + + self.regs.reset(); + self.regs.set_max_slot_count(); + + let erst = EventRingSegmentTable::for_event_rings(&[&self.event_ring])?; + let dcbaa = self.dcbaa.read(); + + self.regs + .configure(&dcbaa, &self.command_ring, &self.event_ring, &erst); + + let bus = UsbBusManager::register_bus(self); + self.bus_address.init(bus); + + for port in 0..self.port_count { + let p = self.regs.ports.read(port); + if p.portsc.current_connect_status() { + self.handle_device_attached(port).ok(); + } + } + + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + log::info!("Init USB xHCI IRQ"); + Ok(()) + } + + fn display_name(&self) -> &'static str { + "USB xHCI" + } +} + +impl InterruptHandler for Xhci { + fn handle_irq(&self, _vector: Option) -> bool { + if let Some(status) = self.regs.handle_interrupt() { + if status.event_interrupt() { + self.handle_event(); + } + + true + } else { + false + } + } +} + +pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { + // TODO Chip Hardware Reset + let bar0 = info + .config_space + .bar(0) + .expect("xHCI doesn't have BAR0 configured") + .as_memory() + .expect("xHCI's BAR0 is not memory-type"); + + let mut cmd = PciCommandRegister::from_bits_retain(info.config_space.command()); + cmd &= !(PciCommandRegister::DISABLE_INTERRUPTS | PciCommandRegister::ENABLE_IO); + cmd |= PciCommandRegister::ENABLE_MEMORY | PciCommandRegister::BUS_MASTER; + info.config_space.set_command(cmd.bits()); + + let regs = unsafe { xhci_lib::Registers::new(bar0.try_into().unwrap(), Mapper::new()) }; + let xhci = Box::leak(Box::new(Xhci::new(regs)?)); + + info.init_interrupts(PreferredInterruptMode::Msi)?; + info.map_interrupt(InterruptAffinity::Any, xhci)?; + + Ok(xhci) +} diff --git a/driver/usb/xhci/src/pipe.rs b/driver/usb/xhci/src/pipe.rs new file mode 100644 index 00000000..a0102bb7 --- /dev/null +++ b/driver/usb/xhci/src/pipe.rs @@ -0,0 +1,88 @@ +use alloc::sync::Arc; +use libk_mm::{address::PhysicalAddress, PageBox}; +use ygg_driver_usb::{ + pipe::{ + control::{ControlTransferSetup, UsbControlPipe}, + interrupt::UsbInterruptPipe, + UsbGenericPipe, + }, + UsbDirection, UsbTransfer, +}; +use yggdrasil_abi::error::Error; + +use crate::{ring::TransferRing, Xhci}; + +pub struct ControlPipe { + xhci: &'static Xhci, + ring: Arc, +} + +#[allow(unused)] +pub struct InterruptPipe { + xhci: &'static Xhci, + + slot_id: u8, + endpoint_id: u8, + dci: u8, + + input: Option>, + output: Option>, +} + +impl UsbGenericPipe for ControlPipe { + fn complete_transfer(&self, transfer: UsbTransfer) { + self.ring.complete_transfer(transfer) + } +} + +impl UsbControlPipe for ControlPipe { + fn start_transfer( + &self, + setup: ControlTransferSetup, + data: Option<(PhysicalAddress, usize, UsbDirection)>, + ) -> Result { + self.ring.start_control_transfer(self.xhci, setup, data) + } +} + +impl ControlPipe { + pub fn new(xhci: &'static Xhci, _slot_id: u8, ring: Arc) -> Self { + Self { xhci, ring } + } +} + +impl UsbGenericPipe for InterruptPipe { + fn complete_transfer(&self, transfer: UsbTransfer) { + match transfer.direction { + UsbDirection::Out => todo!(), + UsbDirection::In => todo!(), + } + } +} + +impl UsbInterruptPipe for InterruptPipe { + fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result { + let input = self.input.as_ref().unwrap(); + + input.start_interrupt_in_transfer(self.xhci, buffer) + } +} + +impl InterruptPipe { + pub fn input( + xhci: &'static Xhci, + slot_id: u8, + endpoint_id: u8, + dci: u8, + ring: Arc, + ) -> Self { + Self { + xhci, + slot_id, + endpoint_id, + dci, + input: Some(ring), + output: None, + } + } +} diff --git a/driver/usb/xhci/src/regs.rs b/driver/usb/xhci/src/regs.rs new file mode 100644 index 00000000..55bf11ba --- /dev/null +++ b/driver/usb/xhci/src/regs.rs @@ -0,0 +1,240 @@ +use core::{cell::UnsafeCell, num::NonZeroUsize}; + +use alloc::{sync::Arc, vec::Vec}; +use libk_mm::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + device::RawDeviceMemoryMapping, + PageBox, +}; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; +use xhci_lib::{ + accessor::{array, marker}, + registers::{ + operational::UsbStatusRegister, Capability, Doorbell, InterrupterRegisterSet, Operational, + PortRegisterSet, + }, +}; + +use crate::ring::{CommandRing, EventRing, EventRingSegmentTable, GenericRing}; + +#[derive(Clone)] +pub struct Mapper { + mappings: Vec>, +} + +pub struct LockedArray { + array: UnsafeCell>, + locks: Vec>, +} + +unsafe impl Sync for LockedArray {} +unsafe impl Send for LockedArray {} + +pub struct Regs { + operational: IrqSafeRwLock>, + interrupters: IrqSafeRwLock>, + capability: Capability, + doorbells: LockedArray, + pub ports: LockedArray, +} + +impl LockedArray { + #[inline] + #[allow(clippy::mut_from_ref)] + unsafe fn get_mut(&self) -> &mut array::Generic { + &mut *self.array.get() + } + + pub fn update(&self, index: usize, f: U) { + let _guard = self.locks[index].write(); + unsafe { self.get_mut() }.update_volatile_at(index, f); + } + + pub fn read(&self, index: usize) -> T { + let _guard = self.locks[index].read(); + unsafe { self.get_mut() }.read_volatile_at(index) + } +} + +impl From> for LockedArray { + fn from(value: array::Generic) -> Self { + let locks = Vec::from_iter((0..value.len()).map(|_| IrqSafeRwLock::new(()))); + + Self { + array: UnsafeCell::new(value), + locks, + } + } +} + +impl From> for Regs { + fn from(value: xhci_lib::Registers) -> Self { + Self { + operational: IrqSafeRwLock::new(value.operational), + capability: value.capability, + interrupters: IrqSafeRwLock::new(value.interrupter_register_set), + doorbells: LockedArray::from(value.doorbell), + ports: LockedArray::from(value.port_register_set), + } + } +} + +impl Regs { + pub fn reset(&self) { + let mut o = self.operational.write(); + + // TODO Get ownership from firmware + + // Stop the controller + o.usbcmd.update_volatile(|u| { + u.clear_run_stop(); + }); + + while !o.usbsts.read_volatile().hc_halted() { + core::hint::spin_loop(); + } + + // Reset the controller + o.usbcmd.update_volatile(|u| { + u.set_host_controller_reset(); + }); + while o.usbcmd.read_volatile().host_controller_reset() + || o.usbsts.read_volatile().controller_not_ready() + { + core::hint::spin_loop(); + } + } + + pub fn max_slot_count(&self) -> usize { + self.capability + .hcsparams1 + .read_volatile() + .number_of_device_slots() as _ + } + + pub fn set_max_slot_count(&self) -> usize { + let device_slot_count = self.max_slot_count(); + let mut o = self.operational.write(); + // Set max slots enabled + o.config.update_volatile(|u| { + u.set_max_device_slots_enabled(device_slot_count as _); + }); + + device_slot_count as _ + } + + pub fn context_size(&self) -> usize { + match self.capability.hccparams1.read_volatile().context_size() { + true => 64, + false => 32, + } + } + + pub fn port_count(&self) -> usize { + self.capability.hcsparams1.read_volatile().number_of_ports() as _ + } + + pub fn configure( + &self, + dcbaa: &PageBox<[PhysicalAddress]>, + cmd_ring: &CommandRing, + evt_ring: &EventRing, + erst: &EventRingSegmentTable, + ) { + let mut o = self.operational.write(); + let mut i = self.interrupters.write(); + + o.dcbaap.update_volatile(|u| unsafe { + u.set(dcbaa.as_physical_address().into_raw()); + }); + o.crcr.update_volatile(|u| { + u.set_command_ring_pointer(cmd_ring.base().into_raw()); + u.set_ring_cycle_state(); + }); + + let mut intr0 = i.interrupter_mut(0); + intr0.erstsz.update_volatile(|u| { + u.set(erst.capacity().try_into().unwrap()); + }); + intr0.erdp.update_volatile(|u| { + log::debug!("::: Dequeue Pointer: {:#x}", evt_ring.dequeue_pointer()); + u.set_event_ring_dequeue_pointer(evt_ring.dequeue_pointer().into_raw()); + }); + intr0.erstba.update_volatile(|u| { + u.set(erst.physical_address().into_raw()); + }); + // intr0.imod.update_volatile(|u| { + // u.set_interrupt_moderation_interval(0) + // .set_interrupt_moderation_counter(0); + // }); + intr0.iman.update_volatile(|u| { + u.set_interrupt_enable(); + }); + + o.usbcmd.update_volatile(|u| { + u.set_interrupter_enable().set_run_stop(); + }); + } + + pub fn handle_interrupt(&self) -> Option { + let mut o = self.operational.write(); + let mut i = self.interrupters.write(); + + let status = o.usbsts.read_volatile(); + + if !status.event_interrupt() { + return None; + } + + o.usbsts.write_volatile(status); + + if status.host_system_error() { + return Some(status); + } + + // Acknowledge interrupts + let mut intr0 = i.interrupter_mut(0); + intr0.iman.update_volatile(|u| { + u.set_0_interrupt_pending(); + }); + + Some(status) + } + + pub fn set_interrupter_0_dequeue_pointer(&self, pointer: PhysicalAddress) { + let mut i = self.interrupters.write(); + + i.interrupter_mut(0).erdp.update_volatile(|u| { + u.set_event_ring_dequeue_pointer(pointer.into_raw()); + u.clear_event_handler_busy(); + }); + } + + pub fn ring_doorbell(&self, index: usize, target: u8) { + self.doorbells.update(index, |u| { + u.set_doorbell_target(target); + }); + } +} + +impl Mapper { + pub fn new() -> Self { + Self { + mappings: Vec::new(), + } + } +} + +impl xhci_lib::accessor::Mapper for Mapper { + unsafe fn map(&mut self, phys_start: usize, bytes: usize) -> NonZeroUsize { + let mapping = RawDeviceMemoryMapping::map(phys_start as u64, bytes, Default::default()) + .expect("Could not map an USB xHCI region"); + let address = mapping.address; + self.mappings.push(Arc::new(mapping)); + NonZeroUsize::new_unchecked(address) + } + + fn unmap(&mut self, _virt_start: usize, _bytes: usize) { + // TODO + } +} diff --git a/driver/usb/xhci/src/ring.rs b/driver/usb/xhci/src/ring.rs new file mode 100644 index 00000000..0c781cf9 --- /dev/null +++ b/driver/usb/xhci/src/ring.rs @@ -0,0 +1,656 @@ +use core::{ + future::poll_fn, + mem::{size_of, MaybeUninit}, + sync::atomic::{AtomicU64, Ordering}, + task::Poll, +}; + +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; +use libk_mm::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, + PageBox, +}; +use libk_util::{ + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + waker::QueueWaker, +}; +use tock_registers::{ + interfaces::{ReadWriteable, Readable, Writeable}, + register_bitfields, + registers::InMemoryRegister, +}; +use xhci_lib::context::Input; +use ygg_driver_usb::{ + pipe::control::ControlTransferSetup, UsbDirection, UsbTransfer, UsbTransferStatus, + UsbTransferToken, +}; +use yggdrasil_abi::error::Error; + +#[derive(Debug, Clone, Copy)] +pub struct CommandReply { + pub status: u32, + pub slot_id: u8, +} + +impl CommandReply { + pub fn completion_code(&self) -> u8 { + (self.status >> 24) as u8 + } +} + +pub enum Event { + PortChange(usize), + CommandCompletion { + address: PhysicalAddress, + reply: CommandReply, + }, + Transfer { + address: PhysicalAddress, + slot_id: u8, + endpoint_id: u8, + status: u32, + }, +} + +pub trait CommandExecutor { + fn ring_doorbell(&self, index: usize, target: u8); +} + +register_bitfields![ + u32, + pub TrbFlags [ + CYCLE OFFSET(0) NUMBITS(1) [], + LINKSEG OFFSET(1) NUMBITS(1) [], + INTERRUPT_ON_COMPLETION OFFSET(5) NUMBITS(1) [], + IMMEDIATE_DATA OFFSET(6) NUMBITS(1) [], + TYPE OFFSET(10) NUMBITS(6) [ + // Transfer ring types + NORMAL = 1, + SETUP = 2, + DATA = 3, + STATUS = 4, + ISOCH = 5, + LINK = 6, + EVENT = 7, + NOOP = 8, + // Command ring types + COMMAND_ENABLE_SLOT = 9, + COMMAND_DISABLE_SLOT = 10, + COMMAND_ADDRESS_DEVICE = 11, + COMMAND_CONFIGURE_ENDPOINT = 12, + // Event ring types + EVENT_XFER = 32, + EVENT_CMD_COMPLETE = 33, + EVENT_PORT_CHANGE = 34, + EVENT_BW_REQUEST = 35, + EVENT_DOORBELL = 36, + EVENT_HOST_CTRL = 37, + EVENT_DEVICE_NOTIFY = 38, + EVENT_MFINDEX_WRAP = 39 + ], + TRANSFER_TYPE OFFSET(16) NUMBITS(2) [ + OUT = 2, + IN = 3, + ], + TRANSFER_DIRECTION OFFSET(16) NUMBITS(1) [ + OUT = 0, + IN = 1, + ], + ENDPOINT_ID OFFSET(16) NUMBITS(4) [], + SLOT_TYPE OFFSET(16) NUMBITS(4) [], + SLOT_ID OFFSET(24) NUMBITS(8) [], + ], +]; + +#[repr(C, align(16))] +pub struct Trb { + pub address: PhysicalAddress, + pub status: u32, + pub flags: InMemoryRegister, +} + +#[repr(C, align(16))] +pub struct EventRingSegment { + address: PhysicalAddress, + // Number of TRBs supported by the ring segment. Valid values are 16 to 4096 + size: u16, + _0: u16, + _1: u32, +} + +pub struct EventRingSegmentTable { + entries: PageBox<[EventRingSegment]>, +} + +impl EventRingSegmentTable { + pub fn for_event_rings(rings: &[&EventRing]) -> Result { + let entries = PageBox::from_iter_exact(rings.iter().map(|ring| EventRingSegment { + address: ring.base(), + size: ring.capacity().try_into().unwrap(), + _0: 0, + _1: 0, + }))?; + + for entry in entries.iter() { + log::debug!("ERST... = {:#x}", entry.address); + } + + Ok(Self { entries }) + } + + pub fn physical_address(&self) -> PhysicalAddress { + unsafe { self.entries.as_physical_address() } + } + + pub fn capacity(&self) -> usize { + self.entries.len() + } +} + +pub trait GenericRing { + fn capacity(&self) -> usize; + fn base(&self) -> PhysicalAddress; +} + +struct EventRingInner { + trbs: PageBox<[MaybeUninit]>, + dequeue_index: usize, + cycle_bit: bool, +} + +struct CommandRingInner { + trbs: PageBox<[MaybeUninit]>, + enqueue_index: usize, + #[allow(unused)] + dequeue_index: usize, + cycle_bit: bool, +} + +pub struct EventRing { + inner: IrqSafeSpinlock, + capacity: usize, +} + +struct TransferRingInner { + trbs: PageBox<[MaybeUninit]>, + enqueue_index: usize, + dequeue_index: usize, + cycle_bit: bool, +} + +pub struct CommandRing { + inner: IrqSafeSpinlock, + // TODO maybe use Vec of "slots"? + completions: IrqSafeRwLock>, + completion_notify: QueueWaker, + capacity: usize, +} + +impl GenericRing for EventRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl EventRingInner { + fn try_dequeue(&mut self) -> Option { + let trb = unsafe { self.trbs[self.dequeue_index].assume_init_ref() }; + + // TRB cannot be consumed -- its cycle bit not toggled + let trb_cycle = trb.flags.matches_all(TrbFlags::CYCLE::SET); + if trb_cycle != self.cycle_bit { + return None; + } + + self.dequeue_index += 1; + + if self.dequeue_index == self.trbs.len() { + self.dequeue_index = 0; + self.cycle_bit = !self.cycle_bit; + } + + match trb.flags.read_as_enum(TrbFlags::TYPE) { + Some(TrbFlags::TYPE::Value::EVENT_PORT_CHANGE) => Some(Event::PortChange( + IntoRaw::::into_raw(trb.address) & 0xFFusize, + )), + Some(TrbFlags::TYPE::Value::EVENT_CMD_COMPLETE) => Some(Event::CommandCompletion { + address: trb.address, + reply: CommandReply { + status: trb.status, + slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, + }, + }), + Some(TrbFlags::TYPE::Value::EVENT_XFER) => Some(Event::Transfer { + address: trb.address, + slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, + endpoint_id: trb.flags.read(TrbFlags::ENDPOINT_ID) as _, + status: trb.status, + }), + e => { + log::debug!("Unknown TRB ty: {:?}", e); + None + } + } + } +} + +impl EventRing { + pub fn new(capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity)?; + + Ok(Self { + inner: IrqSafeSpinlock::new(EventRingInner { + trbs, + dequeue_index: 0, + cycle_bit: true, + }), + capacity, + }) + } + + pub fn try_dequeue(&self) -> Option { + self.inner.lock().try_dequeue() + } + + pub fn dequeue_pointer(&self) -> PhysicalAddress { + let i = self.inner.lock(); + unsafe { i.trbs.as_physical_address() }.add(i.dequeue_index * size_of::()) + } +} + +impl GenericRing for CommandRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl CommandRingInner { + fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { + trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); + self.trbs[self.enqueue_index].write(trb); + + let address = + unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); + + // Move to the next TRB slot + self.enqueue_index += 1; + if self.enqueue_index >= self.trbs.len() { + todo!(); + } + + address + } +} + +impl CommandRing { + pub fn new(capacity: usize) -> Result { + let mut trbs = PageBox::new_zeroed_slice(capacity)?; + + let base = unsafe { trbs.as_physical_address() }; + let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; + + last.address = base; + last.flags + .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); + + Ok(Self { + inner: IrqSafeSpinlock::new(CommandRingInner { + trbs, + enqueue_index: 0, + dequeue_index: 0, + cycle_bit: true, + }), + completions: IrqSafeRwLock::new(BTreeMap::new()), + completion_notify: QueueWaker::new(), + capacity, + }) + } + + pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { + let mut inner = self.inner.lock(); + let address = inner.enqueue(trb); + log::info!("CommandRing::enqueue({:#x})", address); + address + } + + pub async fn address_device( + &self, + executor: &E, + slot_id: u8, + input: &mut PageBox>, + ) -> Result<(), Error> { + let reply = self + .submit_and_wait( + executor, + Trb { + address: unsafe { input.as_physical_address() }, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::SLOT_ID.val(slot_id as _) + + TrbFlags::TYPE::COMMAND_ADDRESS_DEVICE) + .into(), + ), + }, + ) + .await; + + if reply.completion_code() == 1 { + Ok(()) + } else { + Err(Error::InvalidOperation) + } + } + + pub async fn configure_endpoint( + &self, + executor: &E, + slot_id: u8, + input: &mut PageBox>, + ) -> Result<(), Error> { + let reply = self + .submit_and_wait( + executor, + Trb { + address: unsafe { input.as_physical_address() }, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::SLOT_ID.val(slot_id as _) + + TrbFlags::TYPE::COMMAND_CONFIGURE_ENDPOINT) + .into(), + ), + }, + ) + .await; + + if reply.completion_code() == 1 { + Ok(()) + } else { + Err(Error::InvalidOperation) + } + } + + pub async fn enable_slot(&self, executor: &E) -> Result { + let reply = self + .submit_and_wait( + executor, + Trb { + address: PhysicalAddress::ZERO, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::SLOT_TYPE.val(0) + TrbFlags::TYPE::COMMAND_ENABLE_SLOT).into(), + ), + }, + ) + .await; + + if reply.completion_code() == 1 { + Ok(reply.slot_id) + } else { + Err(Error::InvalidOperation) + } + } + + pub async fn submit_and_wait( + &self, + executor: &E, + trb: Trb, + ) -> CommandReply { + let token = self.enqueue(trb); + executor.ring_doorbell(0, 0); + poll_fn(|cx| { + self.completion_notify.register(cx.waker()); + if let Some(status) = self.get_completion(token) { + self.completion_notify.remove(cx.waker()); + Poll::Ready(status) + } else { + Poll::Pending + } + }) + .await + } + + pub fn get_completion(&self, address: PhysicalAddress) -> Option { + self.completions.write().remove(&address) + } + + pub fn notify(&self, address: PhysicalAddress, reply: CommandReply) { + log::info!("CommandRing::notify({:#x}, {:#x?})", address, reply); + self.completions.write().insert(address, reply); + self.completion_notify.wake_all(); + } +} + +impl GenericRing for TransferRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl TransferRingInner { + fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { + trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); + self.trbs[self.enqueue_index].write(trb); + + let address = + unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); + + // Move to the next TRB slot + self.enqueue_index += 1; + if self.enqueue_index >= self.trbs.len() { + todo!(); + } + + address + } +} + +pub struct TransferRing { + inner: IrqSafeSpinlock, + capacity: usize, + + // TODO this is inefficient and ugly + pending_trbs: IrqSafeRwLock>, + completions: IrqSafeRwLock>>, + + slot_id: u8, + ep_id: u8, + + transfer_id: AtomicU64, +} + +impl TransferRing { + pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { + let mut trbs = PageBox::new_zeroed_slice(capacity)?; + + let base = unsafe { trbs.as_physical_address() }; + let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; + + last.address = base; + last.flags + .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); + + Ok(Self { + inner: IrqSafeSpinlock::new(TransferRingInner { + trbs, + enqueue_index: 0, + dequeue_index: 0, + cycle_bit: true, + }), + completions: IrqSafeRwLock::new(BTreeMap::new()), + pending_trbs: IrqSafeRwLock::new(BTreeMap::new()), + slot_id, + ep_id, + capacity, + + transfer_id: AtomicU64::new(0), + }) + } + + pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { + let mut inner = self.inner.lock(); + let address = inner.enqueue(trb); + address + } + + pub fn new_transfer(&self) -> UsbTransferToken { + let id = self.transfer_id.fetch_add(1, Ordering::AcqRel); + UsbTransferToken(id) + } + + pub fn complete_transfer(&self, transfer: UsbTransfer) { + let mut pending = self.pending_trbs.write(); + for trb in transfer.elements { + pending.remove(&trb); + } + self.completions.write().remove(&transfer.id); + } + + pub fn enqueue_transfer>( + &self, + trbs: I, + length: usize, + ) -> UsbTransfer { + let mut pending = self.pending_trbs.write(); + let id = self.new_transfer(); + let mut addresses = Vec::new(); + for trb in trbs { + let address = self.enqueue(trb); + addresses.push(address); + pending.insert(address, id); + } + let status = Arc::new(UsbTransferStatus::new()); + let transfer = UsbTransfer { + id, + length, + direction: UsbDirection::In, + elements: addresses, + status: status.clone(), + }; + self.completions.write().insert(id, status); + transfer + } + + fn control_setup_trb(ctl: ControlTransferSetup) -> Trb { + let word0 = (ctl.bm_request_type as u32) + | ((ctl.b_request as u32) << 8) + | ((ctl.w_value as u32) << 16); + let word1 = (ctl.w_index as u32) | ((ctl.w_length as u32) << 16); + + Trb { + address: PhysicalAddress::from_raw((word0 as u64) | ((word1 as u64) << 32)), + status: 8, + flags: InMemoryRegister::new( + (TrbFlags::TYPE::SETUP + + TrbFlags::IMMEDIATE_DATA::SET + + TrbFlags::TRANSFER_TYPE::IN) + .into(), + ), + } + } + + fn data_trb(address: PhysicalAddress, len: usize, direction: UsbDirection) -> Trb { + let dir = match direction { + UsbDirection::Out => TrbFlags::TRANSFER_DIRECTION::OUT, + UsbDirection::In => TrbFlags::TRANSFER_DIRECTION::IN, + }; + Trb { + address, + status: len.try_into().unwrap(), + flags: InMemoryRegister::new((TrbFlags::TYPE::DATA + dir).into()), + } + } + + fn control_status_trb() -> Trb { + Trb { + address: PhysicalAddress::ZERO, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::TYPE::STATUS + + TrbFlags::TRANSFER_DIRECTION::IN + + TrbFlags::INTERRUPT_ON_COMPLETION::SET) + .into(), + ), + } + } + + pub fn start_interrupt_in_transfer( + &self, + executor: &E, + buffer: &mut PageBox<[u8]>, + ) -> Result { + let data_trb = Trb { + address: unsafe { buffer.as_physical_address() }, + status: buffer.len() as _, + flags: InMemoryRegister::new( + (TrbFlags::TRANSFER_DIRECTION::IN + + TrbFlags::INTERRUPT_ON_COMPLETION::SET + + TrbFlags::TYPE::NORMAL) + .into(), + ), + }; + + let transfer = self.enqueue_transfer(core::iter::once(data_trb), buffer.len()); + + executor.ring_doorbell(self.slot_id as _, self.ep_id); + + Ok(transfer) + } + + pub fn start_control_transfer( + &self, + executor: &E, + setup: ControlTransferSetup, + buffer: Option<(PhysicalAddress, usize, UsbDirection)>, + ) -> Result { + let setup_trb = Self::control_setup_trb(setup); + let data_trb = buffer.map(|(address, len, dir)| Self::data_trb(address, len, dir)); + let status_trb = Self::control_status_trb(); + + let transfer = self.enqueue_transfer( + core::iter::once(setup_trb) + .chain(data_trb) + .chain(core::iter::once(status_trb)), + 0, + ); + + executor.ring_doorbell(self.slot_id as _, self.ep_id); + + Ok(transfer) + } + + pub fn dequeue_pointer(&self) -> PhysicalAddress { + let inner = self.inner.lock(); + unsafe { inner.trbs.as_physical_address() }.add(inner.dequeue_index * size_of::()) + } + + pub fn notify(&self, address: PhysicalAddress, value: u32) { + if value == 0 { + return; + } + + let completions = self.completions.read(); + if let Some(&token) = self.pending_trbs.read().get(&address) { + let Some(status) = completions.get(&token) else { + log::warn!( + "Notification received for non-existent transfer: {:?}", + token + ); + return; + }; + + status.signal(value); + } + } +} diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 7a76bb9f..43789c13 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -3,7 +3,14 @@ #![cfg_attr(not(test), no_std)] #![allow(clippy::new_ret_no_self, clippy::new_without_default)] #![deny(missing_docs)] -#![feature(if_let_guard, maybe_uninit_slice, trait_alias, let_chains, new_uninit)] +#![feature( + if_let_guard, + maybe_uninit_slice, + trait_alias, + let_chains, + new_uninit, + trait_upcasting +)] #[cfg(test)] extern crate hosted_tests; diff --git a/libk/libk-mm/src/lib.rs b/libk/libk-mm/src/lib.rs index 106c961f..e40e32ea 100644 --- a/libk/libk-mm/src/lib.rs +++ b/libk/libk-mm/src/lib.rs @@ -4,6 +4,7 @@ slice_ptr_get, step_trait, const_trait_impl, + maybe_uninit_as_bytes, effects )] #![no_std] @@ -66,11 +67,17 @@ pub struct PageBox { impl PageBox { #[inline] - fn alloc_slice(count: usize) -> Result<(PhysicalAddress, usize), Error> { + fn alloc_slice(count: usize, zeroed: bool) -> Result<(PhysicalAddress, usize), Error> { // TODO hardcoded page sizes let layout = Layout::array::(count).unwrap(); let page_count = (layout.size() + L3_PAGE_SIZE - 1) / L3_PAGE_SIZE; - Ok((phys::alloc_pages_contiguous(page_count)?, page_count)) + let base = phys::alloc_pages_contiguous(page_count)?; + if zeroed { + let ptr = base.virtualize() as *mut u8; + let slice = unsafe { core::slice::from_raw_parts_mut(ptr, page_count * L3_PAGE_SIZE) }; + slice.fill(0); + } + Ok((base, page_count)) } #[inline] @@ -96,7 +103,7 @@ impl PageBox { where T: Copy, { - let (base, page_count) = Self::alloc_slice(count)?; + let (base, page_count) = Self::alloc_slice(count, false)?; let base_virt_ptr = base.virtualize() as *mut T; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); @@ -130,7 +137,16 @@ impl PageBox { } pub fn new_uninit_slice(count: usize) -> Result]>, Error> { - let (base, page_count) = PageBox::>::alloc_slice(count)?; + let (base, page_count) = PageBox::>::alloc_slice(count, false)?; + let base_virt_ptr = base.virtualize() as *mut MaybeUninit; + let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); + let result = PageBox { value, page_count }; + result.trace_created(); + Ok(result) + } + + pub fn new_zeroed_slice(count: usize) -> Result]>, Error> { + let (base, page_count) = PageBox::>::alloc_slice(count, true)?; let base_virt_ptr = base.virtualize() as *mut MaybeUninit; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); let result = PageBox { value, page_count }; @@ -166,6 +182,21 @@ impl PageBox { } } +impl PageBox<[T]> { + pub fn from_iter_exact>(it: I) -> Result + where + I::IntoIter: ExactSizeIterator, + { + let it = it.into_iter(); + let mut slice = PageBox::new_uninit_slice(it.len())?; + for (i, item) in it.enumerate() { + slice[i].write(item); + } + let slice = unsafe { slice.assume_init_slice() }; + Ok(slice) + } +} + impl PageBox> { /// Consumes the [PageBox], returning a new one with [MaybeUninit] removed. /// @@ -184,6 +215,15 @@ impl PageBox> { PageBox { value, page_count } } + + pub unsafe fn into_byte_slice(self) -> PageBox<[u8]> { + let page_count = self.page_count; + let value = MaybeUninit::slice_assume_init_mut(MaybeUninit::as_bytes_mut(&mut *self.value)); + + core::mem::forget(self); + + PageBox { value, page_count } + } } impl PageBox<[MaybeUninit]> { @@ -221,6 +261,18 @@ impl PageBox<[MaybeUninit]> { pub unsafe fn assume_init_slice_mut(&mut self) -> &mut [T] { MaybeUninit::slice_assume_init_mut(self.deref_mut()) } + + /// Fills a slice of MaybeUninit with zeroes. + /// + /// # Safety + /// + /// Unsafe: will not drop possibly previously written data. Only meant for [Copy] and other + /// trivial types. + pub unsafe fn zero(p: &mut Self) { + let ptr = p.as_mut_ptr() as *mut u8; + let slice = core::slice::from_raw_parts_mut(ptr, p.page_count * L3_PAGE_SIZE); + slice.fill(0); + } } impl AsPhysicalAddress for PageBox { diff --git a/libk/libk-thread/src/lib.rs b/libk/libk-thread/src/lib.rs index ec92f2e8..6360d83e 100644 --- a/libk/libk-thread/src/lib.rs +++ b/libk/libk-thread/src/lib.rs @@ -12,7 +12,7 @@ extern crate alloc; use api::__signal_process_group; -use kernel_arch::KernelTableManagerImpl; +use kernel_arch::{Architecture, ArchitectureImpl, KernelTableManagerImpl}; use libk_mm::phys::GlobalPhysicalAllocator; pub(crate) mod api { @@ -38,10 +38,21 @@ pub mod types; pub type TaskContextImpl = kernel_arch::TaskContextImpl; +use sched::CpuQueue; pub use types::{AtomicThreadState, ThreadAffinity, ThreadId, ThreadState}; use yggdrasil_abi::process::Signal; +/// Returns local CPU index +#[inline] +pub fn cpu_index() -> u32 { + ArchitectureImpl::cpu_index::() +} + +pub fn cpu_count() -> usize { + ArchitectureImpl::cpu_count() +} + pub fn signal_process_group(group_id: u32, signal: Signal) { unsafe { __signal_process_group(group_id, signal) } } diff --git a/libk/libk-thread/src/sync.rs b/libk/libk-thread/src/sync.rs index dc85dec3..175be23a 100644 --- a/libk/libk-thread/src/sync.rs +++ b/libk/libk-thread/src/sync.rs @@ -1,13 +1,15 @@ use core::{ cell::UnsafeCell, + future::poll_fn, ops::{Deref, DerefMut}, - sync::atomic::{AtomicU32, Ordering}, + sync::atomic::{AtomicBool, AtomicU32, Ordering}, + task::{Context, Poll}, }; use alloc::sync::Arc; use crossbeam_queue::ArrayQueue; use kernel_arch::task::Scheduler; -use libk_util::sync::LockMethod; +use libk_util::{sync::LockMethod, waker::QueueWaker}; use yggdrasil_abi::error::Error; use crate::{sched::CpuQueue, thread::Thread}; @@ -17,6 +19,16 @@ struct ThreadedMutexInner { lock: AtomicU32, } +pub struct AsyncMutex { + value: UnsafeCell, + waker: QueueWaker, + lock: AtomicBool, +} + +pub struct AsyncMutexGuard<'a, T> { + mutex: &'a AsyncMutex, +} + pub struct Mutex { value: UnsafeCell, lock: ThreadedMutexInner, @@ -27,6 +39,74 @@ pub struct MutexGuard<'a, T> { lock: &'a ThreadedMutexInner, } +impl AsyncMutex { + pub fn new(value: T) -> Self { + Self { + value: UnsafeCell::new(value), + waker: QueueWaker::new(), + lock: AtomicBool::new(false), + } + } + + pub fn try_lock(&self) -> bool { + self.lock + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } + + pub fn poll_lock(&self, cx: &mut Context<'_>) -> Poll> { + self.waker.register(cx.waker()); + + if self.try_lock() { + self.waker.remove(cx.waker()); + return Poll::Ready(AsyncMutexGuard { mutex: self }); + } + + Poll::Pending + } + + pub async fn lock(&self) -> AsyncMutexGuard { + poll_fn(|cx| self.poll_lock(cx)).await + } + + pub unsafe fn force_unlock(&self) { + self.lock.store(false, Ordering::Release); + self.waker.wake_one(); + } + + #[inline] + pub fn get(&self) -> *mut T { + self.value.get() + } +} + +unsafe impl Sync for AsyncMutex {} + +unsafe impl<'a, T> Send for AsyncMutexGuard<'a, T> {} +unsafe impl<'a, T> Sync for AsyncMutexGuard<'a, T> {} + +impl<'a, T> Deref for AsyncMutexGuard<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.mutex.get() } + } +} + +impl<'a, T> DerefMut for AsyncMutexGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *self.mutex.get() } + } +} + +impl<'a, T> Drop for AsyncMutexGuard<'a, T> { + fn drop(&mut self) { + unsafe { + self.mutex.force_unlock(); + } + } +} + impl ThreadedMutexInner { const UNLOCKED: u32 = 0; const LOCKED: u32 = 1; diff --git a/libk/src/lib.rs b/libk/src/lib.rs index c035bb55..4c65d5ed 100644 --- a/libk/src/lib.rs +++ b/libk/src/lib.rs @@ -16,12 +16,9 @@ trait_alias )] -use arch::Cpu; -use kernel_arch::{Architecture, ArchitectureImpl, CpuImpl}; - extern crate alloc; -pub use libk_thread::{block, runtime}; +pub use libk_thread::{block, cpu_count, cpu_index, runtime}; pub mod arch; @@ -29,17 +26,6 @@ pub mod device { pub use libk_device::*; } -#[inline] -pub fn cpu_count() -> usize { - ArchitectureImpl::cpu_count() -} - -/// Returns local CPU index -#[inline] -pub fn cpu_index() -> u32 { - Cpu::try_local().as_deref().map(CpuImpl::id).unwrap_or(0) -} - #[repr(C)] pub struct AlignedTo { pub align: [Align; 0], diff --git a/src/arch/x86_64/apic/ioapic.rs b/src/arch/x86_64/apic/ioapic.rs index 1861a0cc..80f4f1ad 100644 --- a/src/arch/x86_64/apic/ioapic.rs +++ b/src/arch/x86_64/apic/ioapic.rs @@ -198,14 +198,10 @@ impl ExternalInterruptController for IoApic { ) } else { // Directly mapped to a GSI - ( - (irq - ISA_IRQ_OFFSET) as u32, - options.level, - options.trigger, - ) + (irq - ISA_IRQ_OFFSET, options.level, options.trigger) } } - Irq::External(irq) => (irq as u32, options.level, options.trigger), + Irq::External(irq) => (irq, options.level, options.trigger), Irq::Private(_) => unimplemented!(), }; diff --git a/src/arch/x86_64/apic/local.rs b/src/arch/x86_64/apic/local.rs index ddc93fd4..0ce7443a 100644 --- a/src/arch/x86_64/apic/local.rs +++ b/src/arch/x86_64/apic/local.rs @@ -225,9 +225,7 @@ impl MessageInterruptController for LocalApic { }; drop(table); - if handler.handle_irq(Some(vector)) { - break; - } + handler.handle_irq(Some(vector)); i += 1; } diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 673a195d..db5dc6f9 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -4,10 +4,7 @@ use core::{mem::size_of, ops::DerefMut, ptr::null_mut, sync::atomic::Ordering}; use abi::error::Error; use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel}; use alloc::boxed::Box; -use device_api::{ - interrupt::{Irq, MessageInterruptController}, - Device, -}; +use device_api::{interrupt::Irq, Device}; use git_version::git_version; use kernel_arch_x86_64::{ mem::{ @@ -305,6 +302,13 @@ impl X86_64 { Some(0x01), ygg_driver_ahci::probe, ); + ygg_driver_pci::register_class_driver( + "USB xHCI", + 0x0C, + Some(0x03), + Some(0x30), + ygg_driver_usb_xhci::probe, + ); ygg_driver_pci::register_vendor_driver( "Virtio PCI Network Device", 0x1AF4, @@ -487,8 +491,3 @@ impl X86_64 { pic_slave_cmd.write(0x20); } } - -#[no_mangle] -fn __message_interrupt_controller() -> &'static dyn MessageInterruptController { - Cpu::local().local_apic -} diff --git a/src/arch/x86_64/peripherals/ps2/mod.rs b/src/arch/x86_64/peripherals/ps2/mod.rs index b70005a9..cf9315c3 100644 --- a/src/arch/x86_64/peripherals/ps2/mod.rs +++ b/src/arch/x86_64/peripherals/ps2/mod.rs @@ -10,12 +10,9 @@ use device_api::{ use libk::device::external_interrupt_controller; use libk_util::sync::IrqSafeSpinlock; -use crate::{ - arch::x86_64::{ - intrinsics::{IoPort, IoPortAccess}, - peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, - }, - device::input, +use crate::arch::x86_64::{ + intrinsics::{IoPort, IoPortAccess}, + peripherals::ps2::codeset::{CODE_SET_1_00, CODE_SET_1_E0}, }; mod codeset; @@ -102,7 +99,7 @@ impl InterruptHandler for PS2Controller { KeyboardKeyEvent::Pressed(key) }; - input::send_event(event); + ygg_driver_input::send_event(event); } count != 0 diff --git a/src/device/mod.rs b/src/device/mod.rs index 7046c1c3..6fd64d9e 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -6,7 +6,6 @@ use libk_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; pub mod bus; pub mod display; -pub mod input; pub mod power; pub mod serial; pub mod timer; diff --git a/src/init.rs b/src/init.rs index 2855261d..74c744e2 100644 --- a/src/init.rs +++ b/src/init.rs @@ -8,7 +8,6 @@ use memfs::MemoryFilesystem; use vfs::{IoContext, NodeRef}; use crate::{ - device::input, fs::{FileBlockAllocator, INITRD_DATA}, proc::{self, random}, }; @@ -28,6 +27,8 @@ fn setup_root() -> Result { pub fn kinit() -> Result<(), Error> { infoln!("In main"); + runtime::spawn(ygg_driver_usb::bus::bus_handler())?; + ygg_driver_net_loopback::init(); ygg_driver_net_core::start_network_tasks()?; @@ -41,7 +42,7 @@ pub fn kinit() -> Result<(), Error> { } // Add keyboard device - devfs::add_named_char_device(&input::KEYBOARD_DEVICE, "kbd".to_owned())?; + devfs::add_named_char_device(&ygg_driver_input::KEYBOARD_DEVICE, "kbd".to_owned())?; random::init(); diff --git a/tools/gentables/src/main.rs b/tools/gentables/src/main.rs index 7edfcd9c..3ec366ff 100644 --- a/tools/gentables/src/main.rs +++ b/tools/gentables/src/main.rs @@ -1,5 +1,3 @@ -#![feature(offset_of)] - use std::{ fs::OpenOptions, io::{Read, Seek, SeekFrom, Write}, From b047c9410f242dfd984bc1365c627dcb8172ee80 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 27 Feb 2024 00:39:08 +0200 Subject: [PATCH 195/211] usb/xhci: fix improper clearing of interrupt status --- driver/usb/xhci/src/regs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver/usb/xhci/src/regs.rs b/driver/usb/xhci/src/regs.rs index 55bf11ba..d3b22a21 100644 --- a/driver/usb/xhci/src/regs.rs +++ b/driver/usb/xhci/src/regs.rs @@ -195,7 +195,7 @@ impl Regs { // Acknowledge interrupts let mut intr0 = i.interrupter_mut(0); intr0.iman.update_volatile(|u| { - u.set_0_interrupt_pending(); + u.clear_interrupt_pending(); }); Some(status) From ce3ee30b57fbb0c05a9753e3313843c29f6e0eed Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 27 Feb 2024 00:39:54 +0200 Subject: [PATCH 196/211] aarch64: add xHCI to aarch64 --- arch/aarch64/src/lib.rs | 12 +++++++++++- src/arch/aarch64/mod.rs | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/arch/aarch64/src/lib.rs b/arch/aarch64/src/lib.rs index e4d218b8..e5505b9d 100644 --- a/arch/aarch64/src/lib.rs +++ b/arch/aarch64/src/lib.rs @@ -1,5 +1,11 @@ #![no_std] -#![feature(effects, strict_provenance, asm_const, naked_functions)] +#![feature( + effects, + strict_provenance, + asm_const, + naked_functions, + trait_upcasting +)] extern crate alloc; @@ -49,6 +55,10 @@ impl ArchitectureImpl { impl Architecture for ArchitectureImpl { type PerCpuData = PerCpuData; + fn cpu_index() -> u32 { + (MPIDR_EL1.get() & 0xFF) as u32 + } + fn interrupt_mask() -> bool { DAIF.read(DAIF::I) != 0 } diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index ded07213..5144d879 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -251,6 +251,13 @@ impl AArch64 { Some(0x01), ygg_driver_ahci::probe, ); + ygg_driver_pci::register_class_driver( + "USB xHCI", + 0x0C, + Some(0x03), + Some(0x30), + ygg_driver_usb_xhci::probe, + ); let dt = self.dt.get(); From d7cefb5a7b4392913cf34e7b9aa7f74e330bb708 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 27 Feb 2024 14:26:51 +0200 Subject: [PATCH 197/211] usb/xhci: split ring structs into their modules --- driver/bus/usb/src/class_driver/mod.rs | 1 + driver/usb/xhci/src/ring.rs | 656 ------------------------- driver/usb/xhci/src/ring/command.rs | 215 ++++++++ driver/usb/xhci/src/ring/event.rs | 146 ++++++ driver/usb/xhci/src/ring/mod.rs | 72 +++ driver/usb/xhci/src/ring/transfer.rs | 256 ++++++++++ 6 files changed, 690 insertions(+), 656 deletions(-) delete mode 100644 driver/usb/xhci/src/ring.rs create mode 100644 driver/usb/xhci/src/ring/command.rs create mode 100644 driver/usb/xhci/src/ring/event.rs create mode 100644 driver/usb/xhci/src/ring/mod.rs create mode 100644 driver/usb/xhci/src/ring/transfer.rs diff --git a/driver/bus/usb/src/class_driver/mod.rs b/driver/bus/usb/src/class_driver/mod.rs index 21823a05..ea76d297 100644 --- a/driver/bus/usb/src/class_driver/mod.rs +++ b/driver/bus/usb/src/class_driver/mod.rs @@ -197,6 +197,7 @@ pub async fn keyboard_driver(device: Arc) -> Result<(), Error> let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) }; for &event in events { + log::debug!("Generic Keyboard: {:?}", event); ygg_driver_input::send_event(event); } } diff --git a/driver/usb/xhci/src/ring.rs b/driver/usb/xhci/src/ring.rs deleted file mode 100644 index 0c781cf9..00000000 --- a/driver/usb/xhci/src/ring.rs +++ /dev/null @@ -1,656 +0,0 @@ -use core::{ - future::poll_fn, - mem::{size_of, MaybeUninit}, - sync::atomic::{AtomicU64, Ordering}, - task::Poll, -}; - -use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; -use libk_mm::{ - address::{AsPhysicalAddress, FromRaw, IntoRaw, PhysicalAddress}, - PageBox, -}; -use libk_util::{ - sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, - waker::QueueWaker, -}; -use tock_registers::{ - interfaces::{ReadWriteable, Readable, Writeable}, - register_bitfields, - registers::InMemoryRegister, -}; -use xhci_lib::context::Input; -use ygg_driver_usb::{ - pipe::control::ControlTransferSetup, UsbDirection, UsbTransfer, UsbTransferStatus, - UsbTransferToken, -}; -use yggdrasil_abi::error::Error; - -#[derive(Debug, Clone, Copy)] -pub struct CommandReply { - pub status: u32, - pub slot_id: u8, -} - -impl CommandReply { - pub fn completion_code(&self) -> u8 { - (self.status >> 24) as u8 - } -} - -pub enum Event { - PortChange(usize), - CommandCompletion { - address: PhysicalAddress, - reply: CommandReply, - }, - Transfer { - address: PhysicalAddress, - slot_id: u8, - endpoint_id: u8, - status: u32, - }, -} - -pub trait CommandExecutor { - fn ring_doorbell(&self, index: usize, target: u8); -} - -register_bitfields![ - u32, - pub TrbFlags [ - CYCLE OFFSET(0) NUMBITS(1) [], - LINKSEG OFFSET(1) NUMBITS(1) [], - INTERRUPT_ON_COMPLETION OFFSET(5) NUMBITS(1) [], - IMMEDIATE_DATA OFFSET(6) NUMBITS(1) [], - TYPE OFFSET(10) NUMBITS(6) [ - // Transfer ring types - NORMAL = 1, - SETUP = 2, - DATA = 3, - STATUS = 4, - ISOCH = 5, - LINK = 6, - EVENT = 7, - NOOP = 8, - // Command ring types - COMMAND_ENABLE_SLOT = 9, - COMMAND_DISABLE_SLOT = 10, - COMMAND_ADDRESS_DEVICE = 11, - COMMAND_CONFIGURE_ENDPOINT = 12, - // Event ring types - EVENT_XFER = 32, - EVENT_CMD_COMPLETE = 33, - EVENT_PORT_CHANGE = 34, - EVENT_BW_REQUEST = 35, - EVENT_DOORBELL = 36, - EVENT_HOST_CTRL = 37, - EVENT_DEVICE_NOTIFY = 38, - EVENT_MFINDEX_WRAP = 39 - ], - TRANSFER_TYPE OFFSET(16) NUMBITS(2) [ - OUT = 2, - IN = 3, - ], - TRANSFER_DIRECTION OFFSET(16) NUMBITS(1) [ - OUT = 0, - IN = 1, - ], - ENDPOINT_ID OFFSET(16) NUMBITS(4) [], - SLOT_TYPE OFFSET(16) NUMBITS(4) [], - SLOT_ID OFFSET(24) NUMBITS(8) [], - ], -]; - -#[repr(C, align(16))] -pub struct Trb { - pub address: PhysicalAddress, - pub status: u32, - pub flags: InMemoryRegister, -} - -#[repr(C, align(16))] -pub struct EventRingSegment { - address: PhysicalAddress, - // Number of TRBs supported by the ring segment. Valid values are 16 to 4096 - size: u16, - _0: u16, - _1: u32, -} - -pub struct EventRingSegmentTable { - entries: PageBox<[EventRingSegment]>, -} - -impl EventRingSegmentTable { - pub fn for_event_rings(rings: &[&EventRing]) -> Result { - let entries = PageBox::from_iter_exact(rings.iter().map(|ring| EventRingSegment { - address: ring.base(), - size: ring.capacity().try_into().unwrap(), - _0: 0, - _1: 0, - }))?; - - for entry in entries.iter() { - log::debug!("ERST... = {:#x}", entry.address); - } - - Ok(Self { entries }) - } - - pub fn physical_address(&self) -> PhysicalAddress { - unsafe { self.entries.as_physical_address() } - } - - pub fn capacity(&self) -> usize { - self.entries.len() - } -} - -pub trait GenericRing { - fn capacity(&self) -> usize; - fn base(&self) -> PhysicalAddress; -} - -struct EventRingInner { - trbs: PageBox<[MaybeUninit]>, - dequeue_index: usize, - cycle_bit: bool, -} - -struct CommandRingInner { - trbs: PageBox<[MaybeUninit]>, - enqueue_index: usize, - #[allow(unused)] - dequeue_index: usize, - cycle_bit: bool, -} - -pub struct EventRing { - inner: IrqSafeSpinlock, - capacity: usize, -} - -struct TransferRingInner { - trbs: PageBox<[MaybeUninit]>, - enqueue_index: usize, - dequeue_index: usize, - cycle_bit: bool, -} - -pub struct CommandRing { - inner: IrqSafeSpinlock, - // TODO maybe use Vec of "slots"? - completions: IrqSafeRwLock>, - completion_notify: QueueWaker, - capacity: usize, -} - -impl GenericRing for EventRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity - } -} - -impl EventRingInner { - fn try_dequeue(&mut self) -> Option { - let trb = unsafe { self.trbs[self.dequeue_index].assume_init_ref() }; - - // TRB cannot be consumed -- its cycle bit not toggled - let trb_cycle = trb.flags.matches_all(TrbFlags::CYCLE::SET); - if trb_cycle != self.cycle_bit { - return None; - } - - self.dequeue_index += 1; - - if self.dequeue_index == self.trbs.len() { - self.dequeue_index = 0; - self.cycle_bit = !self.cycle_bit; - } - - match trb.flags.read_as_enum(TrbFlags::TYPE) { - Some(TrbFlags::TYPE::Value::EVENT_PORT_CHANGE) => Some(Event::PortChange( - IntoRaw::::into_raw(trb.address) & 0xFFusize, - )), - Some(TrbFlags::TYPE::Value::EVENT_CMD_COMPLETE) => Some(Event::CommandCompletion { - address: trb.address, - reply: CommandReply { - status: trb.status, - slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, - }, - }), - Some(TrbFlags::TYPE::Value::EVENT_XFER) => Some(Event::Transfer { - address: trb.address, - slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, - endpoint_id: trb.flags.read(TrbFlags::ENDPOINT_ID) as _, - status: trb.status, - }), - e => { - log::debug!("Unknown TRB ty: {:?}", e); - None - } - } - } -} - -impl EventRing { - pub fn new(capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity)?; - - Ok(Self { - inner: IrqSafeSpinlock::new(EventRingInner { - trbs, - dequeue_index: 0, - cycle_bit: true, - }), - capacity, - }) - } - - pub fn try_dequeue(&self) -> Option { - self.inner.lock().try_dequeue() - } - - pub fn dequeue_pointer(&self) -> PhysicalAddress { - let i = self.inner.lock(); - unsafe { i.trbs.as_physical_address() }.add(i.dequeue_index * size_of::()) - } -} - -impl GenericRing for CommandRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity - } -} - -impl CommandRingInner { - fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { - trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); - self.trbs[self.enqueue_index].write(trb); - - let address = - unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); - - // Move to the next TRB slot - self.enqueue_index += 1; - if self.enqueue_index >= self.trbs.len() { - todo!(); - } - - address - } -} - -impl CommandRing { - pub fn new(capacity: usize) -> Result { - let mut trbs = PageBox::new_zeroed_slice(capacity)?; - - let base = unsafe { trbs.as_physical_address() }; - let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; - - last.address = base; - last.flags - .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); - - Ok(Self { - inner: IrqSafeSpinlock::new(CommandRingInner { - trbs, - enqueue_index: 0, - dequeue_index: 0, - cycle_bit: true, - }), - completions: IrqSafeRwLock::new(BTreeMap::new()), - completion_notify: QueueWaker::new(), - capacity, - }) - } - - pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { - let mut inner = self.inner.lock(); - let address = inner.enqueue(trb); - log::info!("CommandRing::enqueue({:#x})", address); - address - } - - pub async fn address_device( - &self, - executor: &E, - slot_id: u8, - input: &mut PageBox>, - ) -> Result<(), Error> { - let reply = self - .submit_and_wait( - executor, - Trb { - address: unsafe { input.as_physical_address() }, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::SLOT_ID.val(slot_id as _) - + TrbFlags::TYPE::COMMAND_ADDRESS_DEVICE) - .into(), - ), - }, - ) - .await; - - if reply.completion_code() == 1 { - Ok(()) - } else { - Err(Error::InvalidOperation) - } - } - - pub async fn configure_endpoint( - &self, - executor: &E, - slot_id: u8, - input: &mut PageBox>, - ) -> Result<(), Error> { - let reply = self - .submit_and_wait( - executor, - Trb { - address: unsafe { input.as_physical_address() }, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::SLOT_ID.val(slot_id as _) - + TrbFlags::TYPE::COMMAND_CONFIGURE_ENDPOINT) - .into(), - ), - }, - ) - .await; - - if reply.completion_code() == 1 { - Ok(()) - } else { - Err(Error::InvalidOperation) - } - } - - pub async fn enable_slot(&self, executor: &E) -> Result { - let reply = self - .submit_and_wait( - executor, - Trb { - address: PhysicalAddress::ZERO, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::SLOT_TYPE.val(0) + TrbFlags::TYPE::COMMAND_ENABLE_SLOT).into(), - ), - }, - ) - .await; - - if reply.completion_code() == 1 { - Ok(reply.slot_id) - } else { - Err(Error::InvalidOperation) - } - } - - pub async fn submit_and_wait( - &self, - executor: &E, - trb: Trb, - ) -> CommandReply { - let token = self.enqueue(trb); - executor.ring_doorbell(0, 0); - poll_fn(|cx| { - self.completion_notify.register(cx.waker()); - if let Some(status) = self.get_completion(token) { - self.completion_notify.remove(cx.waker()); - Poll::Ready(status) - } else { - Poll::Pending - } - }) - .await - } - - pub fn get_completion(&self, address: PhysicalAddress) -> Option { - self.completions.write().remove(&address) - } - - pub fn notify(&self, address: PhysicalAddress, reply: CommandReply) { - log::info!("CommandRing::notify({:#x}, {:#x?})", address, reply); - self.completions.write().insert(address, reply); - self.completion_notify.wake_all(); - } -} - -impl GenericRing for TransferRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity - } -} - -impl TransferRingInner { - fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { - trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); - self.trbs[self.enqueue_index].write(trb); - - let address = - unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); - - // Move to the next TRB slot - self.enqueue_index += 1; - if self.enqueue_index >= self.trbs.len() { - todo!(); - } - - address - } -} - -pub struct TransferRing { - inner: IrqSafeSpinlock, - capacity: usize, - - // TODO this is inefficient and ugly - pending_trbs: IrqSafeRwLock>, - completions: IrqSafeRwLock>>, - - slot_id: u8, - ep_id: u8, - - transfer_id: AtomicU64, -} - -impl TransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let mut trbs = PageBox::new_zeroed_slice(capacity)?; - - let base = unsafe { trbs.as_physical_address() }; - let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; - - last.address = base; - last.flags - .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); - - Ok(Self { - inner: IrqSafeSpinlock::new(TransferRingInner { - trbs, - enqueue_index: 0, - dequeue_index: 0, - cycle_bit: true, - }), - completions: IrqSafeRwLock::new(BTreeMap::new()), - pending_trbs: IrqSafeRwLock::new(BTreeMap::new()), - slot_id, - ep_id, - capacity, - - transfer_id: AtomicU64::new(0), - }) - } - - pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { - let mut inner = self.inner.lock(); - let address = inner.enqueue(trb); - address - } - - pub fn new_transfer(&self) -> UsbTransferToken { - let id = self.transfer_id.fetch_add(1, Ordering::AcqRel); - UsbTransferToken(id) - } - - pub fn complete_transfer(&self, transfer: UsbTransfer) { - let mut pending = self.pending_trbs.write(); - for trb in transfer.elements { - pending.remove(&trb); - } - self.completions.write().remove(&transfer.id); - } - - pub fn enqueue_transfer>( - &self, - trbs: I, - length: usize, - ) -> UsbTransfer { - let mut pending = self.pending_trbs.write(); - let id = self.new_transfer(); - let mut addresses = Vec::new(); - for trb in trbs { - let address = self.enqueue(trb); - addresses.push(address); - pending.insert(address, id); - } - let status = Arc::new(UsbTransferStatus::new()); - let transfer = UsbTransfer { - id, - length, - direction: UsbDirection::In, - elements: addresses, - status: status.clone(), - }; - self.completions.write().insert(id, status); - transfer - } - - fn control_setup_trb(ctl: ControlTransferSetup) -> Trb { - let word0 = (ctl.bm_request_type as u32) - | ((ctl.b_request as u32) << 8) - | ((ctl.w_value as u32) << 16); - let word1 = (ctl.w_index as u32) | ((ctl.w_length as u32) << 16); - - Trb { - address: PhysicalAddress::from_raw((word0 as u64) | ((word1 as u64) << 32)), - status: 8, - flags: InMemoryRegister::new( - (TrbFlags::TYPE::SETUP - + TrbFlags::IMMEDIATE_DATA::SET - + TrbFlags::TRANSFER_TYPE::IN) - .into(), - ), - } - } - - fn data_trb(address: PhysicalAddress, len: usize, direction: UsbDirection) -> Trb { - let dir = match direction { - UsbDirection::Out => TrbFlags::TRANSFER_DIRECTION::OUT, - UsbDirection::In => TrbFlags::TRANSFER_DIRECTION::IN, - }; - Trb { - address, - status: len.try_into().unwrap(), - flags: InMemoryRegister::new((TrbFlags::TYPE::DATA + dir).into()), - } - } - - fn control_status_trb() -> Trb { - Trb { - address: PhysicalAddress::ZERO, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::TYPE::STATUS - + TrbFlags::TRANSFER_DIRECTION::IN - + TrbFlags::INTERRUPT_ON_COMPLETION::SET) - .into(), - ), - } - } - - pub fn start_interrupt_in_transfer( - &self, - executor: &E, - buffer: &mut PageBox<[u8]>, - ) -> Result { - let data_trb = Trb { - address: unsafe { buffer.as_physical_address() }, - status: buffer.len() as _, - flags: InMemoryRegister::new( - (TrbFlags::TRANSFER_DIRECTION::IN - + TrbFlags::INTERRUPT_ON_COMPLETION::SET - + TrbFlags::TYPE::NORMAL) - .into(), - ), - }; - - let transfer = self.enqueue_transfer(core::iter::once(data_trb), buffer.len()); - - executor.ring_doorbell(self.slot_id as _, self.ep_id); - - Ok(transfer) - } - - pub fn start_control_transfer( - &self, - executor: &E, - setup: ControlTransferSetup, - buffer: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { - let setup_trb = Self::control_setup_trb(setup); - let data_trb = buffer.map(|(address, len, dir)| Self::data_trb(address, len, dir)); - let status_trb = Self::control_status_trb(); - - let transfer = self.enqueue_transfer( - core::iter::once(setup_trb) - .chain(data_trb) - .chain(core::iter::once(status_trb)), - 0, - ); - - executor.ring_doorbell(self.slot_id as _, self.ep_id); - - Ok(transfer) - } - - pub fn dequeue_pointer(&self) -> PhysicalAddress { - let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() }.add(inner.dequeue_index * size_of::()) - } - - pub fn notify(&self, address: PhysicalAddress, value: u32) { - if value == 0 { - return; - } - - let completions = self.completions.read(); - if let Some(&token) = self.pending_trbs.read().get(&address) { - let Some(status) = completions.get(&token) else { - log::warn!( - "Notification received for non-existent transfer: {:?}", - token - ); - return; - }; - - status.signal(value); - } - } -} diff --git a/driver/usb/xhci/src/ring/command.rs b/driver/usb/xhci/src/ring/command.rs new file mode 100644 index 00000000..12192050 --- /dev/null +++ b/driver/usb/xhci/src/ring/command.rs @@ -0,0 +1,215 @@ +use core::{ + future::poll_fn, + mem::{size_of, MaybeUninit}, + task::Poll, +}; + +use alloc::collections::BTreeMap; +use libk_mm::{ + address::{AsPhysicalAddress, PhysicalAddress}, + PageBox, +}; +use libk_util::{ + sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, + waker::QueueWaker, +}; +use tock_registers::{ + interfaces::{ReadWriteable, Writeable}, + registers::InMemoryRegister, +}; +use xhci_lib::context; +use yggdrasil_abi::error::Error; + +use super::{CommandExecutor, GenericRing, Trb, TrbFlags}; + +#[derive(Debug, Clone, Copy)] +pub struct CommandReply { + pub status: u32, + pub slot_id: u8, +} + +impl CommandReply { + pub fn completion_code(&self) -> u8 { + (self.status >> 24) as u8 + } +} + +struct CommandRingInner { + trbs: PageBox<[MaybeUninit]>, + enqueue_index: usize, + #[allow(unused)] + dequeue_index: usize, + cycle_bit: bool, +} + +pub struct CommandRing { + inner: IrqSafeSpinlock, + // TODO maybe use Vec of "slots"? + completions: IrqSafeRwLock>, + completion_notify: QueueWaker, + capacity: usize, +} + +impl GenericRing for CommandRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl CommandRingInner { + fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { + trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); + self.trbs[self.enqueue_index].write(trb); + + let address = + unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); + + // Move to the next TRB slot + self.enqueue_index += 1; + if self.enqueue_index >= self.trbs.len() { + todo!(); + } + + address + } +} + +impl CommandRing { + pub fn new(capacity: usize) -> Result { + let mut trbs = PageBox::new_zeroed_slice(capacity)?; + + let base = unsafe { trbs.as_physical_address() }; + let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; + + last.address = base; + last.flags + .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); + + Ok(Self { + inner: IrqSafeSpinlock::new(CommandRingInner { + trbs, + enqueue_index: 0, + dequeue_index: 0, + cycle_bit: true, + }), + completions: IrqSafeRwLock::new(BTreeMap::new()), + completion_notify: QueueWaker::new(), + capacity, + }) + } + + pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { + let mut inner = self.inner.lock(); + let address = inner.enqueue(trb); + address + } + + pub async fn address_device( + &self, + executor: &E, + slot_id: u8, + input: &mut PageBox>, + ) -> Result<(), Error> { + let reply = self + .submit_and_wait( + executor, + Trb { + address: unsafe { input.as_physical_address() }, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::SLOT_ID.val(slot_id as _) + + TrbFlags::TYPE::COMMAND_ADDRESS_DEVICE) + .into(), + ), + }, + ) + .await; + + if reply.completion_code() == 1 { + Ok(()) + } else { + Err(Error::InvalidOperation) + } + } + + pub async fn configure_endpoint( + &self, + executor: &E, + slot_id: u8, + input: &mut PageBox>, + ) -> Result<(), Error> { + let reply = self + .submit_and_wait( + executor, + Trb { + address: unsafe { input.as_physical_address() }, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::SLOT_ID.val(slot_id as _) + + TrbFlags::TYPE::COMMAND_CONFIGURE_ENDPOINT) + .into(), + ), + }, + ) + .await; + + if reply.completion_code() == 1 { + Ok(()) + } else { + Err(Error::InvalidOperation) + } + } + + pub async fn enable_slot(&self, executor: &E) -> Result { + let reply = self + .submit_and_wait( + executor, + Trb { + address: PhysicalAddress::ZERO, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::SLOT_TYPE.val(0) + TrbFlags::TYPE::COMMAND_ENABLE_SLOT).into(), + ), + }, + ) + .await; + + if reply.completion_code() == 1 { + Ok(reply.slot_id) + } else { + Err(Error::InvalidOperation) + } + } + + pub async fn submit_and_wait( + &self, + executor: &E, + trb: Trb, + ) -> CommandReply { + let token = self.enqueue(trb); + executor.ring_doorbell(0, 0); + poll_fn(|cx| { + self.completion_notify.register(cx.waker()); + if let Some(status) = self.get_completion(token) { + self.completion_notify.remove(cx.waker()); + Poll::Ready(status) + } else { + Poll::Pending + } + }) + .await + } + + pub fn get_completion(&self, address: PhysicalAddress) -> Option { + self.completions.write().remove(&address) + } + + pub fn notify(&self, address: PhysicalAddress, reply: CommandReply) { + self.completions.write().insert(address, reply); + self.completion_notify.wake_all(); + } +} diff --git a/driver/usb/xhci/src/ring/event.rs b/driver/usb/xhci/src/ring/event.rs new file mode 100644 index 00000000..61be83ca --- /dev/null +++ b/driver/usb/xhci/src/ring/event.rs @@ -0,0 +1,146 @@ +use core::mem::{size_of, MaybeUninit}; + +use libk_mm::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + PageBox, +}; +use libk_util::sync::IrqSafeSpinlock; +use tock_registers::interfaces::Readable; +use yggdrasil_abi::error::Error; + +use super::{command::CommandReply, GenericRing, Trb, TrbFlags}; + +pub enum Event { + PortChange(usize), + CommandCompletion { + address: PhysicalAddress, + reply: CommandReply, + }, + Transfer { + address: PhysicalAddress, + slot_id: u8, + endpoint_id: u8, + status: u32, + }, +} + +#[repr(C, align(16))] +pub struct EventRingSegment { + address: PhysicalAddress, + // Number of TRBs supported by the ring segment. Valid values are 16 to 4096 + size: u16, + _0: u16, + _1: u32, +} + +pub struct EventRingSegmentTable { + entries: PageBox<[EventRingSegment]>, +} + +struct EventRingInner { + trbs: PageBox<[MaybeUninit]>, + dequeue_index: usize, + cycle_bit: bool, +} + +pub struct EventRing { + inner: IrqSafeSpinlock, + capacity: usize, +} + +impl EventRingSegmentTable { + pub fn for_event_rings(rings: &[&EventRing]) -> Result { + let entries = PageBox::from_iter_exact(rings.iter().map(|ring| EventRingSegment { + address: ring.base(), + size: ring.capacity().try_into().unwrap(), + _0: 0, + _1: 0, + }))?; + + Ok(Self { entries }) + } + + pub fn physical_address(&self) -> PhysicalAddress { + unsafe { self.entries.as_physical_address() } + } + + pub fn capacity(&self) -> usize { + self.entries.len() + } +} + +impl GenericRing for EventRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl EventRingInner { + fn try_dequeue(&mut self) -> Option { + let trb = unsafe { self.trbs[self.dequeue_index].assume_init_ref() }; + + // TRB cannot be consumed -- its cycle bit not toggled + let trb_cycle = trb.flags.matches_all(TrbFlags::CYCLE::SET); + if trb_cycle != self.cycle_bit { + return None; + } + + self.dequeue_index += 1; + + if self.dequeue_index == self.trbs.len() { + self.dequeue_index = 0; + self.cycle_bit = !self.cycle_bit; + } + + match trb.flags.read_as_enum(TrbFlags::TYPE) { + Some(TrbFlags::TYPE::Value::EVENT_PORT_CHANGE) => Some(Event::PortChange( + IntoRaw::::into_raw(trb.address) & 0xFFusize, + )), + Some(TrbFlags::TYPE::Value::EVENT_CMD_COMPLETE) => Some(Event::CommandCompletion { + address: trb.address, + reply: CommandReply { + status: trb.status, + slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, + }, + }), + Some(TrbFlags::TYPE::Value::EVENT_XFER) => Some(Event::Transfer { + address: trb.address, + slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, + endpoint_id: trb.flags.read(TrbFlags::ENDPOINT_ID) as _, + status: trb.status, + }), + e => { + log::debug!("Unknown TRB ty: {:?}", e); + None + } + } + } +} + +impl EventRing { + pub fn new(capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity)?; + + Ok(Self { + inner: IrqSafeSpinlock::new(EventRingInner { + trbs, + dequeue_index: 0, + cycle_bit: true, + }), + capacity, + }) + } + + pub fn try_dequeue(&self) -> Option { + self.inner.lock().try_dequeue() + } + + pub fn dequeue_pointer(&self) -> PhysicalAddress { + let i = self.inner.lock(); + unsafe { i.trbs.as_physical_address() }.add(i.dequeue_index * size_of::()) + } +} diff --git a/driver/usb/xhci/src/ring/mod.rs b/driver/usb/xhci/src/ring/mod.rs new file mode 100644 index 00000000..697a5dac --- /dev/null +++ b/driver/usb/xhci/src/ring/mod.rs @@ -0,0 +1,72 @@ +use libk_mm::address::PhysicalAddress; +use tock_registers::{register_bitfields, registers::InMemoryRegister}; + +pub mod command; +pub mod event; +pub mod transfer; + +pub use command::CommandRing; +pub use event::{Event, EventRing, EventRingSegmentTable}; +pub use transfer::TransferRing; + +pub trait CommandExecutor { + fn ring_doorbell(&self, index: usize, target: u8); +} + +register_bitfields![ + u32, + pub TrbFlags [ + CYCLE OFFSET(0) NUMBITS(1) [], + LINKSEG OFFSET(1) NUMBITS(1) [], + INTERRUPT_ON_COMPLETION OFFSET(5) NUMBITS(1) [], + IMMEDIATE_DATA OFFSET(6) NUMBITS(1) [], + TYPE OFFSET(10) NUMBITS(6) [ + // Transfer ring types + NORMAL = 1, + SETUP = 2, + DATA = 3, + STATUS = 4, + ISOCH = 5, + LINK = 6, + EVENT = 7, + NOOP = 8, + // Command ring types + COMMAND_ENABLE_SLOT = 9, + COMMAND_DISABLE_SLOT = 10, + COMMAND_ADDRESS_DEVICE = 11, + COMMAND_CONFIGURE_ENDPOINT = 12, + // Event ring types + EVENT_XFER = 32, + EVENT_CMD_COMPLETE = 33, + EVENT_PORT_CHANGE = 34, + EVENT_BW_REQUEST = 35, + EVENT_DOORBELL = 36, + EVENT_HOST_CTRL = 37, + EVENT_DEVICE_NOTIFY = 38, + EVENT_MFINDEX_WRAP = 39 + ], + TRANSFER_TYPE OFFSET(16) NUMBITS(2) [ + OUT = 2, + IN = 3, + ], + TRANSFER_DIRECTION OFFSET(16) NUMBITS(1) [ + OUT = 0, + IN = 1, + ], + ENDPOINT_ID OFFSET(16) NUMBITS(4) [], + SLOT_TYPE OFFSET(16) NUMBITS(4) [], + SLOT_ID OFFSET(24) NUMBITS(8) [], + ], +]; + +#[repr(C, align(16))] +pub struct Trb { + pub address: PhysicalAddress, + pub status: u32, + pub flags: InMemoryRegister, +} + +pub trait GenericRing { + fn capacity(&self) -> usize; + fn base(&self) -> PhysicalAddress; +} diff --git a/driver/usb/xhci/src/ring/transfer.rs b/driver/usb/xhci/src/ring/transfer.rs new file mode 100644 index 00000000..ea9d279c --- /dev/null +++ b/driver/usb/xhci/src/ring/transfer.rs @@ -0,0 +1,256 @@ +use core::{ + mem::{size_of, MaybeUninit}, + sync::atomic::{AtomicU64, Ordering}, +}; + +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; +use libk_mm::{ + address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, + PageBox, +}; +use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}; +use tock_registers::{ + interfaces::{ReadWriteable, Writeable}, + registers::InMemoryRegister, +}; +use ygg_driver_usb::{ + pipe::control::ControlTransferSetup, UsbDirection, UsbTransfer, UsbTransferStatus, + UsbTransferToken, +}; +use yggdrasil_abi::error::Error; + +use super::{CommandExecutor, GenericRing, Trb, TrbFlags}; + +struct TransferRingInner { + trbs: PageBox<[MaybeUninit]>, + enqueue_index: usize, + dequeue_index: usize, + cycle_bit: bool, +} + +impl GenericRing for TransferRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl TransferRingInner { + fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { + trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); + self.trbs[self.enqueue_index].write(trb); + + let address = + unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); + + // Move to the next TRB slot + self.enqueue_index += 1; + if self.enqueue_index >= self.trbs.len() { + todo!(); + } + + address + } +} + +pub struct TransferRing { + inner: IrqSafeSpinlock, + capacity: usize, + + // TODO this is inefficient and ugly + pending_trbs: IrqSafeRwLock>, + completions: IrqSafeRwLock>>, + + slot_id: u8, + ep_id: u8, + + transfer_id: AtomicU64, +} + +impl TransferRing { + pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { + let mut trbs = PageBox::new_zeroed_slice(capacity)?; + + let base = unsafe { trbs.as_physical_address() }; + let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; + + last.address = base; + last.flags + .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); + + Ok(Self { + inner: IrqSafeSpinlock::new(TransferRingInner { + trbs, + enqueue_index: 0, + dequeue_index: 0, + cycle_bit: true, + }), + completions: IrqSafeRwLock::new(BTreeMap::new()), + pending_trbs: IrqSafeRwLock::new(BTreeMap::new()), + slot_id, + ep_id, + capacity, + + transfer_id: AtomicU64::new(0), + }) + } + + pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { + let mut inner = self.inner.lock(); + let address = inner.enqueue(trb); + address + } + + pub fn new_transfer(&self) -> UsbTransferToken { + let id = self.transfer_id.fetch_add(1, Ordering::AcqRel); + UsbTransferToken(id) + } + + pub fn complete_transfer(&self, transfer: UsbTransfer) { + let mut pending = self.pending_trbs.write(); + for trb in transfer.elements { + pending.remove(&trb); + } + self.completions.write().remove(&transfer.id); + } + + pub fn enqueue_transfer>( + &self, + trbs: I, + length: usize, + ) -> UsbTransfer { + let mut pending = self.pending_trbs.write(); + let id = self.new_transfer(); + let mut addresses = Vec::new(); + for trb in trbs { + let address = self.enqueue(trb); + addresses.push(address); + pending.insert(address, id); + } + let status = Arc::new(UsbTransferStatus::new()); + let transfer = UsbTransfer { + id, + length, + direction: UsbDirection::In, + elements: addresses, + status: status.clone(), + }; + self.completions.write().insert(id, status); + transfer + } + + fn control_setup_trb(ctl: ControlTransferSetup) -> Trb { + let word0 = (ctl.bm_request_type as u32) + | ((ctl.b_request as u32) << 8) + | ((ctl.w_value as u32) << 16); + let word1 = (ctl.w_index as u32) | ((ctl.w_length as u32) << 16); + + Trb { + address: PhysicalAddress::from_raw((word0 as u64) | ((word1 as u64) << 32)), + status: 8, + flags: InMemoryRegister::new( + (TrbFlags::TYPE::SETUP + + TrbFlags::IMMEDIATE_DATA::SET + + TrbFlags::TRANSFER_TYPE::IN) + .into(), + ), + } + } + + fn data_trb(address: PhysicalAddress, len: usize, direction: UsbDirection) -> Trb { + let dir = match direction { + UsbDirection::Out => TrbFlags::TRANSFER_DIRECTION::OUT, + UsbDirection::In => TrbFlags::TRANSFER_DIRECTION::IN, + }; + Trb { + address, + status: len.try_into().unwrap(), + flags: InMemoryRegister::new((TrbFlags::TYPE::DATA + dir).into()), + } + } + + fn control_status_trb() -> Trb { + Trb { + address: PhysicalAddress::ZERO, + status: 0, + flags: InMemoryRegister::new( + (TrbFlags::TYPE::STATUS + + TrbFlags::TRANSFER_DIRECTION::IN + + TrbFlags::INTERRUPT_ON_COMPLETION::SET) + .into(), + ), + } + } + + pub fn start_interrupt_in_transfer( + &self, + executor: &E, + buffer: &mut PageBox<[u8]>, + ) -> Result { + let data_trb = Trb { + address: unsafe { buffer.as_physical_address() }, + status: buffer.len() as _, + flags: InMemoryRegister::new( + (TrbFlags::TRANSFER_DIRECTION::IN + + TrbFlags::INTERRUPT_ON_COMPLETION::SET + + TrbFlags::TYPE::NORMAL) + .into(), + ), + }; + + let transfer = self.enqueue_transfer(core::iter::once(data_trb), buffer.len()); + + executor.ring_doorbell(self.slot_id as _, self.ep_id); + + Ok(transfer) + } + + pub fn start_control_transfer( + &self, + executor: &E, + setup: ControlTransferSetup, + buffer: Option<(PhysicalAddress, usize, UsbDirection)>, + ) -> Result { + let setup_trb = Self::control_setup_trb(setup); + let data_trb = buffer.map(|(address, len, dir)| Self::data_trb(address, len, dir)); + let status_trb = Self::control_status_trb(); + + let transfer = self.enqueue_transfer( + core::iter::once(setup_trb) + .chain(data_trb) + .chain(core::iter::once(status_trb)), + 0, + ); + + executor.ring_doorbell(self.slot_id as _, self.ep_id); + + Ok(transfer) + } + + pub fn dequeue_pointer(&self) -> PhysicalAddress { + let inner = self.inner.lock(); + unsafe { inner.trbs.as_physical_address() }.add(inner.dequeue_index * size_of::()) + } + + pub fn notify(&self, address: PhysicalAddress, value: u32) { + if value == 0 { + return; + } + + let completions = self.completions.read(); + if let Some(&token) = self.pending_trbs.read().get(&address) { + let Some(status) = completions.get(&token) else { + log::warn!( + "Notification received for non-existent transfer: {:?}", + token + ); + return; + }; + + status.signal(value); + } + } +} From dee130aba85d547fd48ec328914f67823ed8c55f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 27 Feb 2024 17:30:41 +0200 Subject: [PATCH 198/211] usb/xhci: refactor ring structs --- driver/bus/usb/src/communication.rs | 6 + driver/usb/xhci/src/ring/command.rs | 242 ++++++++++------ driver/usb/xhci/src/ring/event.rs | 150 ++++++++-- driver/usb/xhci/src/ring/mod.rs | 85 ++---- driver/usb/xhci/src/ring/transfer.rs | 403 +++++++++++++++++--------- libk/libk-mm/interface/Cargo.toml | 1 + libk/libk-mm/interface/src/address.rs | 3 +- 7 files changed, 570 insertions(+), 320 deletions(-) diff --git a/driver/bus/usb/src/communication.rs b/driver/bus/usb/src/communication.rs index 849e7819..0389b497 100644 --- a/driver/bus/usb/src/communication.rs +++ b/driver/bus/usb/src/communication.rs @@ -36,6 +36,12 @@ pub struct UsbTransfer { pub status: Arc, } +impl UsbDirection { + pub const fn is_device_to_host(self) -> bool { + matches!(self, UsbDirection::In) + } +} + // TODO this is xHCI-specific impl UsbTransferResult { pub fn is_success(&self) -> bool { diff --git a/driver/usb/xhci/src/ring/command.rs b/driver/usb/xhci/src/ring/command.rs index 12192050..8a581c0a 100644 --- a/driver/usb/xhci/src/ring/command.rs +++ b/driver/usb/xhci/src/ring/command.rs @@ -1,10 +1,12 @@ use core::{ + fmt, future::poll_fn, mem::{size_of, MaybeUninit}, task::Poll, }; use alloc::collections::BTreeMap; +use bytemuck::{Pod, Zeroable}; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, @@ -13,29 +15,13 @@ use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, waker::QueueWaker, }; -use tock_registers::{ - interfaces::{ReadWriteable, Writeable}, - registers::InMemoryRegister, -}; use xhci_lib::context; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{define_bitfields, error::Error}; -use super::{CommandExecutor, GenericRing, Trb, TrbFlags}; - -#[derive(Debug, Clone, Copy)] -pub struct CommandReply { - pub status: u32, - pub slot_id: u8, -} - -impl CommandReply { - pub fn completion_code(&self) -> u8 { - (self.status >> 24) as u8 - } -} +use super::{CommandExecutor, GenericRing, LinkTrb}; struct CommandRingInner { - trbs: PageBox<[MaybeUninit]>, + trbs: PageBox<[MaybeUninit]>, enqueue_index: usize, #[allow(unused)] dequeue_index: usize, @@ -61,12 +47,16 @@ impl GenericRing for CommandRing { } impl CommandRingInner { - fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { - trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); - self.trbs[self.enqueue_index].write(trb); + fn enqueue(&mut self, trb: C) -> PhysicalAddress { + let mut raw: RawCommandTrb = bytemuck::cast(trb); - let address = - unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); + raw.flags.set_ty(C::TRB_TYPE as _); + raw.flags.set_cycle(self.cycle_bit); + + self.trbs[self.enqueue_index].write(raw); + + let address = unsafe { self.trbs.as_physical_address() } + .add(self.enqueue_index * size_of::()); // Move to the next TRB slot self.enqueue_index += 1; @@ -83,11 +73,9 @@ impl CommandRing { let mut trbs = PageBox::new_zeroed_slice(capacity)?; let base = unsafe { trbs.as_physical_address() }; - let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; - last.address = base; - last.flags - .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); + let link = LinkTrb::new(base); + trbs.last_mut().unwrap().write(bytemuck::cast(link)); Ok(Self { inner: IrqSafeSpinlock::new(CommandRingInner { @@ -102,38 +90,21 @@ impl CommandRing { }) } - pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { + pub fn enqueue(&self, trb: C) -> PhysicalAddress { let mut inner = self.inner.lock(); let address = inner.enqueue(trb); address } - pub async fn address_device( + pub async fn address_device( &self, executor: &E, slot_id: u8, - input: &mut PageBox>, + input: &mut PageBox>, ) -> Result<(), Error> { - let reply = self - .submit_and_wait( - executor, - Trb { - address: unsafe { input.as_physical_address() }, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::SLOT_ID.val(slot_id as _) - + TrbFlags::TYPE::COMMAND_ADDRESS_DEVICE) - .into(), - ), - }, - ) - .await; - - if reply.completion_code() == 1 { - Ok(()) - } else { - Err(Error::InvalidOperation) - } + self.submit_and_wait(executor, AddressDeviceCommandTrb::new(input, slot_id)) + .await?; + Ok(()) } pub async fn configure_endpoint( @@ -142,61 +113,32 @@ impl CommandRing { slot_id: u8, input: &mut PageBox>, ) -> Result<(), Error> { - let reply = self - .submit_and_wait( - executor, - Trb { - address: unsafe { input.as_physical_address() }, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::SLOT_ID.val(slot_id as _) - + TrbFlags::TYPE::COMMAND_CONFIGURE_ENDPOINT) - .into(), - ), - }, - ) - .await; - - if reply.completion_code() == 1 { - Ok(()) - } else { - Err(Error::InvalidOperation) - } + self.submit_and_wait(executor, ConfigureEndpointCommandTrb::new(input, slot_id)) + .await?; + Ok(()) } pub async fn enable_slot(&self, executor: &E) -> Result { - let reply = self - .submit_and_wait( - executor, - Trb { - address: PhysicalAddress::ZERO, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::SLOT_TYPE.val(0) + TrbFlags::TYPE::COMMAND_ENABLE_SLOT).into(), - ), - }, - ) - .await; - - if reply.completion_code() == 1 { - Ok(reply.slot_id) - } else { - Err(Error::InvalidOperation) - } + self.submit_and_wait(executor, EnableSlotCommandTrb::new()) + .await } - pub async fn submit_and_wait( + pub async fn submit_and_wait( &self, executor: &E, - trb: Trb, - ) -> CommandReply { + trb: C, + ) -> Result { let token = self.enqueue(trb); executor.ring_doorbell(0, 0); poll_fn(|cx| { self.completion_notify.register(cx.waker()); if let Some(status) = self.get_completion(token) { self.completion_notify.remove(cx.waker()); - Poll::Ready(status) + if status.completion_code == 1 { + Poll::Ready(Ok(status.slot_id)) + } else { + Poll::Ready(Err(Error::InvalidOperation)) + } } else { Poll::Pending } @@ -213,3 +155,117 @@ impl CommandRing { self.completion_notify.wake_all(); } } + +// TRB implementations + +define_bitfields! { + pub EnableSlotCommandFlags : u32 { + (16..20) => slot_type + } +} + +define_bitfields! { + pub AddressDeviceCommandFlags : u32 { + (24..32) => slot_id + } +} + +define_bitfields! { + pub ConfigureEndpointCommandFlags : u32 { + (24..32) => slot_id + } +} + +define_bitfields! { + pub RawCommandFlags : u32 { + (10..16) => ty + set_ty, + 0 => cycle + set_cycle + } +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct EnableSlotCommandTrb { + _0: [u32; 3], + pub flags: EnableSlotCommandFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct AddressDeviceCommandTrb { + pub input_context_address: PhysicalAddress, + _0: u32, + pub flags: AddressDeviceCommandFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct ConfigureEndpointCommandTrb { + pub input_context_address: PhysicalAddress, + _0: u32, + pub flags: ConfigureEndpointCommandFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct RawCommandTrb { + _0: [u32; 3], + flags: RawCommandFlags, +} + +#[derive(Debug, Clone, Copy)] +pub struct CommandReply { + pub completion_code: u8, + pub slot_id: u8, +} + +pub trait CommandTrb: Pod + fmt::Debug { + const TRB_TYPE: u8; +} + +impl EnableSlotCommandTrb { + pub fn new() -> Self { + Self { + _0: [0; 3], + flags: EnableSlotCommandFlags::new(0), + } + } +} + +impl AddressDeviceCommandTrb { + pub fn new( + input_context: &mut PageBox>, + slot_id: u8, + ) -> Self { + Self { + input_context_address: unsafe { input_context.as_physical_address() }, + _0: 0, + flags: AddressDeviceCommandFlags::new(slot_id as _), + } + } +} + +impl ConfigureEndpointCommandTrb { + pub fn new( + input_context: &mut PageBox>, + slot_id: u8, + ) -> Self { + Self { + input_context_address: unsafe { input_context.as_physical_address() }, + _0: 0, + flags: ConfigureEndpointCommandFlags::new(slot_id as _), + } + } +} + +impl CommandTrb for EnableSlotCommandTrb { + const TRB_TYPE: u8 = 9; +} + +impl CommandTrb for AddressDeviceCommandTrb { + const TRB_TYPE: u8 = 11; +} + +impl CommandTrb for ConfigureEndpointCommandTrb { + const TRB_TYPE: u8 = 12; +} diff --git a/driver/usb/xhci/src/ring/event.rs b/driver/usb/xhci/src/ring/event.rs index 61be83ca..b132401a 100644 --- a/driver/usb/xhci/src/ring/event.rs +++ b/driver/usb/xhci/src/ring/event.rs @@ -1,14 +1,14 @@ use core::mem::{size_of, MaybeUninit}; +use bytemuck::{Pod, Zeroable}; use libk_mm::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; use libk_util::sync::IrqSafeSpinlock; -use tock_registers::interfaces::Readable; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{define_bitfields, error::Error}; -use super::{command::CommandReply, GenericRing, Trb, TrbFlags}; +use super::{command::CommandReply, GenericRing}; pub enum Event { PortChange(usize), @@ -38,7 +38,7 @@ pub struct EventRingSegmentTable { } struct EventRingInner { - trbs: PageBox<[MaybeUninit]>, + trbs: PageBox<[MaybeUninit]>, dequeue_index: usize, cycle_bit: bool, } @@ -84,7 +84,7 @@ impl EventRingInner { let trb = unsafe { self.trbs[self.dequeue_index].assume_init_ref() }; // TRB cannot be consumed -- its cycle bit not toggled - let trb_cycle = trb.flags.matches_all(TrbFlags::CYCLE::SET); + let trb_cycle = trb.cycle_bit(); if trb_cycle != self.cycle_bit { return None; } @@ -96,28 +96,7 @@ impl EventRingInner { self.cycle_bit = !self.cycle_bit; } - match trb.flags.read_as_enum(TrbFlags::TYPE) { - Some(TrbFlags::TYPE::Value::EVENT_PORT_CHANGE) => Some(Event::PortChange( - IntoRaw::::into_raw(trb.address) & 0xFFusize, - )), - Some(TrbFlags::TYPE::Value::EVENT_CMD_COMPLETE) => Some(Event::CommandCompletion { - address: trb.address, - reply: CommandReply { - status: trb.status, - slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, - }, - }), - Some(TrbFlags::TYPE::Value::EVENT_XFER) => Some(Event::Transfer { - address: trb.address, - slot_id: trb.flags.read(TrbFlags::SLOT_ID) as _, - endpoint_id: trb.flags.read(TrbFlags::ENDPOINT_ID) as _, - status: trb.status, - }), - e => { - log::debug!("Unknown TRB ty: {:?}", e); - None - } - } + trb.into_event() } } @@ -141,6 +120,119 @@ impl EventRing { pub fn dequeue_pointer(&self) -> PhysicalAddress { let i = self.inner.lock(); - unsafe { i.trbs.as_physical_address() }.add(i.dequeue_index * size_of::()) + unsafe { i.trbs.as_physical_address() }.add(i.dequeue_index * size_of::()) + } +} + +// TRB implementations + +define_bitfields! { + pub TransferEventStatus : u32 { + (24..32) => completion_code, + (0..24) => sub_length + } +} + +define_bitfields! { + pub TransferEventFlags : u32 { + (24..32) => slot_id, + (16..20) => endpoint_id, + } +} + +define_bitfields! { + pub CommandCompletionEventStatus : u32 { + (24..32) => completion_code, + (0..24) => completion_parameter + } +} + +define_bitfields! { + pub CommandCompletionEventFlags : u32 { + (24..32) => slot_id, + } +} + +define_bitfields! { + pub PortStatusChangeEventAddress : u32 { + (24..32) => port_id + } +} + +define_bitfields! { + pub RawEventFlags : u32 { + (10..16) => ty, + 0 => cycle + } +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct TransferEventTrb { + pub address: PhysicalAddress, + pub status: TransferEventStatus, + pub flags: TransferEventFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct CommandCompletionEventTrb { + pub address: PhysicalAddress, + pub status: CommandCompletionEventStatus, + pub flags: CommandCompletionEventFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct PortStatusChangeEventTrb { + pub address: PortStatusChangeEventAddress, + _0: [u32; 3], +} + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct RawEventTrb { + _0: [u32; 3], + pub flags: RawEventFlags, +} + +impl RawEventTrb { + pub fn into_event(self) -> Option { + match self.flags.ty() { + 32 => { + let transfer: TransferEventTrb = bytemuck::cast(self); + + Some(Event::Transfer { + address: transfer.address, + slot_id: transfer.flags.slot_id() as _, + endpoint_id: transfer.flags.endpoint_id() as _, + status: transfer.status.into_raw(), + }) + } + 33 => { + let command: CommandCompletionEventTrb = bytemuck::cast(self); + + Some(Event::CommandCompletion { + address: command.address, + reply: CommandReply { + completion_code: command.status.completion_code() as _, + slot_id: command.flags.slot_id() as _, + }, + }) + } + 34 => { + let port_status: PortStatusChangeEventTrb = bytemuck::cast(self); + + Some(Event::PortChange(port_status.address.port_id() as _)) + } + ty => { + log::warn!("Unhandled event TRB with type: {}", ty); + None + } + } + } + + pub fn cycle_bit(&self) -> bool { + self.flags.cycle() } } diff --git a/driver/usb/xhci/src/ring/mod.rs b/driver/usb/xhci/src/ring/mod.rs index 697a5dac..d702c45f 100644 --- a/driver/usb/xhci/src/ring/mod.rs +++ b/driver/usb/xhci/src/ring/mod.rs @@ -1,5 +1,6 @@ +use bytemuck::{Pod, Zeroable}; use libk_mm::address::PhysicalAddress; -use tock_registers::{register_bitfields, registers::InMemoryRegister}; +use yggdrasil_abi::define_bitfields; pub mod command; pub mod event; @@ -13,60 +14,36 @@ pub trait CommandExecutor { fn ring_doorbell(&self, index: usize, target: u8); } -register_bitfields![ - u32, - pub TrbFlags [ - CYCLE OFFSET(0) NUMBITS(1) [], - LINKSEG OFFSET(1) NUMBITS(1) [], - INTERRUPT_ON_COMPLETION OFFSET(5) NUMBITS(1) [], - IMMEDIATE_DATA OFFSET(6) NUMBITS(1) [], - TYPE OFFSET(10) NUMBITS(6) [ - // Transfer ring types - NORMAL = 1, - SETUP = 2, - DATA = 3, - STATUS = 4, - ISOCH = 5, - LINK = 6, - EVENT = 7, - NOOP = 8, - // Command ring types - COMMAND_ENABLE_SLOT = 9, - COMMAND_DISABLE_SLOT = 10, - COMMAND_ADDRESS_DEVICE = 11, - COMMAND_CONFIGURE_ENDPOINT = 12, - // Event ring types - EVENT_XFER = 32, - EVENT_CMD_COMPLETE = 33, - EVENT_PORT_CHANGE = 34, - EVENT_BW_REQUEST = 35, - EVENT_DOORBELL = 36, - EVENT_HOST_CTRL = 37, - EVENT_DEVICE_NOTIFY = 38, - EVENT_MFINDEX_WRAP = 39 - ], - TRANSFER_TYPE OFFSET(16) NUMBITS(2) [ - OUT = 2, - IN = 3, - ], - TRANSFER_DIRECTION OFFSET(16) NUMBITS(1) [ - OUT = 0, - IN = 1, - ], - ENDPOINT_ID OFFSET(16) NUMBITS(4) [], - SLOT_TYPE OFFSET(16) NUMBITS(4) [], - SLOT_ID OFFSET(24) NUMBITS(8) [], - ], -]; - -#[repr(C, align(16))] -pub struct Trb { - pub address: PhysicalAddress, - pub status: u32, - pub flags: InMemoryRegister, -} - pub trait GenericRing { fn capacity(&self) -> usize; fn base(&self) -> PhysicalAddress; } + +define_bitfields! { + pub LinkTrbFlags : u32 { + (10..16) => ty, + 4 => chain, + 1 => linkseg, + 0 => cycle + } +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct LinkTrb { + pub address: PhysicalAddress, + _0: u32, + pub flags: LinkTrbFlags, +} + +impl LinkTrb { + const LINK_FLAGS: LinkTrbFlags = LinkTrbFlags::new(11, true, true, true); + + pub fn new(address: PhysicalAddress) -> Self { + Self { + address, + _0: 0, + flags: Self::LINK_FLAGS, + } + } +} diff --git a/driver/usb/xhci/src/ring/transfer.rs b/driver/usb/xhci/src/ring/transfer.rs index ea9d279c..5899e8ea 100644 --- a/driver/usb/xhci/src/ring/transfer.rs +++ b/driver/usb/xhci/src/ring/transfer.rs @@ -4,58 +4,30 @@ use core::{ }; use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; +use bytemuck::{Pod, Zeroable}; use libk_mm::{ - address::{AsPhysicalAddress, FromRaw, PhysicalAddress}, + address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; -use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}; -use tock_registers::{ - interfaces::{ReadWriteable, Writeable}, - registers::InMemoryRegister, -}; +use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}; use ygg_driver_usb::{ pipe::control::ControlTransferSetup, UsbDirection, UsbTransfer, UsbTransferStatus, UsbTransferToken, }; -use yggdrasil_abi::error::Error; +use yggdrasil_abi::{define_bitfields, error::Error}; -use super::{CommandExecutor, GenericRing, Trb, TrbFlags}; +use crate::ring::LinkTrb; + +use super::{CommandExecutor, GenericRing}; struct TransferRingInner { - trbs: PageBox<[MaybeUninit]>, + trbs: PageBox<[MaybeUninit]>, enqueue_index: usize, dequeue_index: usize, cycle_bit: bool, } -impl GenericRing for TransferRing { - fn base(&self) -> PhysicalAddress { - unsafe { self.inner.lock().trbs.as_physical_address() } - } - - fn capacity(&self) -> usize { - self.capacity - } -} - -impl TransferRingInner { - fn enqueue(&mut self, trb: Trb) -> PhysicalAddress { - trb.flags.modify(TrbFlags::CYCLE.val(self.cycle_bit as _)); - self.trbs[self.enqueue_index].write(trb); - - let address = - unsafe { self.trbs.as_physical_address() }.add(self.enqueue_index * size_of::()); - - // Move to the next TRB slot - self.enqueue_index += 1; - if self.enqueue_index >= self.trbs.len() { - todo!(); - } - - address - } -} - +// TODO split TransferRing into Normal, Control, etc pub struct TransferRing { inner: IrqSafeSpinlock, capacity: usize, @@ -70,16 +42,96 @@ pub struct TransferRing { transfer_id: AtomicU64, } +struct TransferBuilder<'a> { + ring: &'a TransferRing, + ring_inner: IrqSafeSpinlockGuard<'a, TransferRingInner>, + + token: UsbTransferToken, + direction: UsbDirection, + addresses: Vec, + status: Arc, +} + +impl<'a> TransferBuilder<'a> { + pub fn new(ring: &'a TransferRing, direction: UsbDirection) -> Self { + let ring_inner = ring.inner.lock(); + let token = UsbTransferToken(ring.transfer_id.fetch_add(1, Ordering::AcqRel)); + let status = Arc::new(UsbTransferStatus::new()); + + ring.completions.write().insert(token, status.clone()); + + Self { + ring, + ring_inner, + + token, + direction, + status, + addresses: Vec::new(), + } + } + + pub fn push_trb(&mut self, trb: C) -> &mut Self { + let address = self.ring_inner.enqueue(trb); + self.addresses.push(address); + self.ring.pending_trbs.write().insert(address, self.token); + self + } + + pub fn start(self, executor: &E, length: usize) -> UsbTransfer { + executor.ring_doorbell(self.ring.slot_id as _, self.ring.ep_id); + + UsbTransfer { + id: self.token, + length, + + direction: self.direction, + elements: self.addresses, + status: self.status, + } + } +} + +impl TransferRingInner { + fn enqueue(&mut self, trb: C) -> PhysicalAddress { + let mut raw: RawTransferTrb = bytemuck::cast(trb); + + raw.flags.set_ty(C::TRB_TYPE as _); + raw.flags.set_cycle(self.cycle_bit); + + self.trbs[self.enqueue_index].write(raw); + + let address = unsafe { self.trbs.as_physical_address() } + .add(self.enqueue_index * size_of::()); + + // Move to the next TRB slot + self.enqueue_index += 1; + if self.enqueue_index >= self.trbs.len() { + todo!(); + } + + address + } +} + +impl GenericRing for TransferRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + impl TransferRing { pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { let mut trbs = PageBox::new_zeroed_slice(capacity)?; let base = unsafe { trbs.as_physical_address() }; - let last: &mut Trb = unsafe { trbs.last_mut().unwrap().assume_init_mut() }; - last.address = base; - last.flags - .write(TrbFlags::TYPE::LINK + TrbFlags::LINKSEG::SET + TrbFlags::CYCLE::SET); + let link = LinkTrb::new(base); + trbs.last_mut().unwrap().write(bytemuck::cast(link)); Ok(Self { inner: IrqSafeSpinlock::new(TransferRingInner { @@ -98,17 +150,6 @@ impl TransferRing { }) } - pub fn enqueue(&self, trb: Trb) -> PhysicalAddress { - let mut inner = self.inner.lock(); - let address = inner.enqueue(trb); - address - } - - pub fn new_transfer(&self) -> UsbTransferToken { - let id = self.transfer_id.fetch_add(1, Ordering::AcqRel); - UsbTransferToken(id) - } - pub fn complete_transfer(&self, transfer: UsbTransfer) { let mut pending = self.pending_trbs.write(); for trb in transfer.elements { @@ -117,93 +158,20 @@ impl TransferRing { self.completions.write().remove(&transfer.id); } - pub fn enqueue_transfer>( - &self, - trbs: I, - length: usize, - ) -> UsbTransfer { - let mut pending = self.pending_trbs.write(); - let id = self.new_transfer(); - let mut addresses = Vec::new(); - for trb in trbs { - let address = self.enqueue(trb); - addresses.push(address); - pending.insert(address, id); - } - let status = Arc::new(UsbTransferStatus::new()); - let transfer = UsbTransfer { - id, - length, - direction: UsbDirection::In, - elements: addresses, - status: status.clone(), - }; - self.completions.write().insert(id, status); - transfer - } - - fn control_setup_trb(ctl: ControlTransferSetup) -> Trb { - let word0 = (ctl.bm_request_type as u32) - | ((ctl.b_request as u32) << 8) - | ((ctl.w_value as u32) << 16); - let word1 = (ctl.w_index as u32) | ((ctl.w_length as u32) << 16); - - Trb { - address: PhysicalAddress::from_raw((word0 as u64) | ((word1 as u64) << 32)), - status: 8, - flags: InMemoryRegister::new( - (TrbFlags::TYPE::SETUP - + TrbFlags::IMMEDIATE_DATA::SET - + TrbFlags::TRANSFER_TYPE::IN) - .into(), - ), - } - } - - fn data_trb(address: PhysicalAddress, len: usize, direction: UsbDirection) -> Trb { - let dir = match direction { - UsbDirection::Out => TrbFlags::TRANSFER_DIRECTION::OUT, - UsbDirection::In => TrbFlags::TRANSFER_DIRECTION::IN, - }; - Trb { - address, - status: len.try_into().unwrap(), - flags: InMemoryRegister::new((TrbFlags::TYPE::DATA + dir).into()), - } - } - - fn control_status_trb() -> Trb { - Trb { - address: PhysicalAddress::ZERO, - status: 0, - flags: InMemoryRegister::new( - (TrbFlags::TYPE::STATUS - + TrbFlags::TRANSFER_DIRECTION::IN - + TrbFlags::INTERRUPT_ON_COMPLETION::SET) - .into(), - ), - } - } - pub fn start_interrupt_in_transfer( &self, executor: &E, buffer: &mut PageBox<[u8]>, ) -> Result { - let data_trb = Trb { - address: unsafe { buffer.as_physical_address() }, - status: buffer.len() as _, - flags: InMemoryRegister::new( - (TrbFlags::TRANSFER_DIRECTION::IN - + TrbFlags::INTERRUPT_ON_COMPLETION::SET - + TrbFlags::TYPE::NORMAL) - .into(), - ), - }; + let mut builder = TransferBuilder::new(self, UsbDirection::In); - let transfer = self.enqueue_transfer(core::iter::once(data_trb), buffer.len()); + builder.push_trb(NormalTransferTrb::new( + unsafe { buffer.as_physical_address() }, + buffer.len(), + true, + )); - executor.ring_doorbell(self.slot_id as _, self.ep_id); + let transfer = builder.start(executor, buffer.len()); Ok(transfer) } @@ -214,25 +182,23 @@ impl TransferRing { setup: ControlTransferSetup, buffer: Option<(PhysicalAddress, usize, UsbDirection)>, ) -> Result { - let setup_trb = Self::control_setup_trb(setup); - let data_trb = buffer.map(|(address, len, dir)| Self::data_trb(address, len, dir)); - let status_trb = Self::control_status_trb(); + let mut builder = TransferBuilder::new(self, UsbDirection::In); - let transfer = self.enqueue_transfer( - core::iter::once(setup_trb) - .chain(data_trb) - .chain(core::iter::once(status_trb)), - 0, - ); + builder.push_trb(ControlTransferSetupTrb::new(setup)); + if let Some((address, length, direction)) = buffer { + builder.push_trb(ControlTransferDataTrb::new(address, length, direction)); + } + builder.push_trb(ControlTransferStatusTrb::new(UsbDirection::In, true)); - executor.ring_doorbell(self.slot_id as _, self.ep_id); + let transfer = builder.start(executor, 0); Ok(transfer) } pub fn dequeue_pointer(&self) -> PhysicalAddress { let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() }.add(inner.dequeue_index * size_of::()) + unsafe { inner.trbs.as_physical_address() } + .add(inner.dequeue_index * size_of::()) } pub fn notify(&self, address: PhysicalAddress, value: u32) { @@ -254,3 +220,154 @@ impl TransferRing { } } } + +// TRB implementations + +define_bitfields! { + pub RawTransferFlags : u32 { + (10..16) => ty + set_ty, + 0 => cycle + set_cycle + } +} + +define_bitfields! { + pub NormalTransferFlags: u64 { + (0..16) => trb_length, + 37 => interrupt_on_completion, + } +} + +define_bitfields! { + pub ControlTransferSetupRequest : u64 { + (0..8) => bm_request_type, + (8..16) => b_request, + (16..32) => w_value, + (32..48) => w_index, + (48..64) => w_length + } +} + +define_bitfields! { + pub ControlTransferSetupFlags : u64 { + (0..16) => trb_length, + 38 => immediate_data, + (48..50) => transfer_type + } +} + +define_bitfields! { + pub ControlTransferDataFlags : u64 { + (0..16) => trb_length, + 48 => direction, + } +} + +define_bitfields! { + pub ControlTransferStatusFlags : u32 { + 16 => direction, + 5 => interrupt_on_completion + } +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct NormalTransferTrb { + pub buffer: PhysicalAddress, + pub flags: NormalTransferFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct ControlTransferSetupTrb { + pub request: ControlTransferSetupRequest, + pub flags: ControlTransferSetupFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct ControlTransferDataTrb { + pub buffer: PhysicalAddress, + pub flags: ControlTransferDataFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct ControlTransferStatusTrb { + _0: [u32; 3], + pub flags: ControlTransferStatusFlags, +} + +#[derive(Clone, Copy, Debug, Pod, Zeroable)] +#[repr(C, align(16))] +pub struct RawTransferTrb { + _0: [u32; 3], + pub flags: RawTransferFlags, +} + +pub trait TransferTrb: Pod { + const TRB_TYPE: u8; +} + +impl NormalTransferTrb { + pub fn new(buffer: PhysicalAddress, length: usize, interrupt_on_completion: bool) -> Self { + Self { + buffer, + flags: NormalTransferFlags::new(length.try_into().unwrap(), interrupt_on_completion), + } + } +} + +impl ControlTransferSetupTrb { + pub const fn new(setup: ControlTransferSetup) -> Self { + Self { + request: ControlTransferSetupRequest::new( + setup.bm_request_type as _, + setup.b_request as _, + setup.w_value as _, + setup.w_index as _, + setup.w_length as _, + ), + flags: ControlTransferSetupFlags::new(8, true, 3), + } + } +} + +impl ControlTransferDataTrb { + pub fn new(buffer: PhysicalAddress, length: usize, direction: UsbDirection) -> Self { + Self { + buffer, + flags: ControlTransferDataFlags::new( + length.try_into().unwrap(), + direction.is_device_to_host(), + ), + } + } +} + +impl ControlTransferStatusTrb { + pub const fn new(direction: UsbDirection, interrupt_on_completion: bool) -> Self { + Self { + _0: [0; 3], + flags: ControlTransferStatusFlags::new( + direction.is_device_to_host(), + interrupt_on_completion, + ), + } + } +} + +impl TransferTrb for NormalTransferTrb { + const TRB_TYPE: u8 = 1; +} + +impl TransferTrb for ControlTransferSetupTrb { + const TRB_TYPE: u8 = 2; +} + +impl TransferTrb for ControlTransferDataTrb { + const TRB_TYPE: u8 = 3; +} + +impl TransferTrb for ControlTransferStatusTrb { + const TRB_TYPE: u8 = 4; +} diff --git a/libk/libk-mm/interface/Cargo.toml b/libk/libk-mm/interface/Cargo.toml index 9eb6d23d..43cf71cb 100644 --- a/libk/libk-mm/interface/Cargo.toml +++ b/libk/libk-mm/interface/Cargo.toml @@ -11,3 +11,4 @@ yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } kernel-arch-interface = { path = "../../../arch/interface" } bitflags = "2.3.3" +bytemuck = { version = "1.14.0", features = ["derive"] } diff --git a/libk/libk-mm/interface/src/address.rs b/libk/libk-mm/interface/src/address.rs index ff27c3b3..a07ffe72 100644 --- a/libk/libk-mm/interface/src/address.rs +++ b/libk/libk-mm/interface/src/address.rs @@ -5,10 +5,11 @@ use core::{ ops::{Add, Sub}, }; +use bytemuck::{Pod, Zeroable}; use kernel_arch_interface::mem::KernelTableManager; /// Wrapper type to represent a physical memory address -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash, Pod, Zeroable)] #[repr(transparent)] pub struct PhysicalAddress(pub(crate) u64); From 99977ee987f6d1c8f75f39a61fd7c80b1aebd5db Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 29 Feb 2024 10:54:36 +0200 Subject: [PATCH 199/211] usb/xhci: proper transfer/command ring wraparound --- driver/bus/usb/src/class_driver/mod.rs | 5 +- driver/bus/usb/src/communication.rs | 18 +- driver/bus/usb/src/device.rs | 9 +- driver/bus/usb/src/lib.rs | 2 +- driver/bus/usb/src/pipe/control.rs | 6 +- driver/bus/usb/src/pipe/interrupt.rs | 19 ++- driver/bus/usb/src/pipe/mod.rs | 6 +- driver/usb/xhci/src/lib.rs | 65 +++---- driver/usb/xhci/src/pipe.rs | 65 ++++--- driver/usb/xhci/src/ring/command.rs | 31 +++- driver/usb/xhci/src/ring/mod.rs | 15 +- driver/usb/xhci/src/ring/transfer.rs | 226 +++++++++++++++++-------- 12 files changed, 297 insertions(+), 170 deletions(-) diff --git a/driver/bus/usb/src/class_driver/mod.rs b/driver/bus/usb/src/class_driver/mod.rs index ea76d297..013471f2 100644 --- a/driver/bus/usb/src/class_driver/mod.rs +++ b/driver/bus/usb/src/class_driver/mod.rs @@ -11,7 +11,6 @@ use yggdrasil_abi::{ use crate::{ device::UsbDeviceAccess, info::{UsbDeviceClass, UsbDeviceProtocol}, - UsbDirection, }; pub async fn spawn_driver(device: Arc) -> Result<(), Error> { @@ -178,7 +177,7 @@ pub async fn keyboard_driver(device: Arc) -> Result<(), Error> let config = device.select_configuration(|_| true).await?.unwrap(); assert_eq!(config.endpoints.len(), 1); - let pipe = device.open_interrupt_pipe(1, UsbDirection::In).await?; + let pipe = device.open_interrupt_in_pipe(1).await?; let mut buffer = PageBox::new_slice(0, 8)?; let mut state = KeyboardState::new(); @@ -187,7 +186,7 @@ pub async fn keyboard_driver(device: Arc) -> Result<(), Error> loop { let mut event_count = 0; - let data = pipe.listen(&mut buffer).await?; + let data = pipe.read(&mut buffer).await?; event_count += state.retain_modifiers(data[0], &mut events); event_count += state.press_modifiers(data[0], &mut events[event_count..]); diff --git a/driver/bus/usb/src/communication.rs b/driver/bus/usb/src/communication.rs index 0389b497..d62bc20d 100644 --- a/driver/bus/usb/src/communication.rs +++ b/driver/bus/usb/src/communication.rs @@ -28,7 +28,7 @@ pub struct UsbTransferStatus { pub notify: AtomicWaker, } -pub struct UsbTransfer { +pub struct UsbControlTransfer { pub id: UsbTransferToken, pub length: usize, pub direction: UsbDirection, @@ -36,6 +36,13 @@ pub struct UsbTransfer { pub status: Arc, } +pub struct UsbInterruptTransfer { + pub address: PhysicalAddress, + pub length: usize, + pub direction: UsbDirection, + pub status: Arc, +} + impl UsbDirection { pub const fn is_device_to_host(self) -> bool { matches!(self, UsbDirection::In) @@ -53,7 +60,14 @@ impl UsbTransferResult { } } -impl UsbTransfer { +impl UsbControlTransfer { + pub async fn wait(&self) -> Result { + let sub_length = self.status.wait().await?; + Ok(self.length.saturating_sub(sub_length)) + } +} + +impl UsbInterruptTransfer { pub async fn wait(&self) -> Result { let sub_length = self.status.wait().await?; Ok(self.length.saturating_sub(sub_length)) diff --git a/driver/bus/usb/src/device.rs b/driver/bus/usb/src/device.rs index d738d4bc..9c78ec70 100644 --- a/driver/bus/usb/src/device.rs +++ b/driver/bus/usb/src/device.rs @@ -10,9 +10,9 @@ use crate::{ info::{UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbInterfaceInfo}, pipe::{ control::{ConfigurationDescriptorEntry, UsbControlPipeAccess}, - interrupt::UsbInterruptPipeAccess, + interrupt::UsbInterruptInPipeAccess, }, - UsbDirection, UsbHostController, + UsbHostController, }; // High-level structures for info provided through descriptors @@ -35,11 +35,10 @@ pub trait UsbDevice: Send + Sync { // Endpoint "0" fn control_pipe(&self) -> Option<&UsbControlPipeAccess>; - fn open_interrupt_pipe<'a>( + fn open_interrupt_in_pipe<'a>( &'a self, number: u8, - direction: UsbDirection, - ) -> BoxFuture> { + ) -> BoxFuture> { unimplemented!() } diff --git a/driver/bus/usb/src/lib.rs b/driver/bus/usb/src/lib.rs index e51ef978..64f51c78 100644 --- a/driver/bus/usb/src/lib.rs +++ b/driver/bus/usb/src/lib.rs @@ -12,7 +12,7 @@ pub mod pipe; pub mod class_driver; -pub use communication::{UsbDirection, UsbTransfer, UsbTransferStatus, UsbTransferToken}; +pub use communication::{UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken}; pub trait UsbEndpoint {} diff --git a/driver/bus/usb/src/pipe/control.rs b/driver/bus/usb/src/pipe/control.rs index 2f8a17a5..14618001 100644 --- a/driver/bus/usb/src/pipe/control.rs +++ b/driver/bus/usb/src/pipe/control.rs @@ -17,7 +17,7 @@ use crate::{ UsbConfigurationDescriptor, UsbDeviceDescriptor, UsbDeviceQualifier, UsbEndpointDescriptor, UsbInterfaceDescriptor, UsbOtherSpeedConfiguration, }, - UsbDirection, UsbTransfer, + UsbControlTransfer, UsbDirection, }; use super::UsbGenericPipe; @@ -88,7 +88,9 @@ pub trait UsbControlPipe: UsbGenericPipe + Send + Sync { &self, setup: ControlTransferSetup, data: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result; + ) -> Result; + + fn complete_transfer(&self, transfer: UsbControlTransfer); } pub struct UsbControlPipeAccess(pub Box); diff --git a/driver/bus/usb/src/pipe/interrupt.rs b/driver/bus/usb/src/pipe/interrupt.rs index a42d6d36..649e1beb 100644 --- a/driver/bus/usb/src/pipe/interrupt.rs +++ b/driver/bus/usb/src/pipe/interrupt.rs @@ -4,27 +4,28 @@ use alloc::boxed::Box; use libk_mm::PageBox; use yggdrasil_abi::error::Error; -use crate::UsbTransfer; +use crate::communication::UsbInterruptTransfer; use super::UsbGenericPipe; -pub trait UsbInterruptPipe: UsbGenericPipe + Send + Sync { - fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result; +pub trait UsbInterruptInPipe: UsbGenericPipe + Send + Sync { + fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result; + fn complete_transfer(&self, transfer: UsbInterruptTransfer); } -pub struct UsbInterruptPipeAccess(pub Box); +pub struct UsbInterruptInPipeAccess(pub Box); -impl UsbInterruptPipeAccess { - // TODO timeout - pub async fn listen<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], Error> { +impl UsbInterruptInPipeAccess { + pub async fn read<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], Error> { let transfer = self.start_read(buffer)?; let len = transfer.wait().await?; + self.complete_transfer(transfer); Ok(&buffer[..len]) } } -impl Deref for UsbInterruptPipeAccess { - type Target = dyn UsbInterruptPipe; +impl Deref for UsbInterruptInPipeAccess { + type Target = dyn UsbInterruptInPipe; fn deref(&self) -> &Self::Target { &*self.0 diff --git a/driver/bus/usb/src/pipe/mod.rs b/driver/bus/usb/src/pipe/mod.rs index 45fab67b..a45d7632 100644 --- a/driver/bus/usb/src/pipe/mod.rs +++ b/driver/bus/usb/src/pipe/mod.rs @@ -1,11 +1,7 @@ -use super::UsbTransfer; - pub mod control; pub mod interrupt; -pub trait UsbGenericPipe { - fn complete_transfer(&self, transfer: UsbTransfer); -} +pub trait UsbGenericPipe {} pub enum UsbPipe { Control(control::UsbControlPipeAccess), diff --git a/driver/usb/xhci/src/lib.rs b/driver/usb/xhci/src/lib.rs index 5b778e9f..649d1604 100644 --- a/driver/usb/xhci/src/lib.rs +++ b/driver/usb/xhci/src/lib.rs @@ -24,7 +24,9 @@ use libk_thread::runtime::{self, FutureTimeout}; use libk_util::{sync::spin_rwlock::IrqSafeRwLock, waker::QueueWaker, OneTimeInit}; use pipe::ControlPipe; use regs::{Mapper, Regs}; -use ring::{CommandExecutor, CommandRing, EventRing}; +use ring::{ + transfer::InterruptInTransferRing, CommandExecutor, CommandRing, EventRing, GenericTransferRing, +}; use xhci_lib::context::{self, InputHandler}; use ygg_driver_pci::{ device::{PciDeviceInfo, PreferredInterruptMode}, @@ -34,14 +36,14 @@ use ygg_driver_usb::{ bus::UsbBusManager, device::{UsbBusAddress, UsbDevice, UsbDeviceAccess}, info::UsbEndpointType, - pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptPipeAccess}, + pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess}, UsbDirection, UsbHostController, }; use yggdrasil_abi::error::Error; use crate::{ - pipe::InterruptPipe, - ring::{Event, EventRingSegmentTable, TransferRing}, + pipe::InterruptInPipe, + ring::{ControlTransferRing, Event, EventRingSegmentTable}, }; mod pipe; @@ -76,7 +78,7 @@ pub struct Xhci { context_size: usize, dcbaa: IrqSafeRwLock>, - endpoints: IrqSafeRwLock>>, + endpoints: IrqSafeRwLock>>, event_ring: EventRing, command_ring: CommandRing, @@ -101,11 +103,12 @@ impl XhciBusDevice { async fn setup_endpoint_inner( &self, - number: u8, + ring: Arc, + dci: u8, ty: UsbEndpointType, direction: UsbDirection, - ) -> Result, Error> { - log::debug!("Setup endpoint #{}: {:?} {:?}", number, ty, direction); + ) -> Result<(), Error> { + log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction); let mut input = self.context.input.write(); @@ -113,18 +116,13 @@ impl XhciBusDevice { (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn, _ => todo!(), }; - // number = 1 - // dci = (number * 2) + IN = 1 * 2 + 1 = 3 - let dci = Self::dci(ty, direction, number); - - let ring = Arc::new(TransferRing::new(self.slot_id, dci as _, 128)?); { let control = input.control_mut(); control.set_add_context_flag(0); control.clear_add_context_flag(1); - control.set_add_context_flag(dci); + control.set_add_context_flag(dci as _); } { @@ -134,7 +132,7 @@ impl XhciBusDevice { } { - let ep_cx = input.device_mut().endpoint_mut(dci); + let ep_cx = input.device_mut().endpoint_mut(dci as _); ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); ep_cx.set_dequeue_cycle_state(); @@ -150,10 +148,9 @@ impl XhciBusDevice { .configure_endpoint(self.xhci, self.slot_id, &mut input) .await?; - self.xhci - .register_endpoint(self.slot_id, dci as _, ring.clone()); + self.xhci.register_endpoint(self.slot_id, dci, ring); - Ok(ring) + Ok(()) } } @@ -174,20 +171,25 @@ impl UsbDevice for XhciBusDevice { self.xhci } - fn open_interrupt_pipe<'a>( + fn open_interrupt_in_pipe<'a>( &'a self, number: u8, - direction: UsbDirection, - ) -> BoxFuture> { + ) -> BoxFuture> { async move { - let ring = self - .setup_endpoint_inner(number, UsbEndpointType::Interrupt, direction) - .await?; + let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8; + let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?); - let dci = Self::dci(UsbEndpointType::Interrupt, direction, number) + 1; - let pipe = InterruptPipe::input(self.xhci, self.slot_id, number, dci as _, ring); + self.setup_endpoint_inner( + ring.clone(), + dci, + UsbEndpointType::Interrupt, + UsbDirection::In, + ) + .await?; - Ok(UsbInterruptPipeAccess(Box::new(pipe))) + let pipe = InterruptInPipe::new(self.xhci, self.slot_id, number, dci, ring); + + Ok(UsbInterruptInPipeAccess(Box::new(pipe))) } .boxed() } @@ -243,7 +245,12 @@ impl Xhci { self.dcbaa.write()[slot_id as usize] = context; } - pub fn register_endpoint(&self, slot_id: u8, endpoint_id: u8, ring: Arc) { + pub fn register_endpoint( + &self, + slot_id: u8, + endpoint_id: u8, + ring: Arc, + ) { self.endpoints.write().insert((slot_id, endpoint_id), ring); } @@ -265,7 +272,7 @@ impl Xhci { root_hub_port_number: u8, ) -> Result, Error> { let address = 1; - let ring = Arc::new(TransferRing::new(slot_id, 1, 128)?); + let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?); let context = XhciContext::new_32byte()?; let mut input = context.input.write(); diff --git a/driver/usb/xhci/src/pipe.rs b/driver/usb/xhci/src/pipe.rs index a0102bb7..6f72ab8a 100644 --- a/driver/usb/xhci/src/pipe.rs +++ b/driver/usb/xhci/src/pipe.rs @@ -1,88 +1,85 @@ use alloc::sync::Arc; use libk_mm::{address::PhysicalAddress, PageBox}; use ygg_driver_usb::{ + communication::UsbInterruptTransfer, pipe::{ control::{ControlTransferSetup, UsbControlPipe}, - interrupt::UsbInterruptPipe, + interrupt::UsbInterruptInPipe, UsbGenericPipe, }, - UsbDirection, UsbTransfer, + UsbControlTransfer, UsbDirection, }; use yggdrasil_abi::error::Error; -use crate::{ring::TransferRing, Xhci}; +use crate::{ + ring::{transfer::InterruptInTransferRing, ControlTransferRing}, + Xhci, +}; pub struct ControlPipe { xhci: &'static Xhci, - ring: Arc, + ring: Arc, } #[allow(unused)] -pub struct InterruptPipe { +pub struct InterruptInPipe { xhci: &'static Xhci, slot_id: u8, endpoint_id: u8, dci: u8, - input: Option>, - output: Option>, + ring: Arc, } -impl UsbGenericPipe for ControlPipe { - fn complete_transfer(&self, transfer: UsbTransfer) { - self.ring.complete_transfer(transfer) - } -} +impl UsbGenericPipe for ControlPipe {} impl UsbControlPipe for ControlPipe { fn start_transfer( &self, setup: ControlTransferSetup, data: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { - self.ring.start_control_transfer(self.xhci, setup, data) + ) -> Result { + self.ring.start_transfer(self.xhci, setup, data) + } + + fn complete_transfer(&self, transfer: UsbControlTransfer) { + self.ring.complete_transfer(transfer) } } impl ControlPipe { - pub fn new(xhci: &'static Xhci, _slot_id: u8, ring: Arc) -> Self { + pub fn new(xhci: &'static Xhci, _slot_id: u8, ring: Arc) -> Self { Self { xhci, ring } } } -impl UsbGenericPipe for InterruptPipe { - fn complete_transfer(&self, transfer: UsbTransfer) { - match transfer.direction { - UsbDirection::Out => todo!(), - UsbDirection::In => todo!(), - } +impl UsbGenericPipe for InterruptInPipe {} + +impl UsbInterruptInPipe for InterruptInPipe { + fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result { + self.ring.start_transfer(self.xhci, buffer) + } + + fn complete_transfer(&self, transfer: UsbInterruptTransfer) { + self.ring.complete_transfer(transfer) } } -impl UsbInterruptPipe for InterruptPipe { - fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result { - let input = self.input.as_ref().unwrap(); - - input.start_interrupt_in_transfer(self.xhci, buffer) - } -} - -impl InterruptPipe { - pub fn input( +impl InterruptInPipe { + pub fn new( xhci: &'static Xhci, slot_id: u8, endpoint_id: u8, dci: u8, - ring: Arc, + ring: Arc, ) -> Self { Self { xhci, slot_id, endpoint_id, dci, - input: Some(ring), - output: None, + ring, } } } diff --git a/driver/usb/xhci/src/ring/command.rs b/driver/usb/xhci/src/ring/command.rs index 8a581c0a..a45ab9c4 100644 --- a/driver/usb/xhci/src/ring/command.rs +++ b/driver/usb/xhci/src/ring/command.rs @@ -60,22 +60,36 @@ impl CommandRingInner { // Move to the next TRB slot self.enqueue_index += 1; - if self.enqueue_index >= self.trbs.len() { - todo!(); + if self.enqueue_index >= self.trbs.len() - 1 { + self.enqueue_link(); + + // Wrap around + self.cycle_bit = !self.cycle_bit; + self.enqueue_index = 0; } address } + + fn enqueue_link(&mut self) { + let base = unsafe { self.trbs.as_physical_address() }; + + let link = LinkTrb::new(base, self.cycle_bit); + self.trbs[self.enqueue_index].write(bytemuck::cast(link)); + } + + fn advance(&mut self) { + self.dequeue_index += 1; + + if self.dequeue_index >= self.trbs.len() - 1 { + self.dequeue_index = 0; + } + } } impl CommandRing { pub fn new(capacity: usize) -> Result { - let mut trbs = PageBox::new_zeroed_slice(capacity)?; - - let base = unsafe { trbs.as_physical_address() }; - - let link = LinkTrb::new(base); - trbs.last_mut().unwrap().write(bytemuck::cast(link)); + let trbs = PageBox::new_zeroed_slice(capacity)?; Ok(Self { inner: IrqSafeSpinlock::new(CommandRingInner { @@ -151,6 +165,7 @@ impl CommandRing { } pub fn notify(&self, address: PhysicalAddress, reply: CommandReply) { + self.inner.lock().advance(); self.completions.write().insert(address, reply); self.completion_notify.wake_all(); } diff --git a/driver/usb/xhci/src/ring/mod.rs b/driver/usb/xhci/src/ring/mod.rs index d702c45f..eba9f459 100644 --- a/driver/usb/xhci/src/ring/mod.rs +++ b/driver/usb/xhci/src/ring/mod.rs @@ -8,7 +8,7 @@ pub mod transfer; pub use command::CommandRing; pub use event::{Event, EventRing, EventRingSegmentTable}; -pub use transfer::TransferRing; +pub use transfer::ControlTransferRing; pub trait CommandExecutor { fn ring_doorbell(&self, index: usize, target: u8); @@ -19,11 +19,16 @@ pub trait GenericRing { fn base(&self) -> PhysicalAddress; } +pub trait GenericTransferRing: GenericRing + Send + Sync { + fn dequeue_pointer(&self) -> PhysicalAddress; + fn notify(&self, address: PhysicalAddress, status: u32); +} + define_bitfields! { pub LinkTrbFlags : u32 { (10..16) => ty, 4 => chain, - 1 => linkseg, + 1 => toggle_cycle, 0 => cycle } } @@ -37,13 +42,11 @@ pub struct LinkTrb { } impl LinkTrb { - const LINK_FLAGS: LinkTrbFlags = LinkTrbFlags::new(11, true, true, true); - - pub fn new(address: PhysicalAddress) -> Self { + pub fn new(address: PhysicalAddress, cycle_bit: bool) -> Self { Self { address, _0: 0, - flags: Self::LINK_FLAGS, + flags: LinkTrbFlags::new(6, true, true, cycle_bit), } } } diff --git a/driver/usb/xhci/src/ring/transfer.rs b/driver/usb/xhci/src/ring/transfer.rs index 5899e8ea..57346277 100644 --- a/driver/usb/xhci/src/ring/transfer.rs +++ b/driver/usb/xhci/src/ring/transfer.rs @@ -11,14 +11,14 @@ use libk_mm::{ }; use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}; use ygg_driver_usb::{ - pipe::control::ControlTransferSetup, UsbDirection, UsbTransfer, UsbTransferStatus, - UsbTransferToken, + communication::UsbInterruptTransfer, pipe::control::ControlTransferSetup, UsbControlTransfer, + UsbDirection, UsbTransferStatus, UsbTransferToken, }; use yggdrasil_abi::{define_bitfields, error::Error}; use crate::ring::LinkTrb; -use super::{CommandExecutor, GenericRing}; +use super::{CommandExecutor, GenericRing, GenericTransferRing}; struct TransferRingInner { trbs: PageBox<[MaybeUninit]>, @@ -28,7 +28,7 @@ struct TransferRingInner { } // TODO split TransferRing into Normal, Control, etc -pub struct TransferRing { +pub struct ControlTransferRing { inner: IrqSafeSpinlock, capacity: usize, @@ -42,8 +42,18 @@ pub struct TransferRing { transfer_id: AtomicU64, } +pub struct InterruptInTransferRing { + inner: IrqSafeSpinlock, + capacity: usize, + + completions: IrqSafeRwLock>>, + + slot_id: u8, + ep_id: u8, +} + struct TransferBuilder<'a> { - ring: &'a TransferRing, + ring: &'a ControlTransferRing, ring_inner: IrqSafeSpinlockGuard<'a, TransferRingInner>, token: UsbTransferToken, @@ -53,7 +63,7 @@ struct TransferBuilder<'a> { } impl<'a> TransferBuilder<'a> { - pub fn new(ring: &'a TransferRing, direction: UsbDirection) -> Self { + pub fn new(ring: &'a ControlTransferRing, direction: UsbDirection) -> Self { let ring_inner = ring.inner.lock(); let token = UsbTransferToken(ring.transfer_id.fetch_add(1, Ordering::AcqRel)); let status = Arc::new(UsbTransferStatus::new()); @@ -78,10 +88,10 @@ impl<'a> TransferBuilder<'a> { self } - pub fn start(self, executor: &E, length: usize) -> UsbTransfer { + pub fn start(self, executor: &E, length: usize) -> UsbControlTransfer { executor.ring_doorbell(self.ring.slot_id as _, self.ring.ep_id); - UsbTransfer { + UsbControlTransfer { id: self.token, length, @@ -94,6 +104,10 @@ impl<'a> TransferBuilder<'a> { impl TransferRingInner { fn enqueue(&mut self, trb: C) -> PhysicalAddress { + if (self.enqueue_index + 1) % (self.trbs.len() - 1) == self.dequeue_index { + todo!("Ring full"); + } + let mut raw: RawTransferTrb = bytemuck::cast(trb); raw.flags.set_ty(C::TRB_TYPE as _); @@ -106,15 +120,34 @@ impl TransferRingInner { // Move to the next TRB slot self.enqueue_index += 1; - if self.enqueue_index >= self.trbs.len() { - todo!(); + if self.enqueue_index >= self.trbs.len() - 1 { + self.enqueue_link(); + + // Wrap around + self.cycle_bit = !self.cycle_bit; + self.enqueue_index = 0; } address } + + fn enqueue_link(&mut self) { + let base = unsafe { self.trbs.as_physical_address() }; + + let link = LinkTrb::new(base, self.cycle_bit); + self.trbs[self.enqueue_index].write(bytemuck::cast(link)); + } + + fn advance(&mut self) { + self.dequeue_index += 1; + + if self.dequeue_index >= self.trbs.len() - 1 { + self.dequeue_index = 0; + } + } } -impl GenericRing for TransferRing { +impl GenericRing for ControlTransferRing { fn base(&self) -> PhysicalAddress { unsafe { self.inner.lock().trbs.as_physical_address() } } @@ -124,14 +157,116 @@ impl GenericRing for TransferRing { } } -impl TransferRing { +impl GenericTransferRing for ControlTransferRing { + fn dequeue_pointer(&self) -> PhysicalAddress { + let inner = self.inner.lock(); + unsafe { inner.trbs.as_physical_address() } + .add(inner.dequeue_index * size_of::()) + } + + fn notify(&self, address: PhysicalAddress, value: u32) { + if value == 0 { + return; + } + + let completions = self.completions.read(); + if let Some(&token) = self.pending_trbs.read().get(&address) { + let Some(status) = completions.get(&token) else { + log::warn!( + "Notification received for non-existent transfer: {:?}", + token + ); + return; + }; + + status.signal(value); + } + } +} + +impl GenericRing for InterruptInTransferRing { + fn base(&self) -> PhysicalAddress { + unsafe { self.inner.lock().trbs.as_physical_address() } + } + + fn capacity(&self) -> usize { + self.capacity + } +} + +impl GenericTransferRing for InterruptInTransferRing { + fn dequeue_pointer(&self) -> PhysicalAddress { + let inner = self.inner.lock(); + unsafe { inner.trbs.as_physical_address() } + .add(inner.dequeue_index * size_of::()) + } + + fn notify(&self, address: PhysicalAddress, value: u32) { + if value == 0 { + return; + } + + let mut completions = self.completions.write(); + if let Some(status) = completions.remove(&address) { + status.signal(value); + } + } +} + +impl InterruptInTransferRing { pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let mut trbs = PageBox::new_zeroed_slice(capacity)?; + let trbs = PageBox::new_zeroed_slice(capacity)?; - let base = unsafe { trbs.as_physical_address() }; + Ok(Self { + inner: IrqSafeSpinlock::new(TransferRingInner { + trbs, + enqueue_index: 0, + dequeue_index: 0, + cycle_bit: true, + }), + completions: IrqSafeRwLock::new(BTreeMap::new()), + slot_id, + ep_id, + capacity, + }) + } - let link = LinkTrb::new(base); - trbs.last_mut().unwrap().write(bytemuck::cast(link)); + pub fn start_transfer( + &self, + executor: &E, + buffer: &mut PageBox<[u8]>, + ) -> Result { + let status = Arc::new(UsbTransferStatus::new()); + let address = self.inner.lock().enqueue(NormalTransferTrb::new( + unsafe { buffer.as_physical_address() }, + buffer.len(), + true, + )); + self.completions.write().insert(address, status.clone()); + + executor.ring_doorbell(self.slot_id as _, self.ep_id); + + Ok(UsbInterruptTransfer { + length: buffer.len(), + direction: UsbDirection::In, + address, + status, + }) + } + + pub fn complete_transfer(&self, _transfer: UsbInterruptTransfer) { + // Interrupt transfers consist of one TRB each + // TODO: Can two transfers happen simultaneously? e.g. + // + // [TRBa, TRBb] are queued in the ring, both are executing and + // TRBb finishes first + self.inner.lock().advance(); + } +} + +impl ControlTransferRing { + pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity)?; Ok(Self { inner: IrqSafeSpinlock::new(TransferRingInner { @@ -150,38 +285,12 @@ impl TransferRing { }) } - pub fn complete_transfer(&self, transfer: UsbTransfer) { - let mut pending = self.pending_trbs.write(); - for trb in transfer.elements { - pending.remove(&trb); - } - self.completions.write().remove(&transfer.id); - } - - pub fn start_interrupt_in_transfer( - &self, - executor: &E, - buffer: &mut PageBox<[u8]>, - ) -> Result { - let mut builder = TransferBuilder::new(self, UsbDirection::In); - - builder.push_trb(NormalTransferTrb::new( - unsafe { buffer.as_physical_address() }, - buffer.len(), - true, - )); - - let transfer = builder.start(executor, buffer.len()); - - Ok(transfer) - } - - pub fn start_control_transfer( + pub fn start_transfer( &self, executor: &E, setup: ControlTransferSetup, buffer: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { + ) -> Result { let mut builder = TransferBuilder::new(self, UsbDirection::In); builder.push_trb(ControlTransferSetupTrb::new(setup)); @@ -195,29 +304,14 @@ impl TransferRing { Ok(transfer) } - pub fn dequeue_pointer(&self) -> PhysicalAddress { - let inner = self.inner.lock(); - unsafe { inner.trbs.as_physical_address() } - .add(inner.dequeue_index * size_of::()) - } - - pub fn notify(&self, address: PhysicalAddress, value: u32) { - if value == 0 { - return; - } - - let completions = self.completions.read(); - if let Some(&token) = self.pending_trbs.read().get(&address) { - let Some(status) = completions.get(&token) else { - log::warn!( - "Notification received for non-existent transfer: {:?}", - token - ); - return; - }; - - status.signal(value); + pub fn complete_transfer(&self, transfer: UsbControlTransfer) { + let mut pending = self.pending_trbs.write(); + let mut inner = self.inner.lock(); + for trb in transfer.elements { + pending.remove(&trb); + inner.advance(); } + self.completions.write().remove(&transfer.id); } } From 395969a90e760724e9eccf78a572b95b3f63d3ff Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Thu, 29 Feb 2024 13:11:23 +0200 Subject: [PATCH 200/211] usb/xhci: implement device attach/detach --- driver/bus/usb/src/bus.rs | 6 + driver/bus/usb/src/communication.rs | 11 + driver/bus/usb/src/device.rs | 2 + driver/bus/usb/src/error.rs | 7 + driver/bus/usb/src/lib.rs | 2 + driver/bus/usb/src/util.rs | 64 ++++ driver/usb/xhci/Cargo.toml | 1 + driver/usb/xhci/src/controller.rs | 395 +++++++++++++++++++++++ driver/usb/xhci/src/device.rs | 151 +++++++++ driver/usb/xhci/src/lib.rs | 452 +-------------------------- driver/usb/xhci/src/ring/mod.rs | 4 + driver/usb/xhci/src/ring/transfer.rs | 52 ++- lib/device-api/src/interrupt.rs | 2 +- 13 files changed, 707 insertions(+), 442 deletions(-) create mode 100644 driver/bus/usb/src/error.rs create mode 100644 driver/bus/usb/src/util.rs create mode 100644 driver/usb/xhci/src/controller.rs create mode 100644 driver/usb/xhci/src/device.rs diff --git a/driver/bus/usb/src/bus.rs b/driver/bus/usb/src/bus.rs index e99b69ad..945379ba 100644 --- a/driver/bus/usb/src/bus.rs +++ b/driver/bus/usb/src/bus.rs @@ -31,6 +31,12 @@ impl UsbBusManager { QUEUE.push_back(device); } + + pub fn detach_device(address: UsbBusAddress) { + if let Some(device) = BUS_MANAGER.devices.write().remove(&address) { + device.handle_detach(); + } + } } pub async fn bus_handler() { diff --git a/driver/bus/usb/src/communication.rs b/driver/bus/usb/src/communication.rs index d62bc20d..ee7b184c 100644 --- a/driver/bus/usb/src/communication.rs +++ b/driver/bus/usb/src/communication.rs @@ -51,6 +51,10 @@ impl UsbDirection { // TODO this is xHCI-specific impl UsbTransferResult { + pub fn is_aborted(&self) -> bool { + self.0 == u32::MAX + } + pub fn is_success(&self) -> bool { (self.0 >> 24) & 0xFF == 1 } @@ -87,6 +91,8 @@ impl UsbTransferStatus { self.poll(cx).map(|v| { if v.is_success() { Ok(v.sub_length()) + } else if v.is_aborted() { + Err(Error::NetworkUnreachable) } else { Err(Error::InvalidOperation) } @@ -100,6 +106,11 @@ impl UsbTransferStatus { self.notify.wake(); } + pub fn abort(&self) { + self.result.store(u32::MAX, Ordering::Release); + self.notify.wake(); + } + pub fn poll(&self, cx: &mut Context<'_>) -> Poll { self.notify.register(cx.waker()); let value = self.result.load(Ordering::Acquire); diff --git a/driver/bus/usb/src/device.rs b/driver/bus/usb/src/device.rs index 9c78ec70..12ac26bf 100644 --- a/driver/bus/usb/src/device.rs +++ b/driver/bus/usb/src/device.rs @@ -46,6 +46,8 @@ pub trait UsbDevice: Send + Sync { fn bus_address(&self) -> UsbBusAddress; fn controller(&self) -> &'static dyn UsbHostController; + fn handle_detach(&self); + fn debug(&self) {} } diff --git a/driver/bus/usb/src/error.rs b/driver/bus/usb/src/error.rs new file mode 100644 index 00000000..044b3597 --- /dev/null +++ b/driver/bus/usb/src/error.rs @@ -0,0 +1,7 @@ +use yggdrasil_abi::error::Error; + +#[derive(Debug)] +pub enum UsbError { + MemoryError(Error), + OutOfAddresses, +} diff --git a/driver/bus/usb/src/lib.rs b/driver/bus/usb/src/lib.rs index 64f51c78..0a4e6d7f 100644 --- a/driver/bus/usb/src/lib.rs +++ b/driver/bus/usb/src/lib.rs @@ -7,8 +7,10 @@ pub mod bus; pub mod communication; pub mod descriptor; pub mod device; +pub mod error; pub mod info; pub mod pipe; +pub mod util; pub mod class_driver; diff --git a/driver/bus/usb/src/util.rs b/driver/bus/usb/src/util.rs new file mode 100644 index 00000000..bd3e09f9 --- /dev/null +++ b/driver/bus/usb/src/util.rs @@ -0,0 +1,64 @@ +use libk_util::sync::spin_rwlock::IrqSafeRwLock; + +use crate::error::UsbError; + +pub struct UsbAddressAllocator { + // 256 bits + bitmap: IrqSafeRwLock, +} + +struct Bitmap { + data: [u64; 4], +} + +impl Bitmap { + const fn bit_index(bit: u8) -> usize { + (bit / 64) as usize + } + + const fn bit_mask(bit: u8) -> u64 { + 1 << (bit % 64) + } + + pub const fn new() -> Self { + // First is 1 to prevent address 0 from being allocated + Self { data: [1, 0, 0, 0] } + } + + pub fn is_set(&self, bit: u8) -> bool { + self.data[Self::bit_index(bit)] & Self::bit_mask(bit) != 0 + } + + pub fn set(&mut self, bit: u8) { + self.data[Self::bit_index(bit)] |= Self::bit_mask(bit); + } + + pub fn clear(&mut self, bit: u8) { + self.data[Self::bit_index(bit)] &= !Self::bit_mask(bit); + } +} + +impl UsbAddressAllocator { + pub fn new() -> Self { + Self { + bitmap: IrqSafeRwLock::new(Bitmap::new()), + } + } + + pub fn allocate(&self) -> Result { + let mut bitmap = self.bitmap.write(); + for bit in 0..=255 { + if !bitmap.is_set(bit) { + bitmap.set(bit); + return Ok(bit); + } + } + Err(UsbError::OutOfAddresses) + } + + pub fn free(&self, address: u8) { + let mut bitmap = self.bitmap.write(); + assert!(bitmap.is_set(address)); + bitmap.clear(address); + } +} diff --git a/driver/usb/xhci/Cargo.toml b/driver/usb/xhci/Cargo.toml index 9c4359f9..e4f964f3 100644 --- a/driver/usb/xhci/Cargo.toml +++ b/driver/usb/xhci/Cargo.toml @@ -16,6 +16,7 @@ libk-thread = { path = "../../../libk/libk-thread" } xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" } +atomic_enum = "0.2.0" log = "0.4.20" tock-registers = "0.9.0" bytemuck = { version = "1.14.0", features = ["derive"] } diff --git a/driver/usb/xhci/src/controller.rs b/driver/usb/xhci/src/controller.rs new file mode 100644 index 00000000..5d84d330 --- /dev/null +++ b/driver/usb/xhci/src/controller.rs @@ -0,0 +1,395 @@ +use core::{future::poll_fn, sync::atomic::Ordering, task::Poll, time::Duration}; + +use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec, vec::Vec}; +use atomic_enum::atomic_enum; +use device_api::{interrupt::InterruptHandler, Device}; +use futures_util::task::AtomicWaker; +use libk_mm::{ + address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + PageBox, +}; +use libk_thread::runtime::{self, FutureTimeout}; +use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; +use xhci_lib::context::{self, InputHandler}; +use ygg_driver_usb::{ + bus::UsbBusManager, + device::{UsbBusAddress, UsbDevice, UsbDeviceAccess}, + pipe::control::UsbControlPipeAccess, + util::UsbAddressAllocator, + UsbHostController, +}; +use yggdrasil_abi::error::Error; + +use crate::{ + device::XhciBusDevice, + pipe::ControlPipe, + regs::{Mapper, Regs}, + ring::{ + CommandExecutor, CommandRing, ControlTransferRing, Event, EventRing, EventRingSegmentTable, + GenericTransferRing, + }, + XhciContext, +}; + +#[atomic_enum] +#[derive(PartialEq, Eq)] +pub enum PortState { + Disconnected, // Default + set by "handle detach" + Init, // Set by "port task" + Running, // Set by "port task" +} + +struct PortStruct { + state: AtomicPortState, + notify: AtomicWaker, + address: IrqSafeRwLock>, +} + +pub struct Xhci { + regs: Regs, + + bus_address: OneTimeInit, + address_allocator: UsbAddressAllocator, + + port_count: usize, + // TODO use to allocate proper contexts + #[allow(unused)] + context_size: usize, + + dcbaa: IrqSafeRwLock>, + endpoints: IrqSafeRwLock>>, + event_ring: EventRing, + pub(crate) command_ring: CommandRing, + + port_states: Vec, +} + +impl Xhci { + pub fn new(regs: xhci_lib::Registers) -> Result { + let event_ring = EventRing::new(128)?; + let command_ring = CommandRing::new(128)?; + + let regs = Regs::from(regs); + + let port_count = regs.port_count(); + let slot_count = regs.max_slot_count(); + let context_size = regs.context_size(); + + Ok(Self { + regs, + + bus_address: OneTimeInit::new(), + address_allocator: UsbAddressAllocator::new(), + + port_count, + context_size, + + event_ring, + command_ring, + + dcbaa: IrqSafeRwLock::new(PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1)?), + endpoints: IrqSafeRwLock::new(BTreeMap::new()), + + port_states: Vec::from_iter((0..port_count).map(|_| PortStruct { + state: AtomicPortState::new(PortState::Disconnected), + notify: AtomicWaker::new(), + address: IrqSafeRwLock::new(None), + })), + }) + } + + pub fn register_device_context(&self, slot_id: u8, context: PhysicalAddress) { + self.dcbaa.write()[slot_id as usize] = context; + } + + pub fn register_endpoint( + &self, + slot_id: u8, + endpoint_id: u8, + ring: Arc, + ) { + self.endpoints.write().insert((slot_id, endpoint_id), ring); + } + + pub fn shutdown_endpoint(&self, slot_id: u8, endpoint_id: u8) { + if let Some(endpoint) = self.endpoints.write().remove(&(slot_id, endpoint_id)) { + endpoint.shutdown(); + } else { + log::warn!( + "Endpoint {}:{} does not exist, maybe already shut down?", + slot_id, + endpoint_id + ); + } + } + + pub fn notify_transfer( + &self, + slot_id: u8, + endpoint_id: u8, + address: PhysicalAddress, + status: u32, + ) { + if let Some(ep) = self.endpoints.read().get(&(slot_id, endpoint_id)) { + ep.notify(address, status); + } + } + + async fn assign_device( + &'static self, + slot_id: u8, + root_hub_port_number: u8, + ) -> Result, Error> { + let address = self.address_allocator.allocate().unwrap(); + let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?); + + let context = XhciContext::new_32byte()?; + let mut input = context.input.write(); + + // Setup input context + { + let control = input.control_mut(); + + control.set_add_context_flag(0); + control.set_add_context_flag(1); + } + + { + let slot = input.device_mut().slot_mut(); + + slot.set_context_entries(1); + slot.set_interrupter_target(0); + slot.set_usb_device_address(address); + slot.set_root_hub_port_number(root_hub_port_number); + } + + { + let ep0 = input.device_mut().endpoint_mut(1); + + ep0.set_endpoint_type(context::EndpointType::Control); + ep0.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); + ep0.set_dequeue_cycle_state(); + ep0.set_error_count(3); + } + + self.register_device_context(slot_id, unsafe { context.output.as_physical_address() }); + + self.command_ring + .address_device(self, slot_id, &mut input) + .await?; + + self.register_endpoint(slot_id, 1, ring.clone()); + + let pipe = ControlPipe::new(self, slot_id, ring.clone()); + + drop(input); + + let bus_address = UsbBusAddress { + bus: *self.bus_address.get(), + device: address, + }; + + let device = XhciBusDevice { + xhci: self, + slot_id, + port_id: root_hub_port_number, + bus_address, + context: Arc::new(context), + rings: IrqSafeRwLock::new(vec![ring]), + control_pipe: UsbControlPipeAccess(Box::new(pipe)), + }; + + Ok(Box::new(device)) + } + + async fn port_task(&'static self, index: usize) -> Result<(), Error> { + let state = &self.port_states[index]; + log::debug!("Task spawned for port {}", index); + + self.reset_port(index).await?; + + let slot_id = self.command_ring.enable_slot(self).await?; + log::debug!("Enabled slot: {}", slot_id); + + let device = self.assign_device(slot_id, (index + 1) as _).await?; + let device = UsbDeviceAccess::setup(device).await?; + + let old = state.address.write().replace(device.bus_address()); + assert!(old.is_none()); + + UsbBusManager::register_device(Arc::new(device)); + + state.state.store(PortState::Running, Ordering::Release); + + Ok(()) + } + + fn handle_device_attached(&'static self, port: usize) -> Result<(), Error> { + log::info!("Port {}: device attached", port); + + if let Err(err) = self.port_states[port].state.compare_exchange( + PortState::Disconnected, + PortState::Init, + Ordering::Acquire, + Ordering::Relaxed, + ) { + log::warn!("Could not start port init task: port state is {:?}", err); + return Err(Error::InvalidArgument); + } + + runtime::spawn(async move { self.port_task(port).await })?; + Ok(()) + } + + fn handle_device_detached(&'static self, port: usize) -> Result<(), Error> { + let state = &self.port_states[port]; + + match state.state.swap(PortState::Disconnected, Ordering::Release) { + PortState::Init => todo!(), + PortState::Running => { + log::info!("Port {}: device detached", port); + let address = state + .address + .write() + .take() + .expect("Port marked as Running, but has no address"); + + UsbBusManager::detach_device(address); + state.notify.wake(); + + Ok(()) + } + // Already disconnected + PortState::Disconnected => Ok(()), + } + } + + fn handle_port_event(&'static self, port: usize) -> Result<(), Error> { + let state = &self.port_states[port]; + let port_regs = self.regs.ports.read(port); + + if port_regs.portsc.connect_status_change() { + if port_regs.portsc.current_connect_status() { + self.handle_device_attached(port) + } else { + self.handle_device_detached(port) + } + } else { + // Some other event + state.notify.wake(); + Ok(()) + } + } + + async fn reset_port(&self, port: usize) -> Result<(), Error> { + log::debug!("Reset port {}", port); + + self.regs.ports.update(port, |u| { + u.portsc.set_port_reset(); + }); + + // Wait for port reset + // TODO handle disconnect during reset? + let result = runtime::run_with_timeout( + Duration::from_secs(1), + poll_fn(|cx| { + let state = &self.port_states[port]; + + state.notify.register(cx.waker()); + if !self.regs.ports.read(port).portsc.port_reset() { + Poll::Ready(()) + } else { + Poll::Pending + } + }), + ) + .await; + + match result { + FutureTimeout::Ok(()) => Ok(()), + FutureTimeout::Timeout => Err(Error::TimedOut), + } + } + + fn handle_event(&'static self) { + while let Some(event) = self.event_ring.try_dequeue() { + match event { + Event::PortChange(port) => { + self.handle_port_event(port - 1).ok(); + } + Event::CommandCompletion { address, reply } => { + self.command_ring.notify(address, reply); + } + Event::Transfer { + address, + slot_id, + endpoint_id, + status, + } => { + self.notify_transfer(slot_id, endpoint_id, address, status); + } + } + } + + self.regs + .set_interrupter_0_dequeue_pointer(self.event_ring.dequeue_pointer()); + } +} + +impl UsbHostController for Xhci {} + +impl CommandExecutor for Xhci { + fn ring_doorbell(&self, index: usize, target: u8) { + self.regs.ring_doorbell(index, target); + } +} + +impl Device for Xhci { + unsafe fn init(&'static self) -> Result<(), Error> { + log::info!("Init USB xHCI"); + + self.regs.reset(); + self.regs.set_max_slot_count(); + + let erst = EventRingSegmentTable::for_event_rings(&[&self.event_ring])?; + let dcbaa = self.dcbaa.read(); + + self.regs + .configure(&dcbaa, &self.command_ring, &self.event_ring, &erst); + + let bus = UsbBusManager::register_bus(self); + self.bus_address.init(bus); + + for port in 0..self.port_count { + let p = self.regs.ports.read(port); + if p.portsc.current_connect_status() { + self.handle_device_attached(port).ok(); + } + } + + Ok(()) + } + + unsafe fn init_irq(&'static self) -> Result<(), Error> { + log::info!("Init USB xHCI IRQ"); + Ok(()) + } + + fn display_name(&self) -> &'static str { + "USB xHCI" + } +} + +impl InterruptHandler for Xhci { + fn handle_irq(&'static self, _vector: Option) -> bool { + if let Some(status) = self.regs.handle_interrupt() { + if status.event_interrupt() { + self.handle_event(); + } + + true + } else { + false + } + } +} diff --git a/driver/usb/xhci/src/device.rs b/driver/usb/xhci/src/device.rs new file mode 100644 index 00000000..71d6d750 --- /dev/null +++ b/driver/usb/xhci/src/device.rs @@ -0,0 +1,151 @@ +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use futures_util::{future::BoxFuture, FutureExt}; +use libk_mm::address::IntoRaw; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; +use xhci_lib::context::{self, InputHandler}; +use ygg_driver_usb::{ + device::{UsbBusAddress, UsbDevice}, + info::UsbEndpointType, + pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess}, + UsbDirection, UsbHostController, +}; +use yggdrasil_abi::error::Error; + +use crate::{ + pipe::InterruptInPipe, + ring::{transfer::InterruptInTransferRing, GenericTransferRing}, + Xhci, XhciContext, +}; + +// TODO device context information +pub struct XhciBusDevice { + pub(crate) port_id: u8, + pub(crate) slot_id: u8, + pub(crate) bus_address: UsbBusAddress, + + pub(crate) xhci: &'static Xhci, + + pub(crate) context: Arc>, + pub(crate) rings: IrqSafeRwLock>>, + pub(crate) control_pipe: UsbControlPipeAccess, +} + +impl XhciBusDevice { + fn dci(ty: UsbEndpointType, dir: UsbDirection, number: u8) -> usize { + match ty { + UsbEndpointType::Control => (number as usize) * 2 + 1, + _ => { + let off = match dir { + UsbDirection::Out => 0, + UsbDirection::In => 1, + }; + + (number as usize * 2) + off + } + } + } + + async fn setup_endpoint_inner( + &self, + ring: Arc, + dci: u8, + ty: UsbEndpointType, + direction: UsbDirection, + ) -> Result<(), Error> { + log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction); + + let mut input = self.context.input.write(); + + let ep_type = match (ty, direction) { + (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn, + _ => todo!(), + }; + + { + let control = input.control_mut(); + + control.set_add_context_flag(0); + control.clear_add_context_flag(1); + control.set_add_context_flag(dci as _); + } + + { + let slot = input.device_mut().slot_mut(); + + slot.set_context_entries(31); + } + + { + let ep_cx = input.device_mut().endpoint_mut(dci as _); + + ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); + ep_cx.set_dequeue_cycle_state(); + ep_cx.set_endpoint_type(ep_type); + ep_cx.set_error_count(3); + + // TODO get from endpoint info + ep_cx.set_max_packet_size(8); + } + + self.xhci + .command_ring + .configure_endpoint(self.xhci, self.slot_id, &mut input) + .await?; + + self.rings.write().push(ring.clone()); + self.xhci.register_endpoint(self.slot_id, dci, ring); + + Ok(()) + } +} + +impl UsbDevice for XhciBusDevice { + fn control_pipe(&self) -> Option<&UsbControlPipeAccess> { + Some(&self.control_pipe) + } + + fn port_number(&self) -> u8 { + self.port_id + } + + fn bus_address(&self) -> UsbBusAddress { + self.bus_address + } + + fn controller(&self) -> &'static dyn UsbHostController { + self.xhci + } + + fn open_interrupt_in_pipe<'a>( + &'a self, + number: u8, + ) -> BoxFuture> { + async move { + let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8; + let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?); + + self.setup_endpoint_inner( + ring.clone(), + dci, + UsbEndpointType::Interrupt, + UsbDirection::In, + ) + .await?; + + let pipe = InterruptInPipe::new(self.xhci, self.slot_id, number, dci, ring); + + Ok(UsbInterruptInPipeAccess(Box::new(pipe))) + } + .boxed() + } + + fn handle_detach(&self) { + log::info!("Device detach handler"); + for ring in self.rings.write().drain(..) { + self.xhci + .shutdown_endpoint(ring.slot_id(), ring.endpoint_id()); + } + } + + fn debug(&self) {} +} diff --git a/driver/usb/xhci/src/lib.rs b/driver/usb/xhci/src/lib.rs index 649d1604..69bdef3b 100644 --- a/driver/usb/xhci/src/lib.rs +++ b/driver/usb/xhci/src/lib.rs @@ -3,202 +3,30 @@ extern crate alloc; -use core::{ - future::poll_fn, - sync::atomic::{AtomicBool, Ordering}, - task::{Context, Poll}, - time::Duration, -}; +mod controller; +mod device; +mod pipe; +mod regs; +mod ring; -use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec::Vec}; -use device_api::{ - interrupt::{InterruptAffinity, InterruptHandler}, - Device, -}; -use futures_util::{future::BoxFuture, FutureExt}; -use libk_mm::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, - PageBox, -}; -use libk_thread::runtime::{self, FutureTimeout}; -use libk_util::{sync::spin_rwlock::IrqSafeRwLock, waker::QueueWaker, OneTimeInit}; -use pipe::ControlPipe; -use regs::{Mapper, Regs}; -use ring::{ - transfer::InterruptInTransferRing, CommandExecutor, CommandRing, EventRing, GenericTransferRing, -}; -use xhci_lib::context::{self, InputHandler}; +use alloc::boxed::Box; +pub use controller::Xhci; +use device_api::{interrupt::InterruptAffinity, Device}; +use libk_mm::PageBox; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; +use regs::Mapper; +use xhci_lib::context; use ygg_driver_pci::{ device::{PciDeviceInfo, PreferredInterruptMode}, PciCommandRegister, PciConfigurationSpace, }; -use ygg_driver_usb::{ - bus::UsbBusManager, - device::{UsbBusAddress, UsbDevice, UsbDeviceAccess}, - info::UsbEndpointType, - pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess}, - UsbDirection, UsbHostController, -}; use yggdrasil_abi::error::Error; -use crate::{ - pipe::InterruptInPipe, - ring::{ControlTransferRing, Event, EventRingSegmentTable}, -}; - -mod pipe; -mod regs; -mod ring; - pub struct XhciContext { pub(crate) input: IrqSafeRwLock>>, pub(crate) output: PageBox>, } -// TODO device context information -pub struct XhciBusDevice { - port_id: u8, - slot_id: u8, - bus_address: UsbBusAddress, - - xhci: &'static Xhci, - - context: Arc>, - control_pipe: UsbControlPipeAccess, -} - -pub struct Xhci { - regs: Regs, - - bus_address: OneTimeInit, - - port_count: usize, - // TODO use to allocate proper contexts - #[allow(unused)] - context_size: usize, - - dcbaa: IrqSafeRwLock>, - endpoints: IrqSafeRwLock>>, - event_ring: EventRing, - command_ring: CommandRing, - - spawned_ports: Vec, - port_reset_notify: QueueWaker, -} - -impl XhciBusDevice { - fn dci(ty: UsbEndpointType, dir: UsbDirection, number: u8) -> usize { - match ty { - UsbEndpointType::Control => (number as usize) * 2 + 1, - _ => { - let off = match dir { - UsbDirection::Out => 0, - UsbDirection::In => 1, - }; - - (number as usize * 2) + off - } - } - } - - async fn setup_endpoint_inner( - &self, - ring: Arc, - dci: u8, - ty: UsbEndpointType, - direction: UsbDirection, - ) -> Result<(), Error> { - log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction); - - let mut input = self.context.input.write(); - - let ep_type = match (ty, direction) { - (UsbEndpointType::Interrupt, UsbDirection::In) => context::EndpointType::InterruptIn, - _ => todo!(), - }; - - { - let control = input.control_mut(); - - control.set_add_context_flag(0); - control.clear_add_context_flag(1); - control.set_add_context_flag(dci as _); - } - - { - let slot = input.device_mut().slot_mut(); - - slot.set_context_entries(31); - } - - { - let ep_cx = input.device_mut().endpoint_mut(dci as _); - - ep_cx.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); - ep_cx.set_dequeue_cycle_state(); - ep_cx.set_endpoint_type(ep_type); - ep_cx.set_error_count(3); - - // TODO get from endpoint info - ep_cx.set_max_packet_size(8); - } - - self.xhci - .command_ring - .configure_endpoint(self.xhci, self.slot_id, &mut input) - .await?; - - self.xhci.register_endpoint(self.slot_id, dci, ring); - - Ok(()) - } -} - -impl UsbDevice for XhciBusDevice { - fn control_pipe(&self) -> Option<&UsbControlPipeAccess> { - Some(&self.control_pipe) - } - - fn port_number(&self) -> u8 { - self.port_id - } - - fn bus_address(&self) -> UsbBusAddress { - self.bus_address - } - - fn controller(&self) -> &'static dyn UsbHostController { - self.xhci - } - - fn open_interrupt_in_pipe<'a>( - &'a self, - number: u8, - ) -> BoxFuture> { - async move { - let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8; - let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?); - - self.setup_endpoint_inner( - ring.clone(), - dci, - UsbEndpointType::Interrupt, - UsbDirection::In, - ) - .await?; - - let pipe = InterruptInPipe::new(self.xhci, self.slot_id, number, dci, ring); - - Ok(UsbInterruptInPipeAccess(Box::new(pipe))) - } - .boxed() - } - - fn debug(&self) {} -} - -impl UsbHostController for Xhci {} - impl XhciContext<8> { pub fn new_32byte() -> Result { let input = PageBox::new(context::Input::new_32byte())?; @@ -211,262 +39,6 @@ impl XhciContext<8> { } } -impl Xhci { - pub fn new(regs: xhci_lib::Registers) -> Result { - let event_ring = EventRing::new(128)?; - let command_ring = CommandRing::new(128)?; - - let regs = Regs::from(regs); - - let port_count = regs.port_count(); - let slot_count = regs.max_slot_count(); - let context_size = regs.context_size(); - - Ok(Self { - regs, - - bus_address: OneTimeInit::new(), - - port_count, - context_size, - - event_ring, - command_ring, - - dcbaa: IrqSafeRwLock::new(PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1)?), - endpoints: IrqSafeRwLock::new(BTreeMap::new()), - - spawned_ports: Vec::from_iter((0..port_count).map(|_| AtomicBool::new(false))), - port_reset_notify: QueueWaker::new(), - }) - } - - pub fn register_device_context(&self, slot_id: u8, context: PhysicalAddress) { - self.dcbaa.write()[slot_id as usize] = context; - } - - pub fn register_endpoint( - &self, - slot_id: u8, - endpoint_id: u8, - ring: Arc, - ) { - self.endpoints.write().insert((slot_id, endpoint_id), ring); - } - - pub fn notify_transfer( - &self, - slot_id: u8, - endpoint_id: u8, - address: PhysicalAddress, - status: u32, - ) { - if let Some(ep) = self.endpoints.read().get(&(slot_id, endpoint_id)) { - ep.notify(address, status); - } - } - - async fn assign_device( - &'static self, - slot_id: u8, - root_hub_port_number: u8, - ) -> Result, Error> { - let address = 1; - let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?); - - let context = XhciContext::new_32byte()?; - let mut input = context.input.write(); - - // Setup input context - { - let control = input.control_mut(); - - control.set_add_context_flag(0); - control.set_add_context_flag(1); - } - - { - let slot = input.device_mut().slot_mut(); - - slot.set_context_entries(1); - slot.set_interrupter_target(0); - slot.set_usb_device_address(address); - slot.set_root_hub_port_number(root_hub_port_number); - } - - { - let ep0 = input.device_mut().endpoint_mut(1); - - ep0.set_endpoint_type(context::EndpointType::Control); - ep0.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); - ep0.set_dequeue_cycle_state(); - ep0.set_error_count(3); - } - - self.register_device_context(slot_id, unsafe { context.output.as_physical_address() }); - - self.command_ring - .address_device(self, slot_id, &mut input) - .await?; - - self.register_endpoint(slot_id, 1, ring.clone()); - - let pipe = ControlPipe::new(self, slot_id, ring); - - drop(input); - - let bus_address = UsbBusAddress { - bus: *self.bus_address.get(), - device: address, - }; - - let device = XhciBusDevice { - xhci: self, - slot_id, - port_id: root_hub_port_number, - bus_address, - context: Arc::new(context), - control_pipe: UsbControlPipeAccess(Box::new(pipe)), - }; - - Ok(Box::new(device)) - } - - async fn port_task(&'static self, index: usize) -> Result<(), Error> { - log::debug!("Task spawned for port {}", index); - - self.reset_port(index).await?; - - let slot_id = self.command_ring.enable_slot(self).await?; - log::debug!("Enabled slot: {}", slot_id); - - let device = self.assign_device(slot_id, (index + 1) as _).await?; - let device = UsbDeviceAccess::setup(device).await?; - - UsbBusManager::register_device(Arc::new(device)); - - Ok(()) - } - - fn handle_device_attached(&'static self, port: usize) -> Result<(), Error> { - if !self.spawned_ports[port].swap(true, Ordering::Release) { - // Port has not yet been spawned - runtime::spawn(async move { self.port_task(port).await })?; - } - Ok(()) - } - - fn poll_port_reset(&self, cx: &mut Context<'_>, port: usize) -> Poll<()> { - self.port_reset_notify.register(cx.waker()); - if self.regs.ports.read(port).portsc.port_reset_change() { - self.port_reset_notify.remove(cx.waker()); - Poll::Ready(()) - } else { - Poll::Pending - } - } - - async fn reset_port(&self, port: usize) -> Result<(), Error> { - log::debug!("Reset port {}", port); - - self.regs.ports.update(port, |u| { - u.portsc.set_port_reset(); - }); - - // Wait for port reset - let status = runtime::run_with_timeout( - Duration::from_secs(1), - poll_fn(|cx| self.poll_port_reset(cx, port)), - ) - .await; - - match status { - FutureTimeout::Ok(()) => Ok(()), - FutureTimeout::Timeout => Err(Error::TimedOut), - } - } - - fn handle_event(&self) { - while let Some(event) = self.event_ring.try_dequeue() { - match event { - Event::PortChange(_) => { - self.port_reset_notify.wake_one(); - } - Event::CommandCompletion { address, reply } => { - self.command_ring.notify(address, reply); - } - Event::Transfer { - address, - slot_id, - endpoint_id, - status, - } => { - self.notify_transfer(slot_id, endpoint_id, address, status); - } - } - } - - self.regs - .set_interrupter_0_dequeue_pointer(self.event_ring.dequeue_pointer()); - } -} - -impl CommandExecutor for Xhci { - fn ring_doorbell(&self, index: usize, target: u8) { - self.regs.ring_doorbell(index, target); - } -} - -impl Device for Xhci { - unsafe fn init(&'static self) -> Result<(), Error> { - log::info!("Init USB xHCI"); - - self.regs.reset(); - self.regs.set_max_slot_count(); - - let erst = EventRingSegmentTable::for_event_rings(&[&self.event_ring])?; - let dcbaa = self.dcbaa.read(); - - self.regs - .configure(&dcbaa, &self.command_ring, &self.event_ring, &erst); - - let bus = UsbBusManager::register_bus(self); - self.bus_address.init(bus); - - for port in 0..self.port_count { - let p = self.regs.ports.read(port); - if p.portsc.current_connect_status() { - self.handle_device_attached(port).ok(); - } - } - - Ok(()) - } - - unsafe fn init_irq(&'static self) -> Result<(), Error> { - log::info!("Init USB xHCI IRQ"); - Ok(()) - } - - fn display_name(&self) -> &'static str { - "USB xHCI" - } -} - -impl InterruptHandler for Xhci { - fn handle_irq(&self, _vector: Option) -> bool { - if let Some(status) = self.regs.handle_interrupt() { - if status.event_interrupt() { - self.handle_event(); - } - - true - } else { - false - } - } -} - pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { // TODO Chip Hardware Reset let bar0 = info diff --git a/driver/usb/xhci/src/ring/mod.rs b/driver/usb/xhci/src/ring/mod.rs index eba9f459..8f589fef 100644 --- a/driver/usb/xhci/src/ring/mod.rs +++ b/driver/usb/xhci/src/ring/mod.rs @@ -20,8 +20,12 @@ pub trait GenericRing { } pub trait GenericTransferRing: GenericRing + Send + Sync { + fn slot_id(&self) -> u8; + fn endpoint_id(&self) -> u8; + fn dequeue_pointer(&self) -> PhysicalAddress; fn notify(&self, address: PhysicalAddress, status: u32); + fn shutdown(&self); } define_bitfields! { diff --git a/driver/usb/xhci/src/ring/transfer.rs b/driver/usb/xhci/src/ring/transfer.rs index 57346277..8412d467 100644 --- a/driver/usb/xhci/src/ring/transfer.rs +++ b/driver/usb/xhci/src/ring/transfer.rs @@ -1,6 +1,6 @@ use core::{ mem::{size_of, MaybeUninit}, - sync::atomic::{AtomicU64, Ordering}, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; @@ -40,6 +40,7 @@ pub struct ControlTransferRing { ep_id: u8, transfer_id: AtomicU64, + shutdown: AtomicBool, } pub struct InterruptInTransferRing { @@ -50,6 +51,8 @@ pub struct InterruptInTransferRing { slot_id: u8, ep_id: u8, + + shutdown: AtomicBool, } struct TransferBuilder<'a> { @@ -182,6 +185,22 @@ impl GenericTransferRing for ControlTransferRing { status.signal(value); } } + + fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + let mut completions = self.completions.write(); + while let Some((_, status)) = completions.pop_first() { + status.abort(); + } + } + + fn slot_id(&self) -> u8 { + self.slot_id + } + + fn endpoint_id(&self) -> u8 { + self.ep_id + } } impl GenericRing for InterruptInTransferRing { @@ -211,6 +230,22 @@ impl GenericTransferRing for InterruptInTransferRing { status.signal(value); } } + + fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + let mut completions = self.completions.write(); + while let Some((_, status)) = completions.pop_first() { + status.abort(); + } + } + + fn slot_id(&self) -> u8 { + self.slot_id + } + + fn endpoint_id(&self) -> u8 { + self.ep_id + } } impl InterruptInTransferRing { @@ -228,6 +263,7 @@ impl InterruptInTransferRing { slot_id, ep_id, capacity, + shutdown: AtomicBool::new(false), }) } @@ -236,6 +272,12 @@ impl InterruptInTransferRing { executor: &E, buffer: &mut PageBox<[u8]>, ) -> Result { + // Don't even try to start the transfer + if self.shutdown.load(Ordering::Acquire) { + // TODO use UsbError + return Err(Error::NetworkUnreachable); + } + let status = Arc::new(UsbTransferStatus::new()); let address = self.inner.lock().enqueue(NormalTransferTrb::new( unsafe { buffer.as_physical_address() }, @@ -282,6 +324,8 @@ impl ControlTransferRing { capacity, transfer_id: AtomicU64::new(0), + + shutdown: AtomicBool::new(false), }) } @@ -291,6 +335,12 @@ impl ControlTransferRing { setup: ControlTransferSetup, buffer: Option<(PhysicalAddress, usize, UsbDirection)>, ) -> Result { + // Don't even try to start the transfer + if self.shutdown.load(Ordering::Acquire) { + // TODO use UsbError + return Err(Error::NetworkUnreachable); + } + let mut builder = TransferBuilder::new(self, UsbDirection::In); builder.push_trb(ControlTransferSetupTrb::new(setup)); diff --git a/lib/device-api/src/interrupt.rs b/lib/device-api/src/interrupt.rs index c4959492..e0c768f6 100644 --- a/lib/device-api/src/interrupt.rs +++ b/lib/device-api/src/interrupt.rs @@ -131,7 +131,7 @@ pub trait LocalInterruptController { } pub trait InterruptHandler: Device { - fn handle_irq(&self, vector: Option) -> bool; + fn handle_irq(&'static self, vector: Option) -> bool; } pub struct FixedInterruptTable { From 3f736787352eb94c2e1a363da30f7fbf0feffab5 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 1 Mar 2024 14:56:52 +0200 Subject: [PATCH 201/211] usb/xhci: better port speed handling --- driver/bus/usb/src/bus.rs | 6 +++- driver/bus/usb/src/device.rs | 9 ++++++ driver/usb/xhci/src/controller.rs | 52 +++++++++++-------------------- driver/usb/xhci/src/device.rs | 9 +++++- driver/usb/xhci/src/lib.rs | 50 +++++++++++++++++++++++++++-- driver/usb/xhci/src/regs.rs | 43 +++++++++++++++++++++++++ 6 files changed, 130 insertions(+), 39 deletions(-) diff --git a/driver/bus/usb/src/bus.rs b/driver/bus/usb/src/bus.rs index 945379ba..c882dd77 100644 --- a/driver/bus/usb/src/bus.rs +++ b/driver/bus/usb/src/bus.rs @@ -42,7 +42,11 @@ impl UsbBusManager { pub async fn bus_handler() { loop { let new_device = QUEUE.pop_front().await; - log::debug!("New device connected: {}", new_device.bus_address()); + log::info!( + "New {:?}-speed USB device connected: {}", + new_device.speed(), + new_device.bus_address() + ); class_driver::spawn_driver(new_device).await.ok(); } diff --git a/driver/bus/usb/src/device.rs b/driver/bus/usb/src/device.rs index 12ac26bf..4e5794ea 100644 --- a/driver/bus/usb/src/device.rs +++ b/driver/bus/usb/src/device.rs @@ -30,6 +30,14 @@ pub struct UsbDeviceAccess { pub current_configuration: IrqSafeRwLock>, } +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub enum UsbSpeed { + Low, + Full, + High, + Super, +} + #[allow(unused)] pub trait UsbDevice: Send + Sync { // Endpoint "0" @@ -44,6 +52,7 @@ pub trait UsbDevice: Send + Sync { fn port_number(&self) -> u8; fn bus_address(&self) -> UsbBusAddress; + fn speed(&self) -> UsbSpeed; fn controller(&self) -> &'static dyn UsbHostController; fn handle_detach(&self); diff --git a/driver/usb/xhci/src/controller.rs b/driver/usb/xhci/src/controller.rs index 5d84d330..b733de2b 100644 --- a/driver/usb/xhci/src/controller.rs +++ b/driver/usb/xhci/src/controller.rs @@ -5,12 +5,11 @@ use atomic_enum::atomic_enum; use device_api::{interrupt::InterruptHandler, Device}; use futures_util::task::AtomicWaker; use libk_mm::{ - address::{AsPhysicalAddress, IntoRaw, PhysicalAddress}, + address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; use libk_thread::runtime::{self, FutureTimeout}; use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; -use xhci_lib::context::{self, InputHandler}; use ygg_driver_usb::{ bus::UsbBusManager, device::{UsbBusAddress, UsbDevice, UsbDeviceAccess}, @@ -23,7 +22,7 @@ use yggdrasil_abi::error::Error; use crate::{ device::XhciBusDevice, pipe::ControlPipe, - regs::{Mapper, Regs}, + regs::{Mapper, PortSpeed, Regs}, ring::{ CommandExecutor, CommandRing, ControlTransferRing, Event, EventRing, EventRingSegmentTable, GenericTransferRing, @@ -137,41 +136,17 @@ impl Xhci { async fn assign_device( &'static self, + speed: PortSpeed, slot_id: u8, root_hub_port_number: u8, ) -> Result, Error> { let address = self.address_allocator.allocate().unwrap(); let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?); - let context = XhciContext::new_32byte()?; + let context = + XhciContext::new_32byte_address_device(&ring, speed, address, root_hub_port_number)?; let mut input = context.input.write(); - // Setup input context - { - let control = input.control_mut(); - - control.set_add_context_flag(0); - control.set_add_context_flag(1); - } - - { - let slot = input.device_mut().slot_mut(); - - slot.set_context_entries(1); - slot.set_interrupter_target(0); - slot.set_usb_device_address(address); - slot.set_root_hub_port_number(root_hub_port_number); - } - - { - let ep0 = input.device_mut().endpoint_mut(1); - - ep0.set_endpoint_type(context::EndpointType::Control); - ep0.set_tr_dequeue_pointer(ring.dequeue_pointer().into_raw()); - ep0.set_dequeue_cycle_state(); - ep0.set_error_count(3); - } - self.register_device_context(slot_id, unsafe { context.output.as_physical_address() }); self.command_ring @@ -182,6 +157,12 @@ impl Xhci { let pipe = ControlPipe::new(self, slot_id, ring.clone()); + // TODO: If the device is a Full-speed one, determine its max packet size for the control + // endpoint + if speed == PortSpeed::Full { + todo!() + } + drop(input); let bus_address = UsbBusAddress { @@ -194,6 +175,7 @@ impl Xhci { slot_id, port_id: root_hub_port_number, bus_address, + speed, context: Arc::new(context), rings: IrqSafeRwLock::new(vec![ring]), control_pipe: UsbControlPipeAccess(Box::new(pipe)), @@ -204,14 +186,16 @@ impl Xhci { async fn port_task(&'static self, index: usize) -> Result<(), Error> { let state = &self.port_states[index]; - log::debug!("Task spawned for port {}", index); self.reset_port(index).await?; - let slot_id = self.command_ring.enable_slot(self).await?; - log::debug!("Enabled slot: {}", slot_id); + let regs = self.regs.ports.read(index); + let speed = + PortSpeed::try_from(regs.portsc.port_speed()).map_err(|_| Error::InvalidArgument)?; - let device = self.assign_device(slot_id, (index + 1) as _).await?; + let slot_id = self.command_ring.enable_slot(self).await?; + + let device = self.assign_device(speed, slot_id, (index + 1) as _).await?; let device = UsbDeviceAccess::setup(device).await?; let old = state.address.write().replace(device.bus_address()); diff --git a/driver/usb/xhci/src/device.rs b/driver/usb/xhci/src/device.rs index 71d6d750..fcad35b7 100644 --- a/driver/usb/xhci/src/device.rs +++ b/driver/usb/xhci/src/device.rs @@ -4,7 +4,7 @@ use libk_mm::address::IntoRaw; use libk_util::sync::spin_rwlock::IrqSafeRwLock; use xhci_lib::context::{self, InputHandler}; use ygg_driver_usb::{ - device::{UsbBusAddress, UsbDevice}, + device::{UsbBusAddress, UsbDevice, UsbSpeed}, info::UsbEndpointType, pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess}, UsbDirection, UsbHostController, @@ -13,6 +13,7 @@ use yggdrasil_abi::error::Error; use crate::{ pipe::InterruptInPipe, + regs::PortSpeed, ring::{transfer::InterruptInTransferRing, GenericTransferRing}, Xhci, XhciContext, }; @@ -23,6 +24,8 @@ pub struct XhciBusDevice { pub(crate) slot_id: u8, pub(crate) bus_address: UsbBusAddress, + pub(crate) speed: PortSpeed, + pub(crate) xhci: &'static Xhci, pub(crate) context: Arc>, @@ -147,5 +150,9 @@ impl UsbDevice for XhciBusDevice { } } + fn speed(&self) -> UsbSpeed { + self.speed.into() + } + fn debug(&self) {} } diff --git a/driver/usb/xhci/src/lib.rs b/driver/usb/xhci/src/lib.rs index 69bdef3b..e6494564 100644 --- a/driver/usb/xhci/src/lib.rs +++ b/driver/usb/xhci/src/lib.rs @@ -12,10 +12,11 @@ mod ring; use alloc::boxed::Box; pub use controller::Xhci; use device_api::{interrupt::InterruptAffinity, Device}; -use libk_mm::PageBox; +use libk_mm::{address::IntoRaw, PageBox}; use libk_util::sync::spin_rwlock::IrqSafeRwLock; -use regs::Mapper; -use xhci_lib::context; +use regs::{Mapper, PortSpeed}; +use ring::{ControlTransferRing, GenericTransferRing}; +use xhci_lib::context::{self, InputHandler}; use ygg_driver_pci::{ device::{PciDeviceInfo, PreferredInterruptMode}, PciCommandRegister, PciConfigurationSpace, @@ -37,6 +38,49 @@ impl XhciContext<8> { output, }) } + + pub fn new_32byte_address_device( + default_control_ring: &ControlTransferRing, + speed: PortSpeed, + address: u8, + root_hub_port_number: u8, + ) -> Result { + let mut input = PageBox::new(context::Input::new_32byte())?; + let output = PageBox::new(context::Device::new_32byte())?; + + // Setup input context + { + let control = input.control_mut(); + + control.set_add_context_flag(0); + control.set_add_context_flag(1); + } + + { + let slot = input.device_mut().slot_mut(); + + slot.set_context_entries(1); + slot.set_interrupter_target(0); + slot.set_usb_device_address(address); + slot.set_root_hub_port_number(root_hub_port_number); + slot.set_speed(speed.into()); + } + + { + let ep0 = input.device_mut().endpoint_mut(1); + + ep0.set_endpoint_type(context::EndpointType::Control); + ep0.set_tr_dequeue_pointer(default_control_ring.dequeue_pointer().into_raw()); + ep0.set_dequeue_cycle_state(); + ep0.set_error_count(3); + ep0.set_max_packet_size(speed.default_max_packet_size() as _); + } + + Ok(Self { + input: IrqSafeRwLock::new(input), + output, + }) + } } pub fn probe(info: &PciDeviceInfo) -> Result<&'static dyn Device, Error> { diff --git a/driver/usb/xhci/src/regs.rs b/driver/usb/xhci/src/regs.rs index d3b22a21..29de28a3 100644 --- a/driver/usb/xhci/src/regs.rs +++ b/driver/usb/xhci/src/regs.rs @@ -14,6 +14,8 @@ use xhci_lib::{ PortRegisterSet, }, }; +use ygg_driver_usb::device::UsbSpeed; +use yggdrasil_abi::primitive_enum; use crate::ring::{CommandRing, EventRing, EventRingSegmentTable, GenericRing}; @@ -238,3 +240,44 @@ impl xhci_lib::accessor::Mapper for Mapper { // TODO } } + +// Register value definitions + +primitive_enum! { + pub enum PortSpeed: u8 { + Full = 1, + Low = 2, + High = 3, + SuperGen1x1 = 4, + SuperGen2x1 = 5, + SuperGen1x2 = 6, + SuperGen2x2 = 7, + } +} + +impl PortSpeed { + pub fn default_max_packet_size(&self) -> usize { + match self { + Self::Low => 8, + Self::High => 64, + Self::SuperGen1x1 | Self::SuperGen1x2 | Self::SuperGen2x1 | Self::SuperGen2x2 => 512, + + // See Section 4.3., point 7. of the initialization list + Self::Full => 8, + } + } +} + +impl From for UsbSpeed { + fn from(value: PortSpeed) -> Self { + match value { + PortSpeed::Low => UsbSpeed::Low, + PortSpeed::Full => UsbSpeed::Full, + PortSpeed::High => UsbSpeed::High, + PortSpeed::SuperGen1x1 + | PortSpeed::SuperGen1x2 + | PortSpeed::SuperGen2x1 + | PortSpeed::SuperGen2x2 => UsbSpeed::Super, + } + } +} From 6cd960655656800544bfe56aeae659fa4c967799 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 1 Mar 2024 15:16:26 +0200 Subject: [PATCH 202/211] usb: use UsbError instead of plain Error --- driver/bus/usb/src/class_driver/mod.rs | 16 ++++++------- driver/bus/usb/src/communication.rs | 13 ++++++----- driver/bus/usb/src/descriptor.rs | 6 ++--- driver/bus/usb/src/device.rs | 27 ++++++++++++---------- driver/bus/usb/src/error.rs | 23 +++++++++++++++++++ driver/bus/usb/src/pipe/control.rs | 31 +++++++++++++------------- driver/bus/usb/src/pipe/interrupt.rs | 7 +++--- driver/usb/xhci/src/controller.rs | 28 +++++++++++++---------- driver/usb/xhci/src/device.rs | 10 ++++----- driver/usb/xhci/src/lib.rs | 14 +++++++----- driver/usb/xhci/src/pipe.rs | 6 ++--- driver/usb/xhci/src/ring/command.rs | 19 +++++++++------- driver/usb/xhci/src/ring/event.rs | 12 +++++----- driver/usb/xhci/src/ring/transfer.rs | 24 +++++++++----------- 14 files changed, 136 insertions(+), 100 deletions(-) diff --git a/driver/bus/usb/src/class_driver/mod.rs b/driver/bus/usb/src/class_driver/mod.rs index 013471f2..0cce9cef 100644 --- a/driver/bus/usb/src/class_driver/mod.rs +++ b/driver/bus/usb/src/class_driver/mod.rs @@ -3,17 +3,15 @@ use core::mem::MaybeUninit; use alloc::sync::Arc; use libk_mm::PageBox; use libk_thread::runtime; -use yggdrasil_abi::{ - error::Error, - io::{KeyboardKey, KeyboardKeyEvent}, -}; +use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent}; use crate::{ device::UsbDeviceAccess, + error::UsbError, info::{UsbDeviceClass, UsbDeviceProtocol}, }; -pub async fn spawn_driver(device: Arc) -> Result<(), Error> { +pub async fn spawn_driver(device: Arc) -> Result<(), UsbError> { // TODO query all configurations? let device_info = &device.info; let config_info = device.query_configuration_info(0).await?; @@ -39,7 +37,9 @@ pub async fn spawn_driver(device: Arc) -> Result<(), Error> { }; match (class, subclass, protocol) { - (UsbDeviceClass::Hid, 0x01, _) => runtime::spawn(keyboard_driver(device)), + (UsbDeviceClass::Hid, 0x01, _) => { + runtime::spawn(keyboard_driver(device)).map_err(UsbError::SystemError) + } (_, _, _) => Ok(()), } } else { @@ -47,7 +47,7 @@ pub async fn spawn_driver(device: Arc) -> Result<(), Error> { } } -pub async fn keyboard_driver(device: Arc) -> Result<(), Error> { +pub async fn keyboard_driver(device: Arc) -> Result<(), UsbError> { const MODIFIER_MAP: &[KeyboardKey] = &[ KeyboardKey::LControl, KeyboardKey::LShift, @@ -179,7 +179,7 @@ pub async fn keyboard_driver(device: Arc) -> Result<(), Error> let pipe = device.open_interrupt_in_pipe(1).await?; - let mut buffer = PageBox::new_slice(0, 8)?; + let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?; let mut state = KeyboardState::new(); let mut events = [MaybeUninit::uninit(); 16]; diff --git a/driver/bus/usb/src/communication.rs b/driver/bus/usb/src/communication.rs index ee7b184c..823a5702 100644 --- a/driver/bus/usb/src/communication.rs +++ b/driver/bus/usb/src/communication.rs @@ -7,7 +7,8 @@ use core::{ use alloc::{sync::Arc, vec::Vec}; use futures_util::task::AtomicWaker; use libk_mm::address::PhysicalAddress; -use yggdrasil_abi::error::Error; + +use crate::error::UsbError; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum UsbDirection { @@ -65,14 +66,14 @@ impl UsbTransferResult { } impl UsbControlTransfer { - pub async fn wait(&self) -> Result { + pub async fn wait(&self) -> Result { let sub_length = self.status.wait().await?; Ok(self.length.saturating_sub(sub_length)) } } impl UsbInterruptTransfer { - pub async fn wait(&self) -> Result { + pub async fn wait(&self) -> Result { let sub_length = self.status.wait().await?; Ok(self.length.saturating_sub(sub_length)) } @@ -86,15 +87,15 @@ impl UsbTransferStatus { } } - pub(crate) async fn wait(&self) -> Result { + pub(crate) async fn wait(&self) -> Result { poll_fn(|cx| { self.poll(cx).map(|v| { if v.is_success() { Ok(v.sub_length()) } else if v.is_aborted() { - Err(Error::NetworkUnreachable) + Err(UsbError::DeviceDisconnected) } else { - Err(Error::InvalidOperation) + Err(UsbError::TransferFailed) } }) }) diff --git a/driver/bus/usb/src/descriptor.rs b/driver/bus/usb/src/descriptor.rs index 6cc4e276..f99468ce 100644 --- a/driver/bus/usb/src/descriptor.rs +++ b/driver/bus/usb/src/descriptor.rs @@ -1,7 +1,7 @@ use bytemuck::{Pod, Zeroable}; -use yggdrasil_abi::error::Error; use crate::{ + error::UsbError, info::{UsbDeviceClass, UsbDeviceProtocol, UsbEndpointType}, UsbDirection, }; @@ -134,13 +134,13 @@ impl UsbDeviceDescriptor { UsbDeviceProtocol::try_from(self.device_protocol).unwrap_or(UsbDeviceProtocol::Unknown) } - pub fn max_packet_size(&self) -> Result { + pub fn max_packet_size(&self) -> Result { match self.max_packet_size_0 { 8 => Ok(8), 16 => Ok(16), 32 => Ok(32), 64 => Ok(64), - _ => Err(Error::InvalidArgument), + _ => Err(UsbError::InvalidDescriptorField), } } } diff --git a/driver/bus/usb/src/device.rs b/driver/bus/usb/src/device.rs index 4e5794ea..693221f8 100644 --- a/driver/bus/usb/src/device.rs +++ b/driver/bus/usb/src/device.rs @@ -4,9 +4,9 @@ use alloc::{boxed::Box, vec::Vec}; use futures_util::future::BoxFuture; use libk_mm::PageBox; use libk_util::sync::spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard}; -use yggdrasil_abi::error::Error; use crate::{ + error::UsbError, info::{UsbConfigurationInfo, UsbDeviceInfo, UsbEndpointInfo, UsbInterfaceInfo}, pipe::{ control::{ConfigurationDescriptorEntry, UsbControlPipeAccess}, @@ -41,12 +41,12 @@ pub enum UsbSpeed { #[allow(unused)] pub trait UsbDevice: Send + Sync { // Endpoint "0" - fn control_pipe(&self) -> Option<&UsbControlPipeAccess>; + fn control_pipe(&self) -> &UsbControlPipeAccess; fn open_interrupt_in_pipe<'a>( &'a self, number: u8, - ) -> BoxFuture> { + ) -> BoxFuture> { unimplemented!() } @@ -67,9 +67,9 @@ impl UsbDeviceAccess { /// * Device is not yet configured /// * Control pipe for the device has been properly set up /// * Device has been assigned a bus address - pub async fn setup(raw: Box) -> Result { - let control = raw.control_pipe().ok_or(Error::InvalidOperation)?; - let mut string_buffer = PageBox::new_uninit()?; + pub async fn setup(raw: Box) -> Result { + let control = raw.control_pipe(); + let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?; let device_desc = control.query_device_descriptor().await?; @@ -111,9 +111,9 @@ impl UsbDeviceAccess { pub async fn select_configuration bool>( &self, predicate: F, - ) -> Result, Error> { + ) -> Result, UsbError> { let mut current_config = self.current_configuration.write(); - let control_pipe = self.control_pipe().ok_or(Error::InvalidOperation)?; + let control_pipe = self.control_pipe(); for i in 0..self.num_configurations { let info = self.query_configuration_info(i).await?; @@ -133,12 +133,15 @@ impl UsbDeviceAccess { Ok(None) } - pub async fn query_configuration_info(&self, index: u8) -> Result { + pub async fn query_configuration_info( + &self, + index: u8, + ) -> Result { if index >= self.num_configurations { - return Err(Error::InvalidArgument); + return Err(UsbError::InvalidConfiguration); } - let mut string_buffer = PageBox::new_uninit()?; - let control_pipe = self.control_pipe().ok_or(Error::InvalidOperation)?; + let mut string_buffer = PageBox::new_uninit().map_err(UsbError::MemoryError)?; + let control_pipe = self.control_pipe(); let query = control_pipe.query_configuration_descriptor(index).await?; let configuration_name = control_pipe diff --git a/driver/bus/usb/src/error.rs b/driver/bus/usb/src/error.rs index 044b3597..a4cc18e9 100644 --- a/driver/bus/usb/src/error.rs +++ b/driver/bus/usb/src/error.rs @@ -2,6 +2,29 @@ use yggdrasil_abi::error::Error; #[derive(Debug)] pub enum UsbError { + /// Could not allocate memory for some device structure MemoryError(Error), + /// Other system errors + SystemError(Error), + // HC-side init stage errors OutOfAddresses, + HostControllerCommandFailed(u8), + PortResetFailed, + PortInitFailed, + // Setup stage errors + InvalidConfiguration, + InvalidDescriptorField, + // Runtime errors + DeviceBusy, + DeviceDisconnected, + TransferFailed, +} + +impl From for Error { + fn from(value: UsbError) -> Self { + match value { + UsbError::MemoryError(e) => e, + _ => Error::InvalidOperation, + } + } } diff --git a/driver/bus/usb/src/pipe/control.rs b/driver/bus/usb/src/pipe/control.rs index 14618001..2d68d165 100644 --- a/driver/bus/usb/src/pipe/control.rs +++ b/driver/bus/usb/src/pipe/control.rs @@ -10,13 +10,13 @@ use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, PageBox, }; -use yggdrasil_abi::error::Error; use crate::{ descriptor::{ UsbConfigurationDescriptor, UsbDeviceDescriptor, UsbDeviceQualifier, UsbEndpointDescriptor, UsbInterfaceDescriptor, UsbOtherSpeedConfiguration, }, + error::UsbError, UsbControlTransfer, UsbDirection, }; @@ -66,9 +66,9 @@ impl UsbDeviceRequest for U { const B_REQUEST: u8 = 0x06; } -fn decode_usb_string(bytes: &[u8]) -> Result { +fn decode_usb_string(bytes: &[u8]) -> Result { if bytes.len() % 2 != 0 { - return Err(Error::InvalidArgument); + return Err(UsbError::InvalidDescriptorField); } char::decode_utf16( @@ -78,7 +78,7 @@ fn decode_usb_string(bytes: &[u8]) -> Result { .map(|[&a, &b]| u16::from_le_bytes([a, b])), ) .collect::>() - .map_err(|_| Error::InvalidArgument) + .map_err(|_| UsbError::InvalidDescriptorField) } // Pipe impl @@ -88,7 +88,7 @@ pub trait UsbControlPipe: UsbGenericPipe + Send + Sync { &self, setup: ControlTransferSetup, data: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result; + ) -> Result; fn complete_transfer(&self, transfer: UsbControlTransfer); } @@ -187,7 +187,7 @@ impl UsbControlPipeAccess { &self, setup: ControlTransferSetup, buffer: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result<(), Error> { + ) -> Result<(), UsbError> { let transfer = self.start_transfer(setup, buffer)?; transfer.status.wait().await?; self.complete_transfer(transfer); @@ -198,7 +198,7 @@ impl UsbControlPipeAccess { &self, index: u8, buffer: &mut PageBox<[MaybeUninit]>, - ) -> Result<(), Error> { + ) -> Result<(), UsbError> { self.perform_value_control( ControlTransferSetup { bm_request_type: 0b10000000, @@ -219,9 +219,10 @@ impl UsbControlPipeAccess { pub async fn query_configuration_descriptor( &self, index: u8, - ) -> Result { + ) -> Result { // First, query the real length of the descriptor - let mut buffer = PageBox::new_uninit_slice(size_of::())?; + let mut buffer = PageBox::new_uninit_slice(size_of::()) + .map_err(UsbError::MemoryError)?; self.fill_configuation_descriptor(index, &mut buffer) .await?; let buffer = unsafe { PageBox::assume_init_slice(buffer) }; @@ -237,7 +238,7 @@ impl UsbControlPipeAccess { } // Otherwise, query the rest of the data - let mut buffer = PageBox::new_uninit_slice(total_len)?; + let mut buffer = PageBox::new_uninit_slice(total_len).map_err(UsbError::MemoryError)?; self.fill_configuation_descriptor(index, &mut buffer) .await?; let buffer = unsafe { PageBox::assume_init_slice(buffer) }; @@ -253,8 +254,8 @@ impl UsbControlPipeAccess { Ok(ConfigurationDescriptorQuery { buffer }) } - pub async fn query_device_descriptor(&self) -> Result, Error> { - let mut output = PageBox::new_uninit()?; + pub async fn query_device_descriptor(&self) -> Result, UsbError> { + let mut output = PageBox::new_uninit().map_err(UsbError::MemoryError)?; self.perform_value_control( ControlTransferSetup { bm_request_type: 0b10000000, @@ -274,7 +275,7 @@ impl UsbControlPipeAccess { &self, index: u8, buffer: &mut PageBox>, - ) -> Result { + ) -> Result { self.perform_value_control( ControlTransferSetup { bm_request_type: 0b10000000, @@ -297,7 +298,7 @@ impl UsbControlPipeAccess { &self, w_value: u16, w_index: u16, - ) -> Result<(), Error> { + ) -> Result<(), UsbError> { self.perform_value_control( ControlTransferSetup { bm_request_type: D::BM_REQUEST_TYPE, @@ -311,7 +312,7 @@ impl UsbControlPipeAccess { .await } - pub async fn set_configuration(&self, value: u16) -> Result<(), Error> { + pub async fn set_configuration(&self, value: u16) -> Result<(), UsbError> { self.perform_action::(value, 0).await } } diff --git a/driver/bus/usb/src/pipe/interrupt.rs b/driver/bus/usb/src/pipe/interrupt.rs index 649e1beb..a474db1a 100644 --- a/driver/bus/usb/src/pipe/interrupt.rs +++ b/driver/bus/usb/src/pipe/interrupt.rs @@ -2,21 +2,20 @@ use core::ops::Deref; use alloc::boxed::Box; use libk_mm::PageBox; -use yggdrasil_abi::error::Error; -use crate::communication::UsbInterruptTransfer; +use crate::{communication::UsbInterruptTransfer, error::UsbError}; use super::UsbGenericPipe; pub trait UsbInterruptInPipe: UsbGenericPipe + Send + Sync { - fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result; + fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result; fn complete_transfer(&self, transfer: UsbInterruptTransfer); } pub struct UsbInterruptInPipeAccess(pub Box); impl UsbInterruptInPipeAccess { - pub async fn read<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], Error> { + pub async fn read<'a>(&self, buffer: &'a mut PageBox<[u8]>) -> Result<&'a [u8], UsbError> { let transfer = self.start_read(buffer)?; let len = transfer.wait().await?; self.complete_transfer(transfer); diff --git a/driver/usb/xhci/src/controller.rs b/driver/usb/xhci/src/controller.rs index b733de2b..787a9fb0 100644 --- a/driver/usb/xhci/src/controller.rs +++ b/driver/usb/xhci/src/controller.rs @@ -13,6 +13,7 @@ use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit}; use ygg_driver_usb::{ bus::UsbBusManager, device::{UsbBusAddress, UsbDevice, UsbDeviceAccess}, + error::UsbError, pipe::control::UsbControlPipeAccess, util::UsbAddressAllocator, UsbHostController, @@ -64,7 +65,7 @@ pub struct Xhci { } impl Xhci { - pub fn new(regs: xhci_lib::Registers) -> Result { + pub fn new(regs: xhci_lib::Registers) -> Result { let event_ring = EventRing::new(128)?; let command_ring = CommandRing::new(128)?; @@ -74,6 +75,9 @@ impl Xhci { let slot_count = regs.max_slot_count(); let context_size = regs.context_size(); + let dcbaa = PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1) + .map_err(UsbError::MemoryError)?; + Ok(Self { regs, @@ -86,7 +90,7 @@ impl Xhci { event_ring, command_ring, - dcbaa: IrqSafeRwLock::new(PageBox::new_slice(PhysicalAddress::ZERO, slot_count + 1)?), + dcbaa: IrqSafeRwLock::new(dcbaa), endpoints: IrqSafeRwLock::new(BTreeMap::new()), port_states: Vec::from_iter((0..port_count).map(|_| PortStruct { @@ -139,7 +143,7 @@ impl Xhci { speed: PortSpeed, slot_id: u8, root_hub_port_number: u8, - ) -> Result, Error> { + ) -> Result, UsbError> { let address = self.address_allocator.allocate().unwrap(); let ring = Arc::new(ControlTransferRing::new(slot_id, 1, 128)?); @@ -184,14 +188,14 @@ impl Xhci { Ok(Box::new(device)) } - async fn port_task(&'static self, index: usize) -> Result<(), Error> { + async fn port_task(&'static self, index: usize) -> Result<(), UsbError> { let state = &self.port_states[index]; self.reset_port(index).await?; let regs = self.regs.ports.read(index); let speed = - PortSpeed::try_from(regs.portsc.port_speed()).map_err(|_| Error::InvalidArgument)?; + PortSpeed::try_from(regs.portsc.port_speed()).map_err(|_| UsbError::PortInitFailed)?; let slot_id = self.command_ring.enable_slot(self).await?; @@ -208,7 +212,7 @@ impl Xhci { Ok(()) } - fn handle_device_attached(&'static self, port: usize) -> Result<(), Error> { + fn handle_device_attached(&'static self, port: usize) -> Result<(), UsbError> { log::info!("Port {}: device attached", port); if let Err(err) = self.port_states[port].state.compare_exchange( @@ -218,14 +222,14 @@ impl Xhci { Ordering::Relaxed, ) { log::warn!("Could not start port init task: port state is {:?}", err); - return Err(Error::InvalidArgument); + return Err(UsbError::DeviceBusy); } - runtime::spawn(async move { self.port_task(port).await })?; + runtime::spawn(async move { self.port_task(port).await }).map_err(UsbError::SystemError)?; Ok(()) } - fn handle_device_detached(&'static self, port: usize) -> Result<(), Error> { + fn handle_device_detached(&'static self, port: usize) -> Result<(), UsbError> { let state = &self.port_states[port]; match state.state.swap(PortState::Disconnected, Ordering::Release) { @@ -248,7 +252,7 @@ impl Xhci { } } - fn handle_port_event(&'static self, port: usize) -> Result<(), Error> { + fn handle_port_event(&'static self, port: usize) -> Result<(), UsbError> { let state = &self.port_states[port]; let port_regs = self.regs.ports.read(port); @@ -265,7 +269,7 @@ impl Xhci { } } - async fn reset_port(&self, port: usize) -> Result<(), Error> { + async fn reset_port(&self, port: usize) -> Result<(), UsbError> { log::debug!("Reset port {}", port); self.regs.ports.update(port, |u| { @@ -291,7 +295,7 @@ impl Xhci { match result { FutureTimeout::Ok(()) => Ok(()), - FutureTimeout::Timeout => Err(Error::TimedOut), + FutureTimeout::Timeout => Err(UsbError::PortResetFailed), } } diff --git a/driver/usb/xhci/src/device.rs b/driver/usb/xhci/src/device.rs index fcad35b7..d93cf15e 100644 --- a/driver/usb/xhci/src/device.rs +++ b/driver/usb/xhci/src/device.rs @@ -5,11 +5,11 @@ use libk_util::sync::spin_rwlock::IrqSafeRwLock; use xhci_lib::context::{self, InputHandler}; use ygg_driver_usb::{ device::{UsbBusAddress, UsbDevice, UsbSpeed}, + error::UsbError, info::UsbEndpointType, pipe::{control::UsbControlPipeAccess, interrupt::UsbInterruptInPipeAccess}, UsbDirection, UsbHostController, }; -use yggdrasil_abi::error::Error; use crate::{ pipe::InterruptInPipe, @@ -54,7 +54,7 @@ impl XhciBusDevice { dci: u8, ty: UsbEndpointType, direction: UsbDirection, - ) -> Result<(), Error> { + ) -> Result<(), UsbError> { log::debug!("Setup endpoint dci #{}: {:?} {:?}", dci, ty, direction); let mut input = self.context.input.write(); @@ -103,8 +103,8 @@ impl XhciBusDevice { } impl UsbDevice for XhciBusDevice { - fn control_pipe(&self) -> Option<&UsbControlPipeAccess> { - Some(&self.control_pipe) + fn control_pipe(&self) -> &UsbControlPipeAccess { + &self.control_pipe } fn port_number(&self) -> u8 { @@ -122,7 +122,7 @@ impl UsbDevice for XhciBusDevice { fn open_interrupt_in_pipe<'a>( &'a self, number: u8, - ) -> BoxFuture> { + ) -> BoxFuture> { async move { let dci = Self::dci(UsbEndpointType::Interrupt, UsbDirection::In, number) as u8; let ring = Arc::new(InterruptInTransferRing::new(self.slot_id, dci as _, 128)?); diff --git a/driver/usb/xhci/src/lib.rs b/driver/usb/xhci/src/lib.rs index e6494564..bc9a635d 100644 --- a/driver/usb/xhci/src/lib.rs +++ b/driver/usb/xhci/src/lib.rs @@ -21,6 +21,7 @@ use ygg_driver_pci::{ device::{PciDeviceInfo, PreferredInterruptMode}, PciCommandRegister, PciConfigurationSpace, }; +use ygg_driver_usb::error::UsbError; use yggdrasil_abi::error::Error; pub struct XhciContext { @@ -29,9 +30,9 @@ pub struct XhciContext { } impl XhciContext<8> { - pub fn new_32byte() -> Result { - let input = PageBox::new(context::Input::new_32byte())?; - let output = PageBox::new(context::Device::new_32byte())?; + pub fn new_32byte() -> Result { + let input = PageBox::new(context::Input::new_32byte()).map_err(UsbError::MemoryError)?; + let output = PageBox::new(context::Device::new_32byte()).map_err(UsbError::MemoryError)?; Ok(Self { input: IrqSafeRwLock::new(input), @@ -44,9 +45,10 @@ impl XhciContext<8> { speed: PortSpeed, address: u8, root_hub_port_number: u8, - ) -> Result { - let mut input = PageBox::new(context::Input::new_32byte())?; - let output = PageBox::new(context::Device::new_32byte())?; + ) -> Result { + let mut input = + PageBox::new(context::Input::new_32byte()).map_err(UsbError::MemoryError)?; + let output = PageBox::new(context::Device::new_32byte()).map_err(UsbError::MemoryError)?; // Setup input context { diff --git a/driver/usb/xhci/src/pipe.rs b/driver/usb/xhci/src/pipe.rs index 6f72ab8a..6697073f 100644 --- a/driver/usb/xhci/src/pipe.rs +++ b/driver/usb/xhci/src/pipe.rs @@ -2,6 +2,7 @@ use alloc::sync::Arc; use libk_mm::{address::PhysicalAddress, PageBox}; use ygg_driver_usb::{ communication::UsbInterruptTransfer, + error::UsbError, pipe::{ control::{ControlTransferSetup, UsbControlPipe}, interrupt::UsbInterruptInPipe, @@ -9,7 +10,6 @@ use ygg_driver_usb::{ }, UsbControlTransfer, UsbDirection, }; -use yggdrasil_abi::error::Error; use crate::{ ring::{transfer::InterruptInTransferRing, ControlTransferRing}, @@ -39,7 +39,7 @@ impl UsbControlPipe for ControlPipe { &self, setup: ControlTransferSetup, data: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { + ) -> Result { self.ring.start_transfer(self.xhci, setup, data) } @@ -57,7 +57,7 @@ impl ControlPipe { impl UsbGenericPipe for InterruptInPipe {} impl UsbInterruptInPipe for InterruptInPipe { - fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result { + fn start_read(&self, buffer: &mut PageBox<[u8]>) -> Result { self.ring.start_transfer(self.xhci, buffer) } diff --git a/driver/usb/xhci/src/ring/command.rs b/driver/usb/xhci/src/ring/command.rs index a45ab9c4..f1da87f8 100644 --- a/driver/usb/xhci/src/ring/command.rs +++ b/driver/usb/xhci/src/ring/command.rs @@ -16,7 +16,8 @@ use libk_util::{ waker::QueueWaker, }; use xhci_lib::context; -use yggdrasil_abi::{define_bitfields, error::Error}; +use ygg_driver_usb::error::UsbError; +use yggdrasil_abi::define_bitfields; use super::{CommandExecutor, GenericRing, LinkTrb}; @@ -88,8 +89,8 @@ impl CommandRingInner { } impl CommandRing { - pub fn new(capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity)?; + pub fn new(capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; Ok(Self { inner: IrqSafeSpinlock::new(CommandRingInner { @@ -115,7 +116,7 @@ impl CommandRing { executor: &E, slot_id: u8, input: &mut PageBox>, - ) -> Result<(), Error> { + ) -> Result<(), UsbError> { self.submit_and_wait(executor, AddressDeviceCommandTrb::new(input, slot_id)) .await?; Ok(()) @@ -126,13 +127,13 @@ impl CommandRing { executor: &E, slot_id: u8, input: &mut PageBox>, - ) -> Result<(), Error> { + ) -> Result<(), UsbError> { self.submit_and_wait(executor, ConfigureEndpointCommandTrb::new(input, slot_id)) .await?; Ok(()) } - pub async fn enable_slot(&self, executor: &E) -> Result { + pub async fn enable_slot(&self, executor: &E) -> Result { self.submit_and_wait(executor, EnableSlotCommandTrb::new()) .await } @@ -141,7 +142,7 @@ impl CommandRing { &self, executor: &E, trb: C, - ) -> Result { + ) -> Result { let token = self.enqueue(trb); executor.ring_doorbell(0, 0); poll_fn(|cx| { @@ -151,7 +152,9 @@ impl CommandRing { if status.completion_code == 1 { Poll::Ready(Ok(status.slot_id)) } else { - Poll::Ready(Err(Error::InvalidOperation)) + Poll::Ready(Err(UsbError::HostControllerCommandFailed( + status.completion_code, + ))) } } else { Poll::Pending diff --git a/driver/usb/xhci/src/ring/event.rs b/driver/usb/xhci/src/ring/event.rs index b132401a..10785ab0 100644 --- a/driver/usb/xhci/src/ring/event.rs +++ b/driver/usb/xhci/src/ring/event.rs @@ -6,7 +6,8 @@ use libk_mm::{ PageBox, }; use libk_util::sync::IrqSafeSpinlock; -use yggdrasil_abi::{define_bitfields, error::Error}; +use ygg_driver_usb::error::UsbError; +use yggdrasil_abi::define_bitfields; use super::{command::CommandReply, GenericRing}; @@ -49,13 +50,14 @@ pub struct EventRing { } impl EventRingSegmentTable { - pub fn for_event_rings(rings: &[&EventRing]) -> Result { + pub fn for_event_rings(rings: &[&EventRing]) -> Result { let entries = PageBox::from_iter_exact(rings.iter().map(|ring| EventRingSegment { address: ring.base(), size: ring.capacity().try_into().unwrap(), _0: 0, _1: 0, - }))?; + })) + .map_err(UsbError::MemoryError)?; Ok(Self { entries }) } @@ -101,8 +103,8 @@ impl EventRingInner { } impl EventRing { - pub fn new(capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity)?; + pub fn new(capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; Ok(Self { inner: IrqSafeSpinlock::new(EventRingInner { diff --git a/driver/usb/xhci/src/ring/transfer.rs b/driver/usb/xhci/src/ring/transfer.rs index 8412d467..34781f98 100644 --- a/driver/usb/xhci/src/ring/transfer.rs +++ b/driver/usb/xhci/src/ring/transfer.rs @@ -11,10 +11,10 @@ use libk_mm::{ }; use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard}; use ygg_driver_usb::{ - communication::UsbInterruptTransfer, pipe::control::ControlTransferSetup, UsbControlTransfer, - UsbDirection, UsbTransferStatus, UsbTransferToken, + communication::UsbInterruptTransfer, error::UsbError, pipe::control::ControlTransferSetup, + UsbControlTransfer, UsbDirection, UsbTransferStatus, UsbTransferToken, }; -use yggdrasil_abi::{define_bitfields, error::Error}; +use yggdrasil_abi::define_bitfields; use crate::ring::LinkTrb; @@ -249,8 +249,8 @@ impl GenericTransferRing for InterruptInTransferRing { } impl InterruptInTransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity)?; + pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; Ok(Self { inner: IrqSafeSpinlock::new(TransferRingInner { @@ -271,11 +271,10 @@ impl InterruptInTransferRing { &self, executor: &E, buffer: &mut PageBox<[u8]>, - ) -> Result { + ) -> Result { // Don't even try to start the transfer if self.shutdown.load(Ordering::Acquire) { - // TODO use UsbError - return Err(Error::NetworkUnreachable); + return Err(UsbError::DeviceDisconnected); } let status = Arc::new(UsbTransferStatus::new()); @@ -307,8 +306,8 @@ impl InterruptInTransferRing { } impl ControlTransferRing { - pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { - let trbs = PageBox::new_zeroed_slice(capacity)?; + pub fn new(slot_id: u8, ep_id: u8, capacity: usize) -> Result { + let trbs = PageBox::new_zeroed_slice(capacity).map_err(UsbError::MemoryError)?; Ok(Self { inner: IrqSafeSpinlock::new(TransferRingInner { @@ -334,11 +333,10 @@ impl ControlTransferRing { executor: &E, setup: ControlTransferSetup, buffer: Option<(PhysicalAddress, usize, UsbDirection)>, - ) -> Result { + ) -> Result { // Don't even try to start the transfer if self.shutdown.load(Ordering::Acquire) { - // TODO use UsbError - return Err(Error::NetworkUnreachable); + return Err(UsbError::DeviceDisconnected); } let mut builder = TransferBuilder::new(self, UsbDirection::In); From b73c1dfdcc45c7893e38524321b6883ec93349e2 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Fri, 1 Mar 2024 15:38:55 +0200 Subject: [PATCH 203/211] usb/core: prettify class drivers --- driver/bus/usb/src/bus.rs | 2 + driver/bus/usb/src/class_driver/mod.rs | 162 +++++++++++++++++++------ 2 files changed, 130 insertions(+), 34 deletions(-) diff --git a/driver/bus/usb/src/bus.rs b/driver/bus/usb/src/bus.rs index c882dd77..9ac24c5e 100644 --- a/driver/bus/usb/src/bus.rs +++ b/driver/bus/usb/src/bus.rs @@ -40,6 +40,8 @@ impl UsbBusManager { } pub async fn bus_handler() { + class_driver::register_default_class_drivers(); + loop { let new_device = QUEUE.pop_front().await; log::info!( diff --git a/driver/bus/usb/src/class_driver/mod.rs b/driver/bus/usb/src/class_driver/mod.rs index 0cce9cef..7aba3634 100644 --- a/driver/bus/usb/src/class_driver/mod.rs +++ b/driver/bus/usb/src/class_driver/mod.rs @@ -1,9 +1,7 @@ -use core::mem::MaybeUninit; - -use alloc::sync::Arc; -use libk_mm::PageBox; +use alloc::{sync::Arc, vec::Vec}; +use futures_util::future::BoxFuture; use libk_thread::runtime; -use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent}; +use libk_util::sync::spin_rwlock::IrqSafeRwLock; use crate::{ device::UsbDeviceAccess, @@ -11,12 +9,29 @@ use crate::{ info::{UsbDeviceClass, UsbDeviceProtocol}, }; -pub async fn spawn_driver(device: Arc) -> Result<(), UsbError> { - // TODO query all configurations? +pub struct UsbClassInfo { + pub class: UsbDeviceClass, + pub subclass: u8, + pub protocol: UsbDeviceProtocol, +} + +pub trait UsbDriver: Send + Sync { + fn name(&self) -> &'static str; + fn run( + self: Arc, + device: Arc, + ) -> BoxFuture<'static, Result<(), UsbError>>; + + fn probe(&self, class: &UsbClassInfo, device: &UsbDeviceAccess) -> bool; +} + +async fn extract_class_info(device: &UsbDeviceAccess) -> Result, UsbError> { + if device.num_configurations != 1 { + return Ok(None); + } let device_info = &device.info; let config_info = device.query_configuration_info(0).await?; - // Select device class/subclass/protocol based on device/interface if config_info.interfaces.len() == 1 { let if_info = &config_info.interfaces[0]; @@ -36,18 +51,78 @@ pub async fn spawn_driver(device: Arc) -> Result<(), UsbError> device_info.device_protocol }; - match (class, subclass, protocol) { - (UsbDeviceClass::Hid, 0x01, _) => { - runtime::spawn(keyboard_driver(device)).map_err(UsbError::SystemError) - } - (_, _, _) => Ok(()), - } + Ok(Some(UsbClassInfo { + class, + subclass, + protocol, + })) } else { - Ok(()) + Ok(None) } } -pub async fn keyboard_driver(device: Arc) -> Result<(), UsbError> { +async fn pick_driver( + device: &UsbDeviceAccess, +) -> Result>, UsbError> { + let Some(class) = extract_class_info(&device).await? else { + return Ok(None); + }; + + for driver in USB_DEVICE_DRIVERS.read().iter() { + if driver.probe(&class, device) { + return Ok(Some(driver.clone())); + } + } + Ok(None) +} + +pub async fn spawn_driver(device: Arc) -> Result<(), UsbError> { + if let Some(driver) = pick_driver(&device).await? { + runtime::spawn(async move { + let name = driver.name(); + match driver.run(device).await { + e @ Err(UsbError::DeviceDisconnected) => { + log::warn!( + "Driver {:?} did not exit cleanly: device disconnected", + name, + ); + + e + } + e => e, + } + }) + .map_err(UsbError::SystemError)?; + } + Ok(()) +} + +pub fn register_driver(driver: Arc) { + // TODO check for duplicates + USB_DEVICE_DRIVERS.write().push(driver); +} + +pub fn register_default_class_drivers() { + register_driver(Arc::new(hid_keyboard::UsbHidKeyboardDriver)); +} + +static USB_DEVICE_DRIVERS: IrqSafeRwLock>> = + IrqSafeRwLock::new(Vec::new()); + +pub mod hid_keyboard { + use core::mem::MaybeUninit; + + use alloc::sync::Arc; + use futures_util::{future::BoxFuture, FutureExt}; + use libk_mm::PageBox; + use yggdrasil_abi::io::{KeyboardKey, KeyboardKeyEvent}; + + use crate::{device::UsbDeviceAccess, error::UsbError, info::UsbDeviceClass}; + + use super::{UsbClassInfo, UsbDriver}; + + pub struct UsbHidKeyboardDriver; + const MODIFIER_MAP: &[KeyboardKey] = &[ KeyboardKey::LControl, KeyboardKey::LShift, @@ -173,31 +248,50 @@ pub async fn keyboard_driver(device: Arc) -> Result<(), UsbErro } } - // TODO not sure whether to use boot protocol (easy) or GetReport - let config = device.select_configuration(|_| true).await?.unwrap(); - assert_eq!(config.endpoints.len(), 1); + impl UsbDriver for UsbHidKeyboardDriver { + fn run( + self: Arc, + device: Arc, + ) -> BoxFuture<'static, Result<(), UsbError>> { + async move { + // TODO not sure whether to use boot protocol (easy) or GetReport + let config = device.select_configuration(|_| true).await?.unwrap(); + assert_eq!(config.endpoints.len(), 1); - let pipe = device.open_interrupt_in_pipe(1).await?; + let pipe = device.open_interrupt_in_pipe(1).await?; - let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?; - let mut state = KeyboardState::new(); - let mut events = [MaybeUninit::uninit(); 16]; + let mut buffer = PageBox::new_slice(0, 8).map_err(UsbError::MemoryError)?; + let mut state = KeyboardState::new(); + let mut events = [MaybeUninit::uninit(); 16]; - loop { - let mut event_count = 0; + loop { + let mut event_count = 0; - let data = pipe.read(&mut buffer).await?; + let data = pipe.read(&mut buffer).await?; - event_count += state.retain_modifiers(data[0], &mut events); - event_count += state.press_modifiers(data[0], &mut events[event_count..]); - event_count += state.retain(&data[2..], &mut events[event_count..]); - event_count += state.press(&data[2..], &mut events[event_count..]); + event_count += state.retain_modifiers(data[0], &mut events); + event_count += state.press_modifiers(data[0], &mut events[event_count..]); + event_count += state.retain(&data[2..], &mut events[event_count..]); + event_count += state.press(&data[2..], &mut events[event_count..]); - let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) }; + let events = + unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) }; - for &event in events { - log::debug!("Generic Keyboard: {:?}", event); - ygg_driver_input::send_event(event); + for &event in events { + log::debug!("Generic Keyboard: {:?}", event); + ygg_driver_input::send_event(event); + } + } + } + .boxed() + } + + fn name(&self) -> &'static str { + "USB HID Keyboard" + } + + fn probe(&self, class: &UsbClassInfo, _device: &UsbDeviceAccess) -> bool { + class.class == UsbDeviceClass::Hid && class.subclass == 0x01 } } } From 7b28731a2fb77c5d405392fbeb7f51e375384bc1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 3 Mar 2024 02:09:02 +0200 Subject: [PATCH 204/211] vfs: PTY node + /dev/tty symlink to current tty --- driver/fs/kernel-fs/src/devfs.rs | 5 ++- lib/vfs/src/file/mod.rs | 43 +++++++++++++++---------- lib/vfs/src/ioctx.rs | 35 ++++++++++----------- lib/vfs/src/node/impls.rs | 35 +++++++++++++++++++++ lib/vfs/src/node/mod.rs | 54 +++++++++++++++++++++++++++++++- lib/vfs/src/node/ops.rs | 1 + lib/vfs/src/pty.rs | 3 ++ src/init.rs | 18 ++++++++++- src/syscall/mod.rs | 10 +++++- 9 files changed, 165 insertions(+), 39 deletions(-) diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index 63c8e814..e5f77b36 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -3,7 +3,10 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::{format, string::String}; use libk_util::OneTimeInit; -use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; +use vfs::{ + impls::{mdir, FnSymlink, MemoryDirectory}, + CharDevice, Node, NodeFlags, NodeRef, +}; use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 296f6b81..a9edadff 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -25,7 +25,7 @@ use crate::{ node::NodeRef, socket::{ConnectionSocketWrapper, ListenerSocketWrapper, PacketSocketWrapper}, traits::{Read, Seek, Write}, - ConnectionSocket, FdPoll, FileReadiness, ListenerSocket, PacketSocket, PseudoTerminal, + ConnectionSocket, FdPoll, FileReadiness, ListenerSocket, Node, PacketSocket, PseudoTerminal, PseudoTerminalMaster, PseudoTerminalSlave, SharedMemory, Socket, TimerFile, }; @@ -73,8 +73,8 @@ pub enum File { Timer(TimerFile), Channel(ChannelDescriptor), SharedMemory(Arc), - PtySlave(PseudoTerminalSlave), - PtyMaster(PseudoTerminalMaster), + PtySlave(Arc, NodeRef), + PtyMaster(Arc, NodeRef), } /// Contains a per-process fd -> FileRef map @@ -115,9 +115,12 @@ impl File { size: TerminalSize, ) -> Result<(Arc, Arc), Error> { let (master, slave) = PseudoTerminal::new(config, size)?; + let master = Arc::new(master); + let slave = Arc::new(slave); + let (master_node, slave_node) = Node::pseudo_terminal_nodes(master.clone(), slave.clone()); Ok(( - Arc::new(Self::PtyMaster(master)), - Arc::new(Self::PtySlave(slave)), + Arc::new(Self::PtyMaster(master, master_node)), + Arc::new(Self::PtySlave(slave, slave_node)), )) } @@ -222,8 +225,12 @@ impl File { Self::Block(_) => todo!(), Self::Regular(file) => Ok(Arc::new(Self::Regular(file.clone()))), Self::SharedMemory(shm) => Ok(Arc::new(Self::SharedMemory(shm.clone()))), - Self::PtySlave(pt) => Ok(Arc::new(Self::PtySlave(pt.clone()))), - Self::PtyMaster(pt) => Ok(Arc::new(Self::PtyMaster(pt.clone()))), + Self::PtySlave(pt, pt_node) => { + Ok(Arc::new(Self::PtySlave(pt.clone(), pt_node.clone()))) + } + Self::PtyMaster(pt, pt_node) => { + Ok(Arc::new(Self::PtyMaster(pt.clone(), pt_node.clone()))) + } _ => { log::info!("Invalid file send(): {:?}", self); Err(Error::InvalidOperation) @@ -246,6 +253,8 @@ impl File { Self::Regular(file) => Some(&file.node), Self::Block(file) => Some(&file.node), Self::Char(file) => Some(&file.node), + Self::PtyMaster(_, node) => Some(&node), + Self::PtySlave(_, node) => Some(&node), _ => None, } } @@ -256,8 +265,8 @@ impl File { Self::Char(f) => f.device.0.poll_read(cx), Self::Channel(ch) => ch.poll_read(cx), Self::Poll(ch) => ch.poll_read(cx), - Self::PtyMaster(f) => f.poll_read(cx), - Self::PtySlave(f) => f.poll_read(cx), + Self::PtyMaster(f, _) => f.poll_read(cx), + Self::PtySlave(f, _) => f.poll_read(cx), Self::PacketSocket(sock) => sock.poll_read(cx), Self::StreamSocket(sock) => sock.poll_read(cx), Self::ListenerSocket(sock) => sock.poll_read(cx), @@ -272,8 +281,8 @@ impl File { match self { Self::Char(f) => f.device.0.device_request(req), Self::Block(f) => f.device.0.device_request(req), - Self::PtySlave(f) => f.device_request(req), - Self::PtyMaster(f) => f.device_request(req), + Self::PtySlave(f, _) => f.device_request(req), + Self::PtyMaster(f, _) => f.device_request(req), _ => Err(Error::InvalidOperation), } } @@ -381,8 +390,8 @@ impl Read for File { Self::Block(file) => file.read(buf), Self::Char(file) => file.read(buf), Self::AnonymousPipe(pipe) => pipe.read(buf), - Self::PtySlave(pt) => pt.read(buf), - Self::PtyMaster(pt) => pt.read(buf), + Self::PtySlave(pt, _) => pt.read(buf), + Self::PtyMaster(pt, _) => pt.read(buf), // TODO maybe allow reading trigger count? Self::Timer(_) => Err(Error::InvalidOperation), // TODO maybe allow reading FDs from poll channels as if they were regular streams? @@ -406,8 +415,8 @@ impl Write for File { Self::Block(file) => file.write(buf), Self::Char(file) => file.write(buf), Self::AnonymousPipe(pipe) => pipe.write(buf), - Self::PtySlave(pt) => pt.write(buf), - Self::PtyMaster(pt) => pt.write(buf), + Self::PtySlave(pt, _) => pt.write(buf), + Self::PtyMaster(pt, _) => pt.write(buf), Self::Timer(timer) => timer.write(buf), // TODO maybe allow adding FDs to poll channels this way Self::Poll(_) => Err(Error::InvalidOperation), @@ -468,8 +477,8 @@ impl fmt::Debug for File { Self::Poll(_) => f.debug_struct("Poll").finish_non_exhaustive(), Self::Channel(_) => f.debug_struct("Channel").finish_non_exhaustive(), Self::SharedMemory(_) => f.debug_struct("SharedMemory").finish_non_exhaustive(), - Self::PtySlave(_) => f.debug_struct("PtySlave").finish_non_exhaustive(), - Self::PtyMaster(_) => f.debug_struct("PtyMaster").finish_non_exhaustive(), + Self::PtySlave(_, _) => f.debug_struct("PtySlave").finish_non_exhaustive(), + Self::PtyMaster(_, _) => f.debug_struct("PtyMaster").finish_non_exhaustive(), Self::PacketSocket(sock) => f .debug_struct("PacketSocket") .field("local", &sock.local_address()) diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index 56893467..110d3c7c 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -323,23 +323,15 @@ impl IoContext { self._find(at, path, follow_links, follow_mount) } - fn _resolve_link(&self, at: &NodeRef, link: &SymlinkData) -> Result { - self.check_access(Action::Read, at)?; - - // If the filesystem itself implements direct target resolution, use that - match link.imp.target(at) { - Err(Error::NotImplemented) => {} - other => return other, + fn _resolve_link(&self, at: &NodeRef) -> Result { + let token = self.check_access(Action::Read, at)?; + // let _path = link.imp.read_to_string()?; + match at.read_symlink_node(token) { + Ok(node) => return Ok(node), + // Need to read the link data and resolve it manually + Err(Error::NotImplemented) => todo!(), + Err(e) => return Err(e), } - - // If that call wasn't implemented, check if the target was cached - if let Some((_, target)) = link.target.lock().as_ref() { - return Ok(target.clone()); - } - - // If no cache is present, read the link's data and resolve the path - let _path = link.imp.read_to_string()?; - todo!() } fn _resolve( @@ -354,9 +346,16 @@ impl IoContext { continue; } - if follow_links && let Ok(link) = at.as_symlink() { + if follow_links { // Resolve the link - at = self._resolve_link(&at, link)?; + match self._resolve_link(&at) { + Ok(node) => { + at = node; + } + // Not a link, not an error + Err(Error::InvalidFile) => break Ok(at), + Err(e) => return Err(e), + } continue; } diff --git a/lib/vfs/src/node/impls.rs b/lib/vfs/src/node/impls.rs index a9e6b52b..a55410c0 100644 --- a/lib/vfs/src/node/impls.rs +++ b/lib/vfs/src/node/impls.rs @@ -34,6 +34,8 @@ pub trait ValueWriteFn = Fn(T) -> Result<(), Error> + Send + Sync; pub trait ReadFn = Fn(u64, &mut [u8]) -> Result + Send + Sync; /// Closure interface for writing bytes pub trait WriteFn = Fn(u64, &[u8]) -> Result + Send + Sync; +/// Closure interface for reading links +pub trait ReadLinkFn = Fn() -> Result + Send + Sync; impl> SliceRead for T { fn read_slice(&self, pos: usize, buf: &mut [u8]) -> usize { @@ -94,6 +96,13 @@ pub struct MemoryDirectory; pub struct FixedSymlink { target: NodeRef, } +/// In-memory functional symlink +pub struct FnSymlink +where + F: ReadLinkFn + 'static, +{ + read: F, +} impl ReadOnlyFnValueNode where @@ -373,6 +382,32 @@ impl SymlinkImpl for FixedSymlink { } } +// In-memory functional symlink + +impl FnSymlink +where + F: ReadLinkFn + 'static, +{ + /// Creates a new [FnSymlink] node + pub fn new(read: F) -> NodeRef { + let data = Self { read }; + Node::symlink( + data, + NodeFlags::IN_MEMORY_SIZE | NodeFlags::IN_MEMORY_PROPS | NodeFlags::NO_LINK_CACHE, + ) + } +} + +impl CommonImpl for FnSymlink where F: ReadLinkFn + 'static {} +impl SymlinkImpl for FnSymlink +where + F: ReadLinkFn + 'static, +{ + fn target(&self, _node: &NodeRef) -> Result { + (self.read)() + } +} + /// Creates a read-only value node with given `value` pub fn const_value_node(value: T) -> NodeRef where diff --git a/lib/vfs/src/node/mod.rs b/lib/vfs/src/node/mod.rs index ea4a8093..0b619638 100644 --- a/lib/vfs/src/node/mod.rs +++ b/lib/vfs/src/node/mod.rs @@ -20,7 +20,10 @@ mod tree; pub use access::AccessToken; pub use traits::{CommonImpl, DirectoryImpl, RegularImpl, SymlinkImpl}; -use crate::device::{BlockDeviceWrapper, CharDevice, CharDeviceWrapper}; +use crate::{ + device::{BlockDeviceWrapper, CharDevice, CharDeviceWrapper}, + PseudoTerminalMaster, PseudoTerminalSlave, +}; /// Wrapper type for a [Node] shared reference pub type NodeRef = Arc; @@ -32,6 +35,8 @@ bitflags! { const IN_MEMORY_PROPS: bit 0; #[doc = "Node's size only exists within the VFS cache"] const IN_MEMORY_SIZE: bit 1; + #[doc = "Don't cache symlink targets"] + const NO_LINK_CACHE: bit 2; } } @@ -68,6 +73,10 @@ enum NodeImpl { Block(BlockDeviceWrapper), Char(CharDeviceWrapper), Symlink(SymlinkData), + + // These map transparently to other types of nodes + PseudoTerminalSlave(Arc), + PseudoTerminalMaster(Arc), } /// Metadata of the node @@ -125,6 +134,24 @@ impl Node { }) } + pub(crate) fn pseudo_terminal_nodes( + master: Arc, + slave: Arc, + ) -> (NodeRef, NodeRef) { + let master = Self::new( + NodeImpl::PseudoTerminalMaster(master), + NodeFlags::IN_MEMORY_PROPS | NodeFlags::IN_MEMORY_SIZE, + Metadata::default_file(), + ); + let slave = Self::new( + NodeImpl::PseudoTerminalSlave(slave), + NodeFlags::IN_MEMORY_PROPS | NodeFlags::IN_MEMORY_SIZE, + Metadata::default_file(), + ); + + (master, slave) + } + /// Creates a new directory node with given [DirectoryImpl] pub fn directory(data: T, flags: NodeFlags) -> NodeRef { let data = NodeImpl::Directory(DirectoryData { @@ -182,6 +209,7 @@ impl Node { NodeImpl::Block(w) => w.as_any(), NodeImpl::Char(w) => w.as_any(), NodeImpl::Symlink(w) => w.imp.as_any(), + NodeImpl::PseudoTerminalSlave(_) | NodeImpl::PseudoTerminalMaster(_) => todo!(), } } @@ -193,6 +221,7 @@ impl Node { NodeImpl::Block(w) => w, NodeImpl::Char(w) => w, NodeImpl::Symlink(w) => w.imp.as_ref(), + NodeImpl::PseudoTerminalSlave(_) | NodeImpl::PseudoTerminalMaster(_) => todo!(), } } @@ -209,6 +238,7 @@ impl Node { NodeImpl::Block(_) => FileType::Block, NodeImpl::Char(_) => FileType::Char, NodeImpl::Symlink(_) => FileType::Symlink, + NodeImpl::PseudoTerminalSlave(_) | NodeImpl::PseudoTerminalMaster(_) => FileType::Char, } } @@ -277,6 +307,22 @@ impl Node { _ => Err(Error::InvalidFile), } } + + pub(crate) fn read_symlink_node(self: &NodeRef, _token: AccessToken) -> Result { + let symlink = self.as_symlink()?; + let mut cache = symlink.target.lock(); + + // If caching is not disabled and the target is cached, return it + if !self.flags.contains(NodeFlags::NO_LINK_CACHE) + && let Some((_, cached)) = cache.as_ref() + { + return Ok(cached.clone()); + } + + let target = symlink.imp.target(self)?; + + Ok(cache.insert((String::new(), target)).1.clone()) + } } impl fmt::Debug for Node { @@ -287,6 +333,12 @@ impl fmt::Debug for Node { NodeImpl::Char(_) => f.debug_struct("CharNode").finish_non_exhaustive(), NodeImpl::Block(_) => f.debug_struct("BlockNode").finish_non_exhaustive(), NodeImpl::Symlink(_) => f.debug_struct("SymlinkNode").finish_non_exhaustive(), + NodeImpl::PseudoTerminalSlave(_) => f + .debug_struct("PseudoTerminalSlave") + .finish_non_exhaustive(), + NodeImpl::PseudoTerminalMaster(_) => f + .debug_struct("PseudoTerminalMaster") + .finish_non_exhaustive(), } } } diff --git a/lib/vfs/src/node/ops.rs b/lib/vfs/src/node/ops.rs index a40308a3..bfb88222 100644 --- a/lib/vfs/src/node/ops.rs +++ b/lib/vfs/src/node/ops.rs @@ -39,6 +39,7 @@ impl Node { // TODO: maybe merge open_directory and open? NodeImpl::Directory(_) => Err(Error::IsADirectory), NodeImpl::Symlink(_) => todo!(), + NodeImpl::PseudoTerminalSlave(_) | NodeImpl::PseudoTerminalMaster(_) => todo!(), } } diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 97bf1a42..01aba7b3 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -22,6 +22,8 @@ use yggdrasil_abi::{ process::Signal, }; +use crate::CharDevice; + const CAPACITY: usize = 8192; struct PtySlaveToMasterHalf { @@ -305,6 +307,7 @@ impl PseudoTerminal { fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { match req { DeviceRequest::SetTerminalGroup(group_id) => { + log::info!("SetTerminalGroup {}", group_id); self.master_to_slave .signal_pgroup .write() diff --git a/src/init.rs b/src/init.rs index 74c744e2..e3660af2 100644 --- a/src/init.rs +++ b/src/init.rs @@ -4,12 +4,14 @@ use abi::error::Error; use alloc::borrow::ToOwned; use kernel_fs::devfs; use libk::runtime; +use libk_thread::{process::Process, thread::Thread}; use memfs::MemoryFilesystem; -use vfs::{IoContext, NodeRef}; +use vfs::{impls::FnSymlink, IoContext, NodeRef}; use crate::{ fs::{FileBlockAllocator, INITRD_DATA}, proc::{self, random}, + task::process::ProcessManagerImpl, }; fn setup_root() -> Result { @@ -29,6 +31,20 @@ pub fn kinit() -> Result<(), Error> { runtime::spawn(ygg_driver_usb::bus::bus_handler())?; + devfs::root().add_child( + "tty", + FnSymlink::new(|| { + let thread = Thread::current(); + let process = thread.process::(); + + if let Some(tty) = process.session_terminal() { + Ok(tty) + } else { + Err(Error::InvalidFile) + } + }), + )?; + ygg_driver_net_loopback::init(); ygg_driver_net_core::start_network_tasks()?; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index fc8a3e51..f576e67d 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -553,7 +553,12 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result } } &SpawnOption::SetProcessGroup(pgroup) => { - child_process.set_group_id(pgroup.into()); + let pgroup = if pgroup == 0 { + child_process.id() + } else { + pgroup.into() + }; + child_process.set_group_id(pgroup); } _ => (), } @@ -572,6 +577,9 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); file.device_request(&mut req)?; + if let Some(node) = file.node() { + child_process.set_session_terminal(node.clone()); + } // node.device_request(&mut req)?; } From 4312a097da4bb1a839a23d34a5aefec5815b2058 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Sun, 3 Mar 2024 03:29:21 +0200 Subject: [PATCH 205/211] vfs: fix vfs warnings --- driver/fs/kernel-fs/src/devfs.rs | 5 +---- lib/vfs/src/ioctx.rs | 2 +- lib/vfs/src/pty.rs | 2 -- src/init.rs | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/driver/fs/kernel-fs/src/devfs.rs b/driver/fs/kernel-fs/src/devfs.rs index e5f77b36..63c8e814 100644 --- a/driver/fs/kernel-fs/src/devfs.rs +++ b/driver/fs/kernel-fs/src/devfs.rs @@ -3,10 +3,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use alloc::{format, string::String}; use libk_util::OneTimeInit; -use vfs::{ - impls::{mdir, FnSymlink, MemoryDirectory}, - CharDevice, Node, NodeFlags, NodeRef, -}; +use vfs::{impls::MemoryDirectory, CharDevice, Node, NodeFlags, NodeRef}; use ygg_driver_block::BlockDevice; use yggdrasil_abi::error::Error; diff --git a/lib/vfs/src/ioctx.rs b/lib/vfs/src/ioctx.rs index 110d3c7c..96878ebf 100644 --- a/lib/vfs/src/ioctx.rs +++ b/lib/vfs/src/ioctx.rs @@ -7,7 +7,7 @@ use yggdrasil_abi::{ }; use crate::{ - node::{AccessToken, CreateInfo, SymlinkData}, + node::{AccessToken, CreateInfo}, File, FileRef, NodeRef, }; diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 01aba7b3..8f3dd662 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -22,8 +22,6 @@ use yggdrasil_abi::{ process::Signal, }; -use crate::CharDevice; - const CAPACITY: usize = 8192; struct PtySlaveToMasterHalf { diff --git a/src/init.rs b/src/init.rs index e3660af2..22190535 100644 --- a/src/init.rs +++ b/src/init.rs @@ -4,7 +4,7 @@ use abi::error::Error; use alloc::borrow::ToOwned; use kernel_fs::devfs; use libk::runtime; -use libk_thread::{process::Process, thread::Thread}; +use libk_thread::thread::Thread; use memfs::MemoryFilesystem; use vfs::{impls::FnSymlink, IoContext, NodeRef}; From 6329e1f102db9bf3ae83749116bb07c350b5d34f Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 4 Mar 2024 16:31:24 +0200 Subject: [PATCH 206/211] actions: setup build test actions --- .gitea/workflows/kernel.yaml | 51 +++++++++++++++++++++++++++++++++ etc/aarch64-unknown-qemu.json | 18 ++++++++++++ etc/aarch64-unknown-qemu.ld | 53 +++++++++++++++++++++++++++++++++++ etc/x86_64-unknown-none.json | 26 +++++++++++++++++ etc/x86_64-unknown-none.ld | 51 +++++++++++++++++++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 .gitea/workflows/kernel.yaml create mode 100644 etc/aarch64-unknown-qemu.json create mode 100644 etc/aarch64-unknown-qemu.ld create mode 100644 etc/x86_64-unknown-none.json create mode 100644 etc/x86_64-unknown-none.ld diff --git a/.gitea/workflows/kernel.yaml b/.gitea/workflows/kernel.yaml new file mode 100644 index 00000000..cd391311 --- /dev/null +++ b/.gitea/workflows/kernel.yaml @@ -0,0 +1,51 @@ +name: Kernel tests +run_name: Kernel tests +on: [pull_request] + +jobs: + Test-x86_64-Build: + runs-on: ubuntu-latest + steps: + - name: Checkout kernel sources + uses: actions/checkout@v3 + - name: Install build dependencies + run: | + apt update && apt install -y nasm gcc + - name: Install nightly Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain nightly + source "$HOME/.cargo/env" + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Update dependencies + run: | + source "$HOME/.cargo/env" + cd ${{ gitea.workspace }} + cargo update yggdrasil-abi elf + - name: Build x86-64 + run: | + source "$HOME/.cargo/env" + cd ${{ gitea.workspace }} + cargo build -Z build-std=core,alloc,compiler_builtins --target=etc/x86_64-unknown-none.json + Test-aarch64-Build: + runs-on: ubuntu-latest + steps: + - name: Checkout kernel sources + uses: actions/checkout@v3 + - name: Install build dependencies + run: | + apt update && apt install -y nasm gcc + - name: Install nightly Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain nightly + source "$HOME/.cargo/env" + rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu + - name: Update dependencies + run: | + source "$HOME/.cargo/env" + cd ${{ gitea.workspace }} + cargo update yggdrasil-abi elf + - name: Build aarch64 + run: | + source "$HOME/.cargo/env" + cd ${{ gitea.workspace }} + cargo build -Z build-std=core,alloc,compiler_builtins --target=etc/aarch64-unknown-qemu.json diff --git a/etc/aarch64-unknown-qemu.json b/etc/aarch64-unknown-qemu.json new file mode 100644 index 00000000..fd88dbf3 --- /dev/null +++ b/etc/aarch64-unknown-qemu.json @@ -0,0 +1,18 @@ +{ + "arch": "aarch64", + "data-layout": "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128", + "disable-redzone": true, + "features": "+v8a,+strict-align,+neon,+fp-armv8", + "is-builtin": false, + "linker": "rust-lld", + "linker-flavor": "ld.lld", + "llvm-target": "aarch64-unknown-none", + "max-atomic-width": 128, + "panic-strategy": "abort", + "relocation-model": "static", + "target-pointer-width": "64", + "eh-frame-header": false, + "post-link-args": { + "ld.lld": ["-Tetc/aarch64-unknown-qemu.ld"] + } +} diff --git a/etc/aarch64-unknown-qemu.ld b/etc/aarch64-unknown-qemu.ld new file mode 100644 index 00000000..f1fc2e63 --- /dev/null +++ b/etc/aarch64-unknown-qemu.ld @@ -0,0 +1,53 @@ +ENTRY(__aarch64_entry); + +KERNEL_PHYS_BASE = 0x40080000; +KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000; + +SECTIONS { + . = KERNEL_PHYS_BASE; + PROVIDE(__kernel_phys_start = .); + + .text.entry : { + *(.text.entry) + } + + . = ALIGN(16); + . = . + KERNEL_VIRT_OFFSET; + + .text : AT(. - KERNEL_VIRT_OFFSET) { + KEEP(*(.text.vectors)); + *(.text*) + } + + . = ALIGN(4K); + .rodata : AT(. - KERNEL_VIRT_OFFSET) { + *(.eh_frame*) + . = ALIGN(16); + PROVIDE(__dt_probes_start = .); + KEEP(*(.dt_probes)); + PROVIDE(__dt_probes_end = .); + *(.rodata*) + } + + . = ALIGN(4K); + .data.tables : AT (. - KERNEL_VIRT_OFFSET) { + KEEP(*(.data.tables)) + } + + . = ALIGN(4K); + .data : AT(. - KERNEL_VIRT_OFFSET) { + *(.data*) + /* *(.got*) */ + } + + . = ALIGN(4K); + PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); + .bss : AT(. - KERNEL_VIRT_OFFSET) { + *(COMMON) + *(.bss*) + } + . = ALIGN(4K); + PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); + + PROVIDE(__kernel_size = . - KERNEL_VIRT_OFFSET - KERNEL_PHYS_BASE); +}; diff --git a/etc/x86_64-unknown-none.json b/etc/x86_64-unknown-none.json new file mode 100644 index 00000000..eed3a3e1 --- /dev/null +++ b/etc/x86_64-unknown-none.json @@ -0,0 +1,26 @@ +{ + "is-builtin": false, + "arch": "x86_64", + "cpu": "x86-64", + "os": "none", + "llvm-target": "x86_64-unknown-linux-gnu", + "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", + "max-atomic-width": 64, + "target-pointer-width": "64", + + "disable-redzone": true, + "executables": true, + "panic-strategy": "abort", + "features": "-avx,-sse,+soft-float", + + "has-thread-local": false, + + "linker": "rust-lld", + "linker-flavor": "ld.lld", + + "pre-link-args": { + "ld.lld": [ + "-Tetc/x86_64-unknown-none.ld" + ] + } +} diff --git a/etc/x86_64-unknown-none.ld b/etc/x86_64-unknown-none.ld new file mode 100644 index 00000000..1be78b30 --- /dev/null +++ b/etc/x86_64-unknown-none.ld @@ -0,0 +1,51 @@ +ENTRY(__x86_64_entry); + +KERNEL_PHYS_BASE = 0x200000; +KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000; + +SECTIONS { + . = KERNEL_PHYS_BASE; + PROVIDE(__kernel_phys_start = .); + PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); + + .text.entry : { + *(.multiboot) + *(.text.entry) + } + + . = ALIGN(16); + . = . + KERNEL_VIRT_OFFSET; + + .text : AT(. - KERNEL_VIRT_OFFSET) { + *(.text*) + } + + . = ALIGN(4K); + .rodata : AT(. - KERNEL_VIRT_OFFSET) { + *(.eh_frame*) + *(.rodata*) + } + + . = ALIGN(4K); + .data.tables : AT (. - KERNEL_VIRT_OFFSET) { + KEEP(*(.data.tables)) + } + + .data : AT(. - KERNEL_VIRT_OFFSET) { + KEEP(*(.data.yboot)) + *(.data*) + *(.got*) + } + + . = ALIGN(4K); + PROVIDE(__bss_start_phys = . - KERNEL_VIRT_OFFSET); + .bss : AT(. - KERNEL_VIRT_OFFSET) { + *(COMMON) + *(.bss*) + } + . = ALIGN(4K); + PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); + + PROVIDE(__kernel_end = .); + PROVIDE(__kernel_size = . - KERNEL_VIRT_OFFSET - KERNEL_PHYS_BASE); +}; From 4833b5c9fc96de2601d4a10a39aff11c81236434 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 4 Mar 2024 17:39:18 +0200 Subject: [PATCH 207/211] fs/memfs: fix tar parsing failing on unknown node types --- driver/fs/memfs/src/tar.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/driver/fs/memfs/src/tar.rs b/driver/fs/memfs/src/tar.rs index 3408f2e5..abdc4927 100644 --- a/driver/fs/memfs/src/tar.rs +++ b/driver/fs/memfs/src/tar.rs @@ -67,22 +67,25 @@ impl<'a> Iterator for TarIterator<'a> { continue; } + let size = usize::from(&hdr.size); + let size_aligned = (size + 511) & !511; + let (data, size_aligned) = match hdr.type_ { 0 | b'0' => { - let size = usize::from(&hdr.size); - if self.offset + 512 + size > self.data.len() { return Some(Err(Error::InvalidArgument)); } let data = &self.data[self.offset + 512..self.offset + 512 + size]; - let size_aligned = (size + 511) & !511; (Some(data), size_aligned) } // Directory b'5' => (None, 0), - _ => todo!("Unknown node kind: {}", hdr.type_), + _ => { + self.offset += size_aligned + 512; + continue; + } }; self.offset += size_aligned + 512; @@ -95,7 +98,7 @@ impl From<&OctalField> for usize { fn from(value: &OctalField) -> Self { let mut acc = 0; for i in 0..N { - if value.data[i] == 0 { + if !(b'0'..b'8').contains(&value.data[i]) { break; } acc <<= 3; From ca4e50d464e064fcbf3bb6b54026fcec66ece69e Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 5 Mar 2024 16:56:25 +0200 Subject: [PATCH 208/211] test: update hosted tests --- arch/Cargo.toml | 7 +- arch/hosted/Cargo.toml | 9 ++ arch/hosted/src/lib.rs | 176 ++++++++++++++++++++++++++++++++++++ arch/src/lib.rs | 4 +- lib/hosted-tests/Cargo.toml | 10 -- lib/hosted-tests/src/lib.rs | 27 ------ lib/vfs/Cargo.toml | 3 - lib/vfs/src/file/mod.rs | 80 +++------------- lib/vfs/src/lib.rs | 4 +- 9 files changed, 206 insertions(+), 114 deletions(-) create mode 100644 arch/hosted/Cargo.toml create mode 100644 arch/hosted/src/lib.rs delete mode 100644 lib/hosted-tests/Cargo.toml delete mode 100644 lib/hosted-tests/src/lib.rs diff --git a/arch/Cargo.toml b/arch/Cargo.toml index 0ac2a342..78a41e21 100644 --- a/arch/Cargo.toml +++ b/arch/Cargo.toml @@ -5,12 +5,15 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[target.'cfg(target_arch = "x86_64")'.dependencies] +[target.'cfg(all(target_os = "none", target_arch = "x86_64"))'.dependencies] kernel-arch-x86_64 = { path = "x86_64" } -[target.'cfg(target_arch = "aarch64")'.dependencies] +[target.'cfg(all(target_os = "none", target_arch = "aarch64"))'.dependencies] kernel-arch-aarch64 = { path = "aarch64" } +[target.'cfg(not(target_os = "none"))'.dependencies] +kernel-arch-hosted = { path = "hosted" } + [dependencies] kernel-arch-interface = { path = "interface" } diff --git a/arch/hosted/Cargo.toml b/arch/hosted/Cargo.toml new file mode 100644 index 00000000..c9db9191 --- /dev/null +++ b/arch/hosted/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "kernel-arch-hosted" +version = "0.1.0" +edition = "2021" + +[dependencies] +kernel-arch-interface = { path = "../interface" } +yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +libk-mm-interface = { path = "../../libk/libk-mm/interface" } diff --git a/arch/hosted/src/lib.rs b/arch/hosted/src/lib.rs new file mode 100644 index 00000000..0a333a5f --- /dev/null +++ b/arch/hosted/src/lib.rs @@ -0,0 +1,176 @@ +#![feature(never_type)] +use std::{ + marker::PhantomData, + sync::atomic::{AtomicBool, Ordering}, +}; + +use kernel_arch_interface::{ + cpu::IpiQueue, + mem::{ + DeviceMemoryAttributes, KernelTableManager, PhysicalMemoryAllocator, RawDeviceMemoryMapping, + }, + task::{Scheduler, TaskContext}, + Architecture, +}; +use libk_mm_interface::{ + address::PhysicalAddress, + process::ProcessAddressSpaceManager, + table::{MapAttributes, TableAllocator}, +}; +use yggdrasil_abi::{error::Error, process::Signal}; + +pub struct ArchitectureImpl; + +#[derive(Debug)] +pub struct KernelTableManagerImpl; + +pub struct ProcessAddressSpaceImpl(!, PhantomData); + +pub struct TaskContextImpl( + !, + PhantomData<(K, PA)>, +); + +static DUMMY_INTERRUPT_MASK: AtomicBool = AtomicBool::new(true); + +impl Architecture for ArchitectureImpl { + type PerCpuData = (); + + fn local_cpu() -> *mut Self::PerCpuData { + unimplemented!() + } + + unsafe fn set_local_cpu(_cpu: *mut Self::PerCpuData) { + unimplemented!() + } + + unsafe fn init_local_cpu(_id: Option, _data: Self::PerCpuData) { + unimplemented!() + } + + unsafe fn init_ipi_queues(_queues: Vec>) { + unimplemented!() + } + + fn idle_task() -> extern "C" fn(usize) -> ! { + unimplemented!() + } + + fn cpu_count() -> usize { + unimplemented!() + } + + fn cpu_index() -> u32 { + unimplemented!() + } + + unsafe fn set_interrupt_mask(mask: bool) -> bool { + DUMMY_INTERRUPT_MASK.swap(mask, Ordering::Acquire) + } + + fn interrupt_mask() -> bool { + unimplemented!() + } + + fn wait_for_interrupt() { + unimplemented!() + } +} + +impl KernelTableManager for KernelTableManagerImpl { + fn virtualize(_phys: u64) -> usize { + unimplemented!() + } + + fn physicalize(_virt: usize) -> u64 { + unimplemented!() + } + + unsafe fn map_device_pages( + _base: u64, + _count: usize, + _attrs: DeviceMemoryAttributes, + ) -> Result, Error> { + unimplemented!() + } + + unsafe fn unmap_device_pages(_mapping: &RawDeviceMemoryMapping) { + unimplemented!() + } +} + +impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { + const LOWER_LIMIT_PFN: usize = 16; + const UPPER_LIMIT_PFN: usize = 1024; + + fn new() -> Result { + unimplemented!() + } + + unsafe fn clear(&mut self) { + unimplemented!() + } + + unsafe fn map_page( + &mut self, + _address: usize, + _physical: PhysicalAddress, + _flags: MapAttributes, + ) -> Result<(), Error> { + unimplemented!() + } + + unsafe fn unmap_page(&mut self, _address: usize) -> Result { + unimplemented!() + } + + fn translate(&self, _address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> { + unimplemented!() + } + + fn as_address_with_asid(&self) -> u64 { + unimplemented!() + } +} + +impl TaskContext + for TaskContextImpl +{ + const USER_STACK_EXTRA_ALIGN: usize = 0; + const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; + + unsafe fn enter(&self) -> ! { + unimplemented!() + } + + unsafe fn switch(&self, _from: &Self) { + unimplemented!() + } + + unsafe fn switch_and_drop(&self, _thread: *const ()) { + unimplemented!() + } + + fn user( + _entry: usize, + _arg: usize, + _cr3: u64, + _user_stack_sp: usize, + _tls_address: usize, + ) -> Result { + unimplemented!() + } + + fn kernel(_entry: extern "C" fn(usize) -> !, _arg: usize) -> Result { + unimplemented!() + } + + fn kernel_closure ! + Send + 'static>(_f: F) -> Result { + unimplemented!() + } +} + +#[no_mangle] +extern "Rust" fn __signal_process_group(_group_id: u32, _signal: Signal) { + unimplemented!() +} diff --git a/arch/src/lib.rs b/arch/src/lib.rs index a1f0bffd..14e23235 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -20,7 +20,9 @@ macro_rules! absolute_address { } cfg_if! { - if #[cfg(target_arch = "aarch64")] { + if #[cfg(any(test, not(target_os = "none")))] { + extern crate kernel_arch_hosted as imp; + } else if #[cfg(target_arch = "aarch64")] { extern crate kernel_arch_aarch64 as imp; } else if #[cfg(target_arch = "x86_64")] { extern crate kernel_arch_x86_64 as imp; diff --git a/lib/hosted-tests/Cargo.toml b/lib/hosted-tests/Cargo.toml deleted file mode 100644 index 9fbf978d..00000000 --- a/lib/hosted-tests/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "hosted-tests" -version = "0.1.0" -edition = "2021" -authors = ["Mark Poliakov "] - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/lib/hosted-tests/src/lib.rs b/lib/hosted-tests/src/lib.rs deleted file mode 100644 index f85277bb..00000000 --- a/lib/hosted-tests/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -#![no_std] - -extern crate alloc; - -use alloc::sync::Arc; -use yggdrasil_abi::error::Error; - -#[no_mangle] -fn __acquire_irq_guard() -> bool { - false -} - -#[no_mangle] -fn __release_irq_guard(_: bool) {} - -#[no_mangle] -fn __current_thread() -> Arc<()> { - Arc::new(()) -} - -#[no_mangle] -fn __suspend_current(_: &()) -> Result<(), Error> { - todo!(); -} - -#[no_mangle] -fn __enqueue(_: &Arc<()>) {} diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 066d2505..f850d83a 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -16,6 +16,3 @@ ygg_driver_block = { path = "../../driver/block/core" } log = "0.4.20" futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } - -[dev-dependencies] -hosted-tests = { path = "../hosted-tests" } diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index a9edadff..a052ef43 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -564,11 +564,13 @@ impl FileSet { #[cfg(test)] mod tests { - use core::{mem::MaybeUninit, str::FromStr}; + use core::{ + mem::MaybeUninit, + str::FromStr, + task::{Context, Poll}, + }; use std::sync::{Arc, Mutex}; - use libk::sync::IrqSafeSpinlock; - use ygg_driver_block::BlockDevice; use yggdrasil_abi::{ error::Error, io::{DirectoryEntry, FileType, OpenOptions, SeekFrom}, @@ -581,7 +583,7 @@ mod tests { impls::const_value_node, node::{AccessToken, CommonImpl, DirectoryImpl, Node, NodeFlags, NodeRef, RegularImpl}, traits::{Read, Seek, Write}, - InstanceData, + FileReadiness, InstanceData, }; #[test] @@ -808,74 +810,16 @@ mod tests { assert_eq!(&buf[..7], b"l123456"); } - #[test] - fn block_device() { - struct B { - data: Arc>>, - } - - impl BlockDevice for B { - fn read(&self, pos: u64, buf: &mut [u8]) -> Result { - let data = self.data.lock(); - let pos = pos as usize; - if pos >= data.len() { - return Ok(0); - } - - let count = core::cmp::min(data.len() - pos, buf.len()); - buf[..count].copy_from_slice(&data[pos..pos + count]); - Ok(count) - } - - fn write(&self, pos: u64, buf: &[u8]) -> Result { - let mut data = self.data.lock(); - let pos = pos as usize; - if pos >= data.len() { - return Ok(0); - } - - let count = core::cmp::min(data.len() - pos, buf.len()); - data[pos..pos + count].copy_from_slice(&buf[..count]); - Ok(count) - } - - fn size(&self) -> Result { - Ok(self.data.lock().len() as _) - } - } - - let vec = vec![0; 1024]; - let state = Arc::new(IrqSafeSpinlock::new(vec)); - let data = state.clone(); - let dev = Box::leak(Box::new(B { data })); - let mut buf = [0; 512]; - - let node = Node::block(dev, NodeFlags::empty()); - - let file = node - .open( - OpenOptions::READ | OpenOptions::WRITE, - AccessToken::test_authorized(), - ) - .unwrap(); - - assert_eq!(file.seek(SeekFrom::End(0)).unwrap(), 1024); - assert_eq!(file.write(b"12345").unwrap(), 0); - assert_eq!(file.seek(SeekFrom::Start(0)).unwrap(), 0); - assert_eq!(file.write(b"12345").unwrap(), 5); - assert_eq!(&state.lock()[..6], b"12345\0"); - - assert_eq!(file.read(&mut buf).unwrap(), 512); - assert_eq!(buf, [0; 512]); - assert_eq!(file.seek(SeekFrom::Start(2)).unwrap(), 2); - assert_eq!(file.read(&mut buf[..8]).unwrap(), 8); - assert_eq!(&buf[..8], b"345\0\0\0\0\0"); - } - #[test] fn char_device() { struct C; + impl FileReadiness for C { + fn poll_read(&self, _cx: &mut Context<'_>) -> Poll> { + unreachable!() + } + } + impl CharDevice for C { fn read(&self, buf: &mut [u8]) -> Result { buf.fill(b'@'); diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 43789c13..03bac5a7 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,6 +1,7 @@ //! Virtual filesystem interfaces and driver implementation #![cfg_attr(not(test), no_std)] +#![cfg_attr(test, allow(unused_imports))] #![allow(clippy::new_ret_no_self, clippy::new_without_default)] #![deny(missing_docs)] #![feature( @@ -12,9 +13,6 @@ trait_upcasting )] -#[cfg(test)] -extern crate hosted_tests; - extern crate alloc; pub(crate) mod channel; From 1c0212238ab6427f6d8000c0002763ae4f273dfd Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 12 Mar 2024 13:46:24 +0200 Subject: [PATCH 209/211] refactor: implement generated ABI --- Cargo.lock | 1055 --------------------- Cargo.toml | 7 + build.rs | 32 +- driver/fs/memfs/src/lib.rs | 4 +- libk/libk-thread/Cargo.toml | 1 + libk/libk-thread/src/process.rs | 9 +- src/arch/aarch64/exception.rs | 2 +- src/arch/aarch64/mod.rs | 1 + src/arch/x86_64/syscall.rs | 3 +- src/main.rs | 3 +- src/syscall/arg.rs | 55 +- src/syscall/mod.rs | 1546 ++++++++++++++++--------------- 12 files changed, 847 insertions(+), 1871 deletions(-) delete mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 245875f0..00000000 --- a/Cargo.lock +++ /dev/null @@ -1,1055 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "aarch64-cpu" -version = "9.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac42a04a61c19fc8196dd728022a784baecc5d63d7e256c01ad1b3fbfab26287" -dependencies = [ - "tock-registers", -] - -[[package]] -name = "accessor" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8b2abd55bf1f9cffbf00fd594566c51a9d31402553284920c1309ca8351086" - -[[package]] -name = "acpi" -version = "4.1.1" -source = "git+https://github.com/alnyan/acpi.git?branch=acpi-system#5efda0f9c17bccb922cf2b6b5fd73088bb887965" -dependencies = [ - "bit_field", - "log", - "rsdp", -] - -[[package]] -name = "acpi-system" -version = "0.1.0" -source = "git+https://github.com/alnyan/acpi-system.git#17192e5f603fdc7d78c9b83885e8a9d9aac45af7" -dependencies = [ - "acpi", - "aml", - "bit_field", - "enum-map", - "log", -] - -[[package]] -name = "aml" -version = "0.16.4" -source = "git+https://github.com/alnyan/acpi.git?branch=acpi-system#5efda0f9c17bccb922cf2b6b5fd73088bb887965" -dependencies = [ - "bit_field", - "bitvec", - "byteorder", - "log", - "spinning_top", -] - -[[package]] -name = "atomic_enum" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - -[[package]] -name = "bit_field" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" - -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - -[[package]] -name = "btree_monstrousity" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4d0977e9c15f276380f16f2e9594257c258172b23af39ffd2e4cf5971cb38c7" -dependencies = [ - "cfg-if", - "rustversion", -] - -[[package]] -name = "bytemuck" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" - -[[package]] -name = "device-api" -version = "0.1.0" -dependencies = [ - "device-api-macros", - "yggdrasil-abi", -] - -[[package]] -name = "device-api-macros" -version = "0.1.0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "discrete_range_map" -version = "0.6.2" -source = "git+https://git.alnyan.me/yggdrasil/discrete_range_map.git#10fd79828d2918bd079a11c2b2c623eede170c3f" -dependencies = [ - "btree_monstrousity", - "either", - "itertools 0.12.0", - "serde", -] - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - -[[package]] -name = "elf" -version = "0.7.2" -source = "git+https://git.alnyan.me/yggdrasil/yggdrasil-elf.git#419cd311de2e9514b5033677cde9a33f7d0ba4a2" -dependencies = [ - "hashbrown", -] - -[[package]] -name = "endian-type-rs" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6419a5c75e40011b9fe0174db3fe24006ab122fbe1b7e9cc5974b338a755c76" - -[[package]] -name = "enum-map" -version = "2.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6866f3bfdf8207509a033af1a75a7b08abda06bbaaeae6669323fd5a097df2e9" -dependencies = [ - "enum-map-derive", -] - -[[package]] -name = "enum-map-derive" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fastrand" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" - -[[package]] -name = "fdt-rs" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99a40cabc11c8258822a593f5c51f2d9f4923e715ca9e2a0630cf77ae15f390b" -dependencies = [ - "endian-type-rs", - "fallible-iterator", - "memoffset 0.5.6", - "num-derive", - "num-traits", - "rustc_version", - "static_assertions", - "unsafe_unwrap", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - -[[package]] -name = "futures-core" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" - -[[package]] -name = "futures-task" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" - -[[package]] -name = "futures-util" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "pin-utils", -] - -[[package]] -name = "getrandom" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "git-version" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" -dependencies = [ - "git-version-macro", -] - -[[package]] -name = "git-version-macro" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "hashbrown" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" - -[[package]] -name = "hosted-tests" -version = "0.1.0" -dependencies = [ - "yggdrasil-abi", -] - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" -dependencies = [ - "either", -] - -[[package]] -name = "kernel-fs" -version = "0.1.0" -dependencies = [ - "kernel-util", - "log", - "vfs", - "ygg_driver_block", - "yggdrasil-abi", -] - -[[package]] -name = "kernel-util" -version = "0.1.0" -dependencies = [ - "crossbeam-queue", - "device-api", - "futures-util", - "log", - "yggdrasil-abi", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.152" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" - -[[package]] -name = "libm" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" - -[[package]] -name = "linked_list_allocator" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afa463f5405ee81cdb9cc2baf37e08ec7e4c8209442b5d72c04cfb2cd6e6286" -dependencies = [ - "spinning_top", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" - -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" - -[[package]] -name = "memfs" -version = "0.1.0" -dependencies = [ - "kernel-util", - "log", - "static_assertions", - "vfs", - "yggdrasil-abi", -] - -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - -[[package]] -name = "memtables" -version = "0.1.0" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "num-derive" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "num-traits" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "proc-macro2" -version = "1.0.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "proptest" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" -dependencies = [ - "bit-set", - "bit-vec", - "bitflags 2.4.1", - "lazy_static", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "rusty-fork", - "tempfile", - "unarray", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quote" -version = "1.0.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_xorshift" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" -dependencies = [ - "rand_core", -] - -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "rsdp" -version = "2.0.0" -source = "git+https://github.com/alnyan/acpi.git?branch=acpi-system#5efda0f9c17bccb922cf2b6b5fd73088bb887965" -dependencies = [ - "log", -] - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - -[[package]] -name = "rustix" -version = "0.38.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" -dependencies = [ - "bitflags 2.4.1", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - -[[package]] -name = "rusty-fork" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - -[[package]] -name = "serde" -version = "1.0.195" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.195" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - -[[package]] -name = "spinning_top" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9eb1a2f4c41445a3a0ff9abc5221c5fcd28e1f13cd7c0397706f9ac938ddb0" -dependencies = [ - "lock_api", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "tempfile" -version = "3.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" -dependencies = [ - "cfg-if", - "fastrand", - "redox_syscall", - "rustix", - "windows-sys", -] - -[[package]] -name = "tock-registers" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" - -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unsafe_unwrap" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1230ec65f13e0f9b28d789da20d2d419511893ea9dac2c1f4ef67b8b14e5da80" - -[[package]] -name = "uuid" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "vfs" -version = "0.1.0" -dependencies = [ - "futures-util", - "hosted-tests", - "kernel-util", - "log", - "ygg_driver_block", - "yggdrasil-abi", -] - -[[package]] -name = "vmalloc" -version = "0.1.0" -dependencies = [ - "discrete_range_map", - "itertools 0.11.0", - "proptest", - "yggdrasil-abi", -] - -[[package]] -name = "wait-timeout" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" -dependencies = [ - "libc", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "xhci" -version = "0.9.2" -source = "git+https://github.com/rust-osdev/xhci.git#6144829ff13c2499dc812cea09df4e5a1d0862cc" -dependencies = [ - "accessor", - "bit_field", - "num-derive", - "num-traits", - "paste", -] - -[[package]] -name = "yboot-proto" -version = "0.1.0" -source = "git+https://git.alnyan.me/yggdrasil/yboot-proto.git#edf5db1b1d5d5dcb9d737cd7ec23b2e124b3b027" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "ygg_driver_ahci" -version = "0.1.0" -dependencies = [ - "bytemuck", - "device-api", - "futures-util", - "kernel-fs", - "kernel-util", - "log", - "memoffset 0.9.0", - "static_assertions", - "tock-registers", - "vfs", - "ygg_driver_block", - "ygg_driver_pci", - "yggdrasil-abi", -] - -[[package]] -name = "ygg_driver_block" -version = "0.1.0" -dependencies = [ - "bytemuck", - "futures-util", - "kernel-util", - "log", - "static_assertions", - "uuid", - "yggdrasil-abi", -] - -[[package]] -name = "ygg_driver_nvme" -version = "0.1.0" -dependencies = [ - "bytemuck", - "device-api", - "futures-util", - "kernel-fs", - "kernel-util", - "log", - "static_assertions", - "tock-registers", - "vfs", - "ygg_driver_block", - "ygg_driver_pci", - "yggdrasil-abi", -] - -[[package]] -name = "ygg_driver_pci" -version = "0.1.0" -dependencies = [ - "acpi", - "bitflags 2.4.1", - "device-api", - "kernel-util", - "log", - "yggdrasil-abi", -] - -[[package]] -name = "ygg_driver_virtio_net" -version = "0.1.0" -dependencies = [ - "ygg_driver_pci", -] - -[[package]] -name = "yggdrasil-abi" -version = "0.1.0" -source = "git+https://git.alnyan.me/yggdrasil/yggdrasil-abi.git#d2173e1e9e92e668399cf83aeb32e09823826821" - -[[package]] -name = "yggdrasil-kernel" -version = "0.1.0" -dependencies = [ - "aarch64-cpu", - "acpi", - "acpi-system", - "aml", - "atomic_enum", - "bitflags 2.4.1", - "bytemuck", - "cfg-if", - "crossbeam-queue", - "device-api", - "device-api-macros", - "elf", - "fdt-rs", - "futures-util", - "git-version", - "kernel-fs", - "kernel-util", - "linked_list_allocator", - "log", - "memfs", - "memtables", - "spinning_top", - "static_assertions", - "tock-registers", - "vfs", - "vmalloc", - "xhci", - "yboot-proto", - "ygg_driver_ahci", - "ygg_driver_block", - "ygg_driver_nvme", - "ygg_driver_pci", - "ygg_driver_virtio_net", - "yggdrasil-abi", -] diff --git a/Cargo.toml b/Cargo.toml index 08935458..c4dcee1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,8 @@ authors = ["Mark Poliakov "] opt-level = 3 [dependencies] +abi-lib = { git = "https://git.alnyan.me/yggdrasil/abi-generator.git" } + yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } vfs = { path = "lib/vfs" } device-api = { path = "lib/device-api", features = ["derive"] } @@ -70,6 +72,11 @@ acpi-system = { git = "https://github.com/alnyan/acpi-system.git" } ygg_driver_nvme = { path = "driver/block/nvme" } kernel-arch-x86_64 = { path = "arch/x86_64" } +[build-dependencies] +prettyplease = "0.2.15" +yggdrasil-abi-def = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi-def.git" } +abi-generator = { git = "https://git.alnyan.me/yggdrasil/abi-generator.git" } + [features] default = ["fb_console"] fb_console = [] diff --git a/build.rs b/build.rs index c015254e..5307bb10 100644 --- a/build.rs +++ b/build.rs @@ -1,10 +1,16 @@ use std::{ - env, + env, fs, io::{self, Write}, - path::PathBuf, + path::{Path, PathBuf}, process::Command, }; +use abi_generator::{ + abi::{ty::TypeWidth, AbiBuilder}, + syntax::UnwrapFancy, + TargetEnv, +}; + fn build_x86_64() { const DEFAULT_8086_AS: &str = "nasm"; const AP_BOOTSTRAP_S: &str = "src/arch/x86_64/boot/ap_boot.S"; @@ -33,9 +39,31 @@ fn build_x86_64() { } } +fn generate_syscall_dispatcher>(out_dir: P) { + let abi: AbiBuilder = AbiBuilder::from_string( + yggdrasil_abi_def::ABI_FILE, + TargetEnv { + thin_pointer_width: TypeWidth::U64, + fat_pointer_width: TypeWidth::U128, + }, + ) + .unwrap_fancy(""); + + let generated_dispatcher = out_dir.as_ref().join("generated_dispatcher.rs"); + let file = prettyplease::unparse( + &abi.emit_syscall_dispatcher("handle_syscall", "impls") + .unwrap_fancy(""), + ); + + fs::write(generated_dispatcher, file.as_bytes()).unwrap(); +} + fn main() { + let out_dir = env::var("OUT_DIR").unwrap(); let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); + generate_syscall_dispatcher(&out_dir); + println!("cargo:rerun-if-changed=build.rs"); match arch.as_str() { diff --git a/driver/fs/memfs/src/lib.rs b/driver/fs/memfs/src/lib.rs index cd16449f..54894160 100644 --- a/driver/fs/memfs/src/lib.rs +++ b/driver/fs/memfs/src/lib.rs @@ -148,8 +148,8 @@ impl MemoryFilesystem { let node = self.make_path(&root, path, FileType::Directory, false)?; assert_eq!(node.ty(), hdr.node_kind()); - let uid = UserId::from(usize::from(&hdr.uid) as u32); - let gid = GroupId::from(usize::from(&hdr.gid) as u32); + let uid = unsafe { UserId::from_raw(usize::from(&hdr.uid) as u32) }; + let gid = unsafe { GroupId::from_raw(usize::from(&hdr.gid) as u32) }; let mode = convert_mode(usize::from(&hdr.mode))?; let access = unsafe { AccessToken::authorized() }; diff --git a/libk/libk-thread/Cargo.toml b/libk/libk-thread/Cargo.toml index 03634422..984d0388 100644 --- a/libk/libk-thread/Cargo.toml +++ b/libk/libk-thread/Cargo.toml @@ -10,6 +10,7 @@ libk-util = { path = "../libk-util" } libk-mm = { path = "../libk-mm" } libk-device = { path = "../libk-device" } kernel-arch = { path = "../../arch" } +abi-lib = { git = "https://git.alnyan.me/yggdrasil/abi-generator.git" } yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } diff --git a/libk/libk-thread/src/process.rs b/libk/libk-thread/src/process.rs index 1ef594b9..fab3a36f 100644 --- a/libk/libk-thread/src/process.rs +++ b/libk/libk-thread/src/process.rs @@ -1,5 +1,6 @@ use core::marker::PhantomData; +use abi_lib::SyscallRegister; use alloc::{ collections::BTreeMap, string::String, @@ -17,7 +18,7 @@ use libk_util::{ }, }; use yggdrasil_abi::{ - error::{Error, SyscallResult}, + error::Error, process::{ExitCode, Signal, ThreadSpawnOptions}, }; @@ -247,12 +248,12 @@ impl, IO: ProcessIo> ProcessImpl { let src_thread = Thread::current(); let src_process = src_thread.process::(); - let rax = src_process + let value = src_process .fork_inner(frame) .map(|ProcessId(p)| p as u32) - .into_syscall_result(); + .into_syscall_register(); - frame.set_return_value(rax as _); + frame.set_return_value(value as _); } /// Replaces the process address space with a new one, loaded from the specified program diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 1c5c8ba6..5334489e 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -11,7 +11,7 @@ use aarch64_cpu::{ }; use abi::{ process::{Signal, SignalEntryData}, - syscall::SyscallFunction, + SyscallFunction, }; use kernel_arch::{task::TaskFrame, Architecture, ArchitectureImpl}; use kernel_arch_aarch64::context::ExceptionFrame; diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 5144d879..e358a2fb 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -93,6 +93,7 @@ impl Platform for AArch64 { } unsafe fn reset(&self) -> ! { + loop {} if let Some(reset) = self.reset.try_get() { reset.reset() } else { diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index faff7bc4..7685e33e 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -2,7 +2,7 @@ use core::arch::global_asm; -use abi::{process::SignalEntryData, syscall::SyscallFunction}; +use abi::{process::SignalEntryData, SyscallFunction}; use kernel_arch::task::TaskFrame; use kernel_arch_x86_64::{ context::SyscallFrame, @@ -23,6 +23,7 @@ fn syscall_inner(frame: &mut SyscallFrame) { return; } } + // FIXME: Fork is temporarily disabled if frame.rax == usize::from(SyscallFunction::Fork) as u64 { unsafe { ProcessImpl::raw_fork(frame); diff --git a/src/main.rs b/src/main.rs index 23818470..df05203a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,8 @@ inline_const, maybe_uninit_uninit_array, const_maybe_uninit_uninit_array, - never_type + never_type, + trace_macros )] #![allow( clippy::new_without_default, diff --git a/src/syscall/arg.rs b/src/syscall/arg.rs index f7049d8f..a7dcf4e8 100644 --- a/src/syscall/arg.rs +++ b/src/syscall/arg.rs @@ -1,26 +1,19 @@ -use abi::io::RawFd; -use libk_thread::{ - mem::{validate_user_region, ForeignPointer}, - thread::Thread, -}; +use libk_thread::{mem::ForeignPointer, thread::Thread}; use yggdrasil_abi::error::Error; -// XXX - -pub(super) fn arg_buffer_ref<'a>(base: usize, len: usize) -> Result<&'a [u8], Error> { +pub(super) fn ref_const<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { let proc = Thread::current(); - validate_user_region(proc.address_space(), base, len, false)?; - Ok(unsafe { core::slice::from_raw_parts(base as *const u8, len) }) + let ptr = addr as *const T; + unsafe { ptr.validate_user_ptr(proc.address_space()) } +} +pub(super) fn ref_mut<'a, T: Sized>(addr: usize) -> Result<&'a mut T, Error> { + let proc = Thread::current(); + let ptr = addr as *mut T; + unsafe { ptr.validate_user_mut(proc.address_space()) } } -pub(super) fn arg_buffer_mut<'a>(base: usize, len: usize) -> Result<&'a mut [u8], Error> { - let proc = Thread::current(); - validate_user_region(proc.address_space(), base, len, true)?; - Ok(unsafe { core::slice::from_raw_parts_mut(base as *mut u8, len) }) -} - -pub(super) fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error> { - let slice = arg_buffer_ref(base, len)?; +pub(super) fn str_ref<'a>(base: usize, len: usize) -> Result<&'a str, Error> { + let slice = slice_ref(base, len)?; if slice.contains(&0) { warnln!("User-supplied string contains NUL characters"); return Err(Error::InvalidArgument); @@ -28,31 +21,13 @@ pub(super) fn arg_user_str<'a>(base: usize, len: usize) -> Result<&'a str, Error Ok(core::str::from_utf8(slice).unwrap()) } -pub(super) fn arg_user_ref<'a, T: Sized>(addr: usize) -> Result<&'a T, Error> { +pub(super) fn slice_ref<'a, T: Sized>(base: usize, count: usize) -> Result<&'a [T], Error> { let proc = Thread::current(); - let ptr = addr as *const T; - unsafe { ptr.validate_user_ptr(proc.address_space()) } + let ptr = base as *const T; + unsafe { ptr.validate_user_slice(count, proc.address_space()) } } - -pub(super) fn arg_user_mut<'a, T: Sized>(addr: usize) -> Result<&'a mut T, Error> { - let proc = Thread::current(); - let ptr = addr as *mut T; - unsafe { ptr.validate_user_mut(proc.address_space()) } -} - -pub(super) fn arg_user_slice_mut<'a, T: Sized>( - base: usize, - count: usize, -) -> Result<&'a mut [T], Error> { +pub(super) fn slice_mut<'a, T: Sized>(base: usize, count: usize) -> Result<&'a mut [T], Error> { let proc = Thread::current(); let ptr = base as *mut T; unsafe { ptr.validate_user_slice_mut(count, proc.address_space()) } } - -pub(super) fn arg_option_fd(raw: u32) -> Option { - if raw == u32::MAX { - None - } else { - Some(RawFd(raw)) - } -} diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index f576e67d..592a0000 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -1,48 +1,12 @@ //! System function call handlers -use core::{mem::MaybeUninit, time::Duration}; -use abi::{ - error::Error, - io::{ - DeviceRequest, DirectoryEntry, FileAttr, FileMetadataUpdate, FileMode, MessageDestination, - OpenOptions, PollControl, RawFd, ReceivedMessageMetadata, SeekFrom, SentMessage, - TerminalOptions, TerminalSize, - }, - mem::MappingSource, - net::{SocketOption, SocketType}, - process::{ - ExecveOptions, ExitCode, MutexOperation, ProcessInfoElement, Signal, SpawnOption, - SpawnOptions, ThreadSpawnOptions, - }, - syscall::SyscallFunction, - system::SystemInfo, -}; -use alloc::{borrow::ToOwned, boxed::Box, sync::Arc, vec::Vec}; -use libk::{block, runtime}; -use libk_mm::{ - phys, - process::VirtualRangeBacking, - table::{EntryLevelExt, MapAttributes}, -}; -use libk_thread::{ - process::{Process, ProcessManager}, - thread::{CurrentThread, Thread}, -}; +use abi::{error::Error, io::RawFd, SyscallFunction}; use libk_util::sync::IrqSafeSpinlockGuard; -use vfs::{File, IoContext, MessagePayload, NodeRef, Read, Seek, Write}; -use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; -use yggdrasil_abi::{error::SyscallResult, io::MountOptions}; +use vfs::NodeRef; -use crate::{ - arch::L3, - debug::LogLevel, - fs, - proc::{self, io::ProcessIoImpl, random}, - task::process::{ProcessId, ProcessImpl, ProcessManagerImpl}, -}; +use crate::{proc::io::ProcessIoImpl, task::process::ProcessImpl}; mod arg; -use arg::*; fn run_with_io) -> T>( proc: &ProcessImpl, @@ -74,754 +38,812 @@ fn run_with_io_at< f(at, io) } -fn sys_execve(thread: CurrentThread, options: &ExecveOptions) -> Result { - let process = thread.process::(); +mod impls { + use abi::process::{ExecveOptions, SpawnOption}; + pub(crate) use abi::{ + error::Error, + io::{ + DeviceRequest, DirectoryEntry, FileAttr, FileMetadataUpdate, FileMode, + MessageDestination, MountOptions, OpenOptions, PollControl, RawFd, + ReceivedMessageMetadata, SeekFrom, SentMessage, TerminalOptions, TerminalSize, + UnmountOptions, + }, + mem::MappingSource, + net::{SocketOption, SocketType}, + process::{ExitCode, MutexOperation, Signal, SignalEntryData, SpawnOptions}, + system::SystemInfo, + }; + use alloc::{boxed::Box, sync::Arc}; + use libk::{block, runtime}; + use vfs::{File, IoContext, MessagePayload, Read, Seek, Write}; + use ygg_driver_net_core::socket::{RawSocket, TcpListener, TcpSocket, UdpSocket}; - // Clone the options into the kernel - let mut argv = Vec::new(); - let mut envp = Vec::new(); + use core::{ + mem::MaybeUninit, net::SocketAddr, num::NonZeroUsize, sync::atomic::AtomicU32, + time::Duration, + }; - for &arg in options.arguments { - argv.push(arg.to_owned()); - } - for &env in options.environment { - envp.push(env.to_owned()); + use libk_mm::{ + phys, + process::VirtualRangeBacking, + table::{EntryLevelExt, MapAttributes}, + }; + use libk_thread::{ + process::{Process, ProcessManager}, + thread::Thread, + types::ProcessId, + }; + + use crate::{ + arch::L3, + debug::LogLevel, + fs, + proc::{self, random}, + task::process::ProcessManagerImpl, + }; + + use super::{run_with_io, run_with_io_at}; + + // Misc + pub(crate) fn debug_trace(message: &str) { + let thread = Thread::current(); + let process = thread.process::(); + + log_print_raw!( + LogLevel::Debug, + "[{}:{}] TRACE: {}\n", + process.id(), + thread.id, + message + ); } - process.exec(options.program, argv, envp) -} + pub(crate) fn get_random(buffer: &mut [u8]) { + random::read(buffer); + } -fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result { - let thread = Thread::current(); - let process = thread.process::(); + pub(crate) fn mount(options: &MountOptions<'_>) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); - match func { - SyscallFunction::DebugTrace => { - let pid = process.id(); - let tid = thread.id; + run_with_io(&process, |mut io| { + let fs_root = fs::create_filesystem(options)?; + io.ioctx_mut().mount(options.target, fs_root)?; + Ok(()) + }) + } - let arg = arg_user_str(args[0] as usize, args[1] as usize)?; + pub(crate) fn unmount(_options: &UnmountOptions) -> Result<(), Error> { + todo!() + } - log_print_raw!(LogLevel::Debug, "[{}:{}] TRACE: {}\n", pid, tid, arg); + // Memory management + pub(crate) fn map_memory( + _hint: Option, + len: usize, + source: &MappingSource, + ) -> Result { + let thread = Thread::current(); + let process = thread.process::(); - Ok(0) - } - SyscallFunction::GetRandom => { - let buf = arg_buffer_mut(args[0] as _, args[1] as _)?; - random::read(buf); - Ok(0) - } - SyscallFunction::Nanosleep => { - let seconds = args[0]; - let nanos = args[1] as u32; - let duration = Duration::new(seconds, nanos); + let space = thread.address_space(); - block! { - runtime::sleep(duration).await - } - .map(|_| 0) - } - // Resource management - SyscallFunction::MapMemory => { - let mut len = args[1] as usize; - let source = arg_user_ref::(args[2] as usize)?; + let len = len.page_align_up::(); - let space = thread.address_space(); - - len = len.page_align_up::(); - - run_with_io(&process, |io| { - let backing = match source { - MappingSource::Anonymous => VirtualRangeBacking::anonymous(), - &MappingSource::File(fd, offset) => { - let file = io.files.file(fd)?; - VirtualRangeBacking::file(offset, file.clone())? - } - }; - - space.allocate( - None, - len, - backing, - MapAttributes::USER_WRITE - | MapAttributes::USER_READ - | MapAttributes::NON_GLOBAL, - ) - }) - } - SyscallFunction::UnmapMemory => { - let addr = args[0] as usize; - let len = args[1] as usize; - - let space = thread.address_space(); - - if len & 0xFFF != 0 { - todo!(); - } - - unsafe { - space.unmap(addr, len)?; - } - - Ok(0) - } - SyscallFunction::SetSignalEntry => { - let entry = args[0] as usize; - let sp = args[1] as usize; - - thread.set_signal_entry(entry, sp); - - Ok(0) - } - // I/O - SyscallFunction::Open => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let opts = OpenOptions::from(args[3] as u32); - let mode = FileMode::from(args[4] as u32); - - run_with_io_at(&process, at, |at, mut io| { - let file = io.ioctx_mut().open(Some(at), path, opts, mode)?; - - // TODO NO_CTTY? - if process.session_terminal().is_none() - && let Some(node) = file.node() - && node.is_terminal() - { - debugln!("Session terminal set for #{}: {}", process.id(), path); - process.set_session_terminal(node.clone()); + run_with_io(&process, |io| { + let backing = match source { + MappingSource::Anonymous => VirtualRangeBacking::anonymous(), + &MappingSource::File(fd, offset) => { + let file = io.files.file(fd)?; + VirtualRangeBacking::file(offset, file.clone())? } + }; - let fd = io.files.place_file(file, true)?; - Ok(fd.0 as usize) - }) - } - SyscallFunction::OpenDirectory => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; + space.allocate( + None, + len, + backing, + MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, + ) + }) + } - run_with_io_at(&process, at, |at, mut io| { - let node = io.ioctx_mut().find(Some(at), path, true, true)?; - let access = io.ioctx_mut().check_access(vfs::Action::Read, &node)?; - let file = node.open_directory(access)?; - let fd = io.files.place_file(file, true)?; + pub(crate) fn unmap_memory(address: usize, len: usize) -> Result<(), Error> { + let thread = Thread::current(); + let space = thread.address_space(); - Ok(fd.0 as usize) - }) - } - SyscallFunction::Read => { - let fd = RawFd(args[0] as u32); - let data = arg_buffer_mut(args[1] as _, args[2] as _)?; - - run_with_io(&process, |io| io.files.file(fd)?.read(data)) - } - SyscallFunction::Write => { - let fd = RawFd(args[0] as u32); - let data = arg_buffer_ref(args[1] as _, args[2] as _)?; - - run_with_io(&process, |io| io.files.file(fd)?.write(data)) - } - SyscallFunction::Seek => { - let fd = RawFd(args[0] as u32); - let pos = SeekFrom::from(args[1]); - - run_with_io(&process, |io| { - io.files.file(fd)?.seek(pos).map(|v| v as usize) - }) - } - SyscallFunction::ReadDirectory => { - let fd = RawFd(args[0] as u32); - let buffer = arg_user_slice_mut::>( - args[1] as usize, - args[2] as usize, - )?; - - run_with_io(&process, |io| io.files.file(fd)?.read_dir(buffer)) - } - SyscallFunction::Close => { - let fd = RawFd(args[0] as u32); - - run_with_io(&process, |mut io| { - let res = io.files.close_file(fd); - - if res == Err(Error::InvalidFile) { - warnln!("Double close of fd {:?} in process {}", fd, process.id()); - } - - res?; - - Ok(0) - }) - } - SyscallFunction::Mount => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(&process, |mut io| { - let fs_root = fs::create_filesystem(options)?; - io.ioctx_mut().mount(options.target, fs_root)?; - Ok(0) - }) - } - SyscallFunction::Unmount => { + if len & 0xFFF != 0 { todo!(); } - SyscallFunction::DeviceRequest => { - let fd = RawFd(args[0] as u32); - let req = arg_user_mut::(args[1] as usize)?; - run_with_io(&process, |io| { - let file = io.files.file(fd)?; - // let node = file.node().ok_or(Error::InvalidFile)?; - file.device_request(req)?; - Ok(0) - }) + unsafe { + space.unmap(address, len)?; } - SyscallFunction::GetMetadata => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let buffer = arg_user_mut::>(args[3] as usize)?; - let follow = args[4] != 0; - run_with_io_at(&process, at, |at, mut io| { - let node = if path.is_empty() { - at - // at.ok_or(Error::InvalidArgument)? + Ok(()) + } + + pub(crate) fn get_system_info(element: &mut SystemInfo) -> Result<(), Error> { + match element { + SystemInfo::MemoryStats(stats) => { + *stats = phys::stats(); + Ok(()) + } + } + } + + // Process/thread management + pub(crate) fn exit_process(code: ExitCode) -> ! { + let thread = Thread::current(); + thread.exit_process::(code) + } + + pub(crate) fn spawn_process(options: &SpawnOptions<'_>) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + // Setup a new process from the file + let (child_process, child_main) = proc::load_binary( + io.ioctx_mut(), + Some(Arc::downgrade(&process)), + options.program, + options.arguments, + options.environment, + )?; + let pid: u32 = child_process.id().into(); + + // Inherit group and session from the creator + child_process.inherit(&process)?; + + // Inherit root from the creator + // let child_ioctx = IoContext::new(io.ioctx().root().clone()); + let child_ioctx = IoContext::inherit(io.ioctx_mut()); + let mut child_io = child_process.io.lock(); + child_io.set_ioctx(child_ioctx); + + for opt in options.optional { + match opt { + &SpawnOption::InheritFile { source, child } => { + if let Ok(src_file) = io.files.file(source) { + child_io.files.set_file(child, src_file.clone())?; + } + } + &SpawnOption::SetProcessGroup(pgroup) => { + let pgroup = if pgroup == 0 { + child_process.id() + } else { + pgroup.into() + }; + child_process.set_group_id(pgroup); + } + _ => (), + } + } + + if let Some(fd) = options.optional.iter().find_map(|item| { + if let &SpawnOption::GainTerminal(fd) = item { + Some(fd) } else { - io.ioctx_mut().find(Some(at), path, follow, true)? - }; + None + } + }) { + debugln!("{} requested terminal {:?}", pid, fd); + let file = child_io.files.file(fd)?; + // let node = file.node().ok_or(Error::InvalidFile)?; + let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); + file.device_request(&mut req)?; - let metadata = node.metadata()?; - let size = node.size()?; + if let Some(node) = file.node() { + child_process.set_session_terminal(node.clone()); + } + // node.device_request(&mut req)?; + } - buffer.write(FileAttr { - size, - ty: node.ty(), - mode: metadata.mode, - uid: metadata.uid, - gid: metadata.gid, - }); + drop(child_io); + child_main.enqueue(); - Ok(0) - }) + Ok(pid as _) + }) + } + + pub(crate) fn wait_process(pid: u32, status: &mut ExitCode) -> Result<(), Error> { + let pid = ProcessId::from(pid); + let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; + *status = block!(target.wait_for_exit().await)?; + Ok(()) + } + + pub(crate) fn get_pid() -> u32 { + let thread = Thread::current(); + let process = thread.process::(); + process.id().into() + } + + pub(crate) fn nanosleep(duration: &Duration) { + block! { + runtime::sleep(*duration).await } - SyscallFunction::UpdateMetadata => { - let at = arg_option_fd(args[0] as u32); - let _path = arg_user_str(args[1] as usize, args[2] as usize)?; - let _update = arg_user_ref::(args[3] as usize)?; - let _follow = args[4] != 0; + .unwrap(); + } - run_with_io_at(&process, at, |_at, _io| { - todo!(); - }) - } - SyscallFunction::CreateDirectory => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; - let mode = FileMode::from(args[3] as u32); + pub(crate) fn set_signal_entry(ip: usize, sp: usize) { + let thread = Thread::current(); + thread.set_signal_entry(ip, sp); + } - run_with_io_at(&process, at, |at, mut io| { - io.ioctx_mut().create_directory(Some(at), path, mode)?; - Ok(0) - }) - } - SyscallFunction::Remove => { - let at = arg_option_fd(args[0] as u32); - let path = arg_user_str(args[1] as usize, args[2] as usize)?; + pub(crate) fn send_signal(target: u32, signal: Signal) -> Result<(), Error> { + let pid = ProcessId(target as _); + let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; + target.raise_signal(signal); + Ok(()) + } - run_with_io_at(&process, at, |at, mut io| { - io.ioctx_mut().remove_file(Some(at), path)?; - Ok(0) - }) - } - SyscallFunction::RemoveDirectory => { - todo!() - } - SyscallFunction::CreatePipe => { - let ends: &mut [MaybeUninit; 2] = arg_user_mut(args[0] as usize)?; + pub(crate) fn mutex(mutex: &AtomicU32, op: &MutexOperation) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + let mutex = process.get_or_insert_mutex((mutex as *const AtomicU32).addr()); + + match op { + &MutexOperation::Wait(value, _timeout) => block! { mutex.wait(value).await }.unwrap(), + MutexOperation::Wake => mutex.wake(), + MutexOperation::WakeAll => mutex.wake_all(), + } + + Ok(()) + } + + pub(crate) fn start_session() -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + let session_terminal = process.clear_session_terminal(); + + if let Some(ctty) = session_terminal { + // Drop all FDs referring to the old session terminal run_with_io(&process, |mut io| { - let (read, write) = File::new_pipe_pair(256); - - let read_fd = io.files.place_file(read, true)?; - let write_fd = io.files.place_file(write, true)?; - - infoln!("Read end: {:?}", read_fd); - infoln!("Write end: {:?}", write_fd); - - ends[0].write(read_fd); - ends[1].write(write_fd); - - Ok(0) - }) + io.files.retain(|_, f| { + f.node() + .map(|node| !Arc::ptr_eq(node, &ctty)) + .unwrap_or(true) + }); + }); } - SyscallFunction::CreatePoll => run_with_io(&process, |mut io| { + + process.set_session_id(process.id()); + process.set_group_id(process.id()); + + Ok(()) + } + + // I/O + pub(crate) fn open( + at: Option, + path: &str, + opts: OpenOptions, + mode: FileMode, + ) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io_at(&process, at, |at, mut io| { + let file = io.ioctx_mut().open(Some(at), path, opts, mode)?; + + // TODO NO_CTTY? + if process.session_terminal().is_none() + && let Some(node) = file.node() + && node.is_terminal() + { + debugln!("Session terminal set for #{}: {}", process.id(), path); + process.set_session_terminal(node.clone()); + } + + let fd = io.files.place_file(file, true)?; + Ok(fd) + }) + } + + pub(crate) fn close(fd: RawFd) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let res = io.files.close_file(fd); + + if res == Err(Error::InvalidFile) { + warnln!("Double close of fd {:?} in process {}", fd, process.id()); + } + + res + }) + } + + pub(crate) fn write(fd: RawFd, buffer: &[u8]) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| io.files.file(fd)?.write(buffer)) + } + + pub(crate) fn read(fd: RawFd, buffer: &mut [u8]) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| io.files.file(fd)?.read(buffer)) + } + + pub(crate) fn seek(fd: RawFd, pos: SeekFrom) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| io.files.file(fd)?.seek(pos)) + } + + pub(crate) fn open_directory(at: Option, path: &str) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io_at(&process, at, |at, mut io| { + let node = io.ioctx_mut().find(Some(at), path, true, true)?; + let access = io.ioctx_mut().check_access(vfs::Action::Read, &node)?; + let file = node.open_directory(access)?; + let fd = io.files.place_file(file, true)?; + + Ok(fd) + }) + } + + pub(crate) fn read_directory_entries( + fd: RawFd, + entries: &mut [MaybeUninit], + ) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| io.files.file(fd)?.read_dir(entries)) + } + + pub(crate) fn create_directory( + at: Option, + path: &str, + mode: FileMode, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io_at(&process, at, |at, mut io| { + io.ioctx_mut().create_directory(Some(at), path, mode)?; + Ok(()) + }) + } + + pub(crate) fn remove_directory(_at: Option, _path: &str) -> Result<(), Error> { + todo!() + } + + pub(crate) fn remove(at: Option, path: &str) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io_at(&process, at, |at, mut io| { + io.ioctx_mut().remove_file(Some(at), path)?; + Ok(()) + }) + } + + pub(crate) fn clone_fd(source_fd: RawFd, target_fd: Option) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = io.files.file(source_fd)?.clone(); + + let fd = match target_fd { + Some(target_fd) => { + io.files.set_file(target_fd, file)?; + target_fd + } + None => io.files.place_file(file, true)?, + }; + + Ok(fd) + }) + } + + pub(crate) fn update_metadata( + at: Option, + _path: &str, + _update: &FileMetadataUpdate, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io_at(&process, at, |_at, _io| { + todo!(); + }) + } + + pub(crate) fn get_metadata( + at: Option, + path: &str, + buffer: &mut MaybeUninit, + follow: bool, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io_at(&process, at, |at, mut io| { + let node = if path.is_empty() { + at + // at.ok_or(Error::InvalidArgument)? + } else { + io.ioctx_mut().find(Some(at), path, follow, true)? + }; + + let metadata = node.metadata()?; + let size = node.size()?; + + buffer.write(FileAttr { + size, + ty: node.ty(), + mode: metadata.mode, + uid: metadata.uid, + gid: metadata.gid, + }); + + Ok(()) + }) + } + + pub(crate) fn device_request(fd: RawFd, req: &mut DeviceRequest) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| io.files.file(fd)?.device_request(req)) + } + + // Misc I/O + pub(crate) fn open_channel(name: &str, subscribe: bool) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = File::new_message_channel(name, subscribe); + let fd = io.files.place_file(file, true)?; + Ok(fd) + }) + } + + pub(crate) fn create_timer(repeat: bool) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = File::new_timer(repeat); + let fd = io.files.place_file(file, true)?; + Ok(fd) + }) + } + + pub(crate) fn create_pty( + options: &TerminalOptions, + size: &TerminalSize, + output: &mut [MaybeUninit; 2], + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let (master, slave) = File::new_pseudo_terminal(*options, *size)?; + let master_fd = io.files.place_file(master, true)?; + let slave_fd = io.files.place_file(slave, true)?; + + output[0].write(master_fd); + output[1].write(slave_fd); + + Ok(()) + }) + } + + pub(crate) fn create_shared_memory(size: usize) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = File::new_shared_memory(size)?; + let fd = io.files.place_file(file, true)?; + Ok(fd) + }) + } + + pub(crate) fn create_poll_channel() -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { let poll = File::new_poll_channel(); let fd = io.files.place_file(poll, true)?; - Ok(fd.0 as usize) - }), - SyscallFunction::PollControl => { - let poll_fd = RawFd::from(args[0] as u32); - let control = - PollControl::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - let fd = RawFd::from(args[2] as u32); - - run_with_io(&process, |io| { - let poll_file = io.files.file(poll_fd)?; - let poll = poll_file.as_poll_channel()?; - - match control { - PollControl::AddFd => { - let polled_file = io.files.file(fd)?.clone(); - poll.add(fd, polled_file); - } - } - - Ok(0) - }) - } - SyscallFunction::PollWait => { - let poll_fd = RawFd::from(args[0] as u32); - let timeout = arg_user_ref::>(args[1] as usize)?; - let output = arg_user_mut::)>>(args[2] as usize)?; - - run_with_io(&process, |io| { - let poll_file = io.files.file(poll_fd)?; - let poll = poll_file.as_poll_channel()?; - - *output = block! { - poll.wait(*timeout).await - }?; - - Ok(0) - }) - } - SyscallFunction::OpenChannel => { - let name = arg_user_str(args[0] as usize, args[1] as usize)?; - let with_sub = args[2] != 0; - - debugln!("OpenChannel {:?}, with_sub={}", name, with_sub); - - run_with_io(&process, |mut io| { - let file = File::new_message_channel(name, with_sub); - let fd = io.files.place_file(file, true)?; - - Ok(fd.0 as usize) - }) - } - SyscallFunction::SendMessage => { - let fd = RawFd::from(args[0] as u32); - let message = arg_user_ref::(args[1] as usize)?; - let destination = MessageDestination::from(args[2]); - - run_with_io(&process, |io| { - let file = io.files.file(fd)?; - let channel = file.as_message_channel()?; - - match message { - &SentMessage::File(fd) => { - let sent_file = io.files.file(fd)?; - - channel - .send_message(MessagePayload::File(sent_file.clone()), destination)?; - } - &SentMessage::Data(data) => { - channel.send_message(MessagePayload::Data(Box::from(data)), destination)?; - } - } - - Ok(0) - }) - } - SyscallFunction::ReceiveMessage => { - let fd = RawFd::from(args[0] as u32); - let metadata = arg_user_mut::>(args[1] as usize)?; - let buf = arg_buffer_mut(args[2] as usize, args[3] as usize)?; - let from = arg_user_mut::>(args[4] as usize)?; - - run_with_io(&process, |mut io| { - let file = io.files.file(fd)?; - let channel = file.as_message_channel()?; - - let message = channel.receive_message()?; - - from.write(message.source); - - match &message.payload { - MessagePayload::Data(data) => { - // TODO allow truncated messages? - let len = data.len(); - if buf.len() < len { - return Err(Error::MissingData); - } - - metadata.write(ReceivedMessageMetadata::Data(len)); - buf[..len].copy_from_slice(data); - - Ok(len) - } - MessagePayload::File(file) => { - let fd = io.files.place_file(file.clone(), true)?; - - metadata.write(ReceivedMessageMetadata::File(fd)); - - Ok(0) - } - } - }) - } - SyscallFunction::CreateSharedMemory => { - let size = args[0] as usize; - let size = size.page_align_up::(); - - run_with_io(&process, |mut io| { - let file = File::new_shared_memory(size)?; - let fd = io.files.place_file(file, true)?; - Ok(fd.0 as usize) - }) - } - SyscallFunction::CreatePty => { - let options = arg_user_ref::(args[0] as usize)?; - let size = arg_user_ref::(args[1] as usize)?; - let output = arg_user_mut::>(args[2] as usize)?; - - run_with_io(&process, |mut io| { - let (master, slave) = File::new_pseudo_terminal(*options, *size)?; - let master_fd = io.files.place_file(master, true)?; - let slave_fd = io.files.place_file(slave, true)?; - - output.write((master_fd, slave_fd)); - - Ok(0) - }) - } - SyscallFunction::CloneFd => { - let source_fd = RawFd::from(args[0] as u32); - let target_fd = RawFd::from(args[1] as u32); - - run_with_io(&process, |mut io| { - let file = io.files.file(source_fd)?.clone(); - - let fd = if target_fd != RawFd::NONE { - io.files.set_file(target_fd, file)?; - target_fd - } else { - io.files.place_file(file, true)? - }; - - Ok(fd.0 as usize) - }) - } - SyscallFunction::CreateTimer => { - let repeat = args[0] != 0; - - run_with_io(&process, |mut io| { - let file = File::new_timer(repeat); - let fd = io.files.place_file(file, true)?; - Ok(fd.0 as _) - }) - } - // Process management - SyscallFunction::SpawnProcess => { - let options = arg_user_ref::(args[0] as usize)?; - - run_with_io(&process, |mut io| { - // Setup a new process from the file - let (child_process, child_main) = proc::load_binary( - io.ioctx_mut(), - Some(Arc::downgrade(&process)), - options.program, - options.arguments, - options.environment, - )?; - let pid: u32 = child_process.id().into(); - - // Inherit group and session from the creator - child_process.inherit(&process)?; - - // Inherit root from the creator - // let child_ioctx = IoContext::new(io.ioctx().root().clone()); - let child_ioctx = IoContext::inherit(io.ioctx_mut()); - let mut child_io = child_process.io.lock(); - child_io.set_ioctx(child_ioctx); - - for opt in options.optional { - match opt { - &SpawnOption::InheritFile { source, child } => { - if let Ok(src_file) = io.files.file(source) { - child_io.files.set_file(child, src_file.clone())?; - } - } - &SpawnOption::SetProcessGroup(pgroup) => { - let pgroup = if pgroup == 0 { - child_process.id() - } else { - pgroup.into() - }; - child_process.set_group_id(pgroup); - } - _ => (), - } - } - - if let Some(fd) = options.optional.iter().find_map(|item| { - if let &SpawnOption::GainTerminal(fd) = item { - Some(fd) - } else { - None - } - }) { - debugln!("{} requested terminal {:?}", pid, fd); - let file = child_io.files.file(fd)?; - // let node = file.node().ok_or(Error::InvalidFile)?; - let mut req = DeviceRequest::SetTerminalGroup(child_process.group_id().into()); - file.device_request(&mut req)?; - - if let Some(node) = file.node() { - child_process.set_session_terminal(node.clone()); - } - // node.device_request(&mut req)?; - } - - drop(child_io); - child_main.enqueue(); - - Ok(pid as _) - }) - } - SyscallFunction::Exec => { - let options = arg_user_ref::(args[0] as usize)?; - sys_execve(thread, options)?; - } - SyscallFunction::SpawnThread => { - let options = arg_user_ref::(args[0] as usize)?; - let id = process.spawn_thread(options)?; - Ok(id.as_user() as _) - } - SyscallFunction::Mutex => { - let address = args[0] as usize; - let op = arg_user_ref::(args[1] as usize)?; - - let mutex = process.get_or_insert_mutex(address); - - match op { - &MutexOperation::Wait(value, _timeout) => { - block! { mutex.wait(value).await }.unwrap() - } - MutexOperation::Wake => mutex.wake(), - MutexOperation::WakeAll => mutex.wake_all(), - } - - Ok(0) - } - SyscallFunction::ExitThread => { - let code = ExitCode::from(args[0] as i32); - thread.exit::(code) - } - SyscallFunction::ExitProcess => { - // A bit different from thread exit: wait for other threads to finish and exit only - // after that - let code = ExitCode::from(args[0] as i32); - - thread.exit_process::(code) - } - SyscallFunction::SendSignal => { - let pid = ProcessId::from(args[0] as u32); - let signal = Signal::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - - let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; - target.raise_signal(signal); - - Ok(0) - } - SyscallFunction::ExitSignal => { - panic!("Handled elsewhere"); - // Process::current().exit_signal(); - } - SyscallFunction::GetPid => Ok(u32::from(process.id()) as usize), - SyscallFunction::GetSessionId => todo!(), - SyscallFunction::GetProcessGroupId => todo!(), - SyscallFunction::SetProcessGroupId => { - let pid = ProcessId::from(args[0] as u32); - let group_id = ProcessId::from(args[1] as u32); - - // Other syscall variants are not currently supported - assert_eq!(pid, group_id); - assert_eq!(process.id(), pid); - - process.set_group_id(group_id); - Ok(0) - } - SyscallFunction::GetProcessInfo => { - let element = arg_user_mut::(args[0] as _)?; - - run_with_io(&process, |mut io| match element { - ProcessInfoElement::Umask(mask) => { - *mask = io.ioctx_mut().umask(); - Ok(0) - } - }) - } - SyscallFunction::SetProcessInfo => { - let element = arg_user_ref::(args[0] as _)?; - - run_with_io(&process, |mut io| match element { - &ProcessInfoElement::Umask(mask) => { - io.ioctx_mut().set_umask(mask); - Ok(0) - } - }) - } - SyscallFunction::GetSystemInfo => { - let element = arg_user_mut::(args[0] as _)?; - - match element { - SystemInfo::MemoryStats(stats) => { - *stats = phys::stats(); - Ok(0) - } - } - } - SyscallFunction::StartSession => { - let session_terminal = process.clear_session_terminal(); - - if let Some(ctty) = session_terminal { - // Drop all FDs referring to the old session terminal - run_with_io(&process, |mut io| { - io.files.retain(|_, f| { - f.node() - .map(|node| !Arc::ptr_eq(node, &ctty)) - .unwrap_or(true) - }); - }); - } - - process.set_session_id(process.id()); - process.set_group_id(process.id()); - - Ok(0) - } - // Waiting and polling - SyscallFunction::WaitProcess => { - let pid = ProcessId::from(args[0] as u32); - debugln!("WaitProcess #{}", pid); - let status = arg_user_mut::(args[1] as _)?; - - let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; - - *status = block!(target.wait_for_exit().await)?; - - Ok(0) - } - - // Networking - SyscallFunction::BindSocket => { - let listen = arg_user_ref::(args[0] as usize)?; - let ty = SocketType::try_from(args[1] as u32).map_err(|_| Error::InvalidArgument)?; - - run_with_io(&process, |mut io| { - let file = match ty { - SocketType::UdpPacket => { - File::from_packet_socket(UdpSocket::bind((*listen).into())?) - } - SocketType::RawPacket => File::from_packet_socket(RawSocket::bind()?), - SocketType::TcpStream => { - File::from_listener_socket(TcpListener::bind((*listen).into())?) - } - }; - let fd = io.files.place_file(file, true)?; - Ok(fd.0 as usize) - }) - } - SyscallFunction::ConnectSocket => { - let socket_fd = RawFd::from(args[0] as u32); - let remote = arg_user_ref::(args[1] as usize)?; - let ty = SocketType::try_from(args[2] as u32).map_err(|_| Error::InvalidArgument)?; - let local_result = - arg_user_mut::>(args[3] as usize)?; - - // TODO connect on existing sockets? - assert_eq!(socket_fd, RawFd::NONE); - - run_with_io(&process, |mut io| { - let (local, file) = match ty { - SocketType::TcpStream => { - let (local, socket) = TcpSocket::connect((*remote).into())?; - (local, File::from_stream_socket(socket)) - } - _ => return Err(Error::InvalidArgument), - }; - let fd = io.files.place_file(file, true)?; - local_result.write(local.into()); - Ok(fd.0 as usize) - }) - } - SyscallFunction::SendTo => { - let socket_fd = RawFd::from(args[0] as u32); - let buffer = arg_buffer_ref(args[1] as usize, args[2] as usize)?; - let recepient = arg_user_ref::>(args[3] as usize)?; - - run_with_io(&process, |io| { - let file = io.files.file(socket_fd)?; - - file.send_to(buffer, recepient.map(Into::into)) - }) - } - SyscallFunction::ReceiveFrom => { - let socket_fd = RawFd::from(args[0] as u32); - let buffer = arg_buffer_mut(args[1] as usize, args[2] as usize)?; - let remote_result = - arg_user_mut::>(args[3] as usize)?; - - run_with_io(&process, |io| { - let file = io.files.file(socket_fd)?; - let mut remote = MaybeUninit::uninit(); - let len = file.receive_from(buffer, &mut remote)?; - remote_result.write(unsafe { remote.assume_init() }.into()); - Ok(len) - }) - } - SyscallFunction::GetSocketOption => { - let socket_fd = RawFd::from(args[0] as u32); - let option = arg_user_mut::(args[1] as usize)?; - - run_with_io(&process, |io| { - let file = io.files.file(socket_fd)?; - let socket = file.as_socket()?; - socket.get_option(option)?; - Ok(0) - }) - } - SyscallFunction::SetSocketOption => { - let socket_fd = RawFd::from(args[0] as u32); - let option = arg_user_ref::(args[1] as usize)?; - - run_with_io(&process, |io| { - let file = io.files.file(socket_fd)?; - let socket = file.as_socket()?; - socket.set_option(option)?; - Ok(0) - }) - } - SyscallFunction::Accept => { - let socket_fd = RawFd::from(args[0] as u32); - let remote_result = - arg_user_mut::>(args[1] as usize)?; - - run_with_io(&process, |mut io| { - let file = io.files.file(socket_fd)?; - let mut remote = MaybeUninit::uninit(); - let accepted_file = file.accept(&mut remote)?; - let accepted_fd = io.files.place_file(accepted_file, true)?; - unsafe { - remote_result.write(remote.assume_init().into()); - } - Ok(accepted_fd.0 as usize) - }) - } - - SyscallFunction::Fork => unreachable!(), + Ok(fd) + }) } + + pub(crate) fn create_pipe(ends: &mut [MaybeUninit; 2]) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let (read, write) = File::new_pipe_pair(256); + + let read_fd = io.files.place_file(read, true)?; + let write_fd = io.files.place_file(write, true)?; + + ends[0].write(read_fd); + ends[1].write(write_fd); + + Ok(()) + }) + } + + pub(crate) fn poll_channel_wait( + poll_fd: RawFd, + timeout: &Option, + output: &mut Option<(RawFd, Result<(), Error>)>, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let poll_file = io.files.file(poll_fd)?; + let poll = poll_file.as_poll_channel()?; + + *output = block! { + poll.wait(*timeout).await + }?; + + Ok(()) + }) + } + + pub(crate) fn poll_channel_control( + poll_fd: RawFd, + control: PollControl, + fd: RawFd, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let poll_file = io.files.file(poll_fd)?; + let poll = poll_file.as_poll_channel()?; + + match control { + PollControl::AddFd => { + let polled_file = io.files.file(fd)?.clone(); + poll.add(fd, polled_file); + } + } + + Ok(()) + }) + } + + pub(crate) fn send_message( + fd: RawFd, + message: &SentMessage<'_>, + destination: MessageDestination, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let file = io.files.file(fd)?; + let channel = file.as_message_channel()?; + + match message { + &SentMessage::File(fd) => { + let sent_file = io.files.file(fd)?; + + channel.send_message(MessagePayload::File(sent_file.clone()), destination)?; + } + &SentMessage::Data(data) => { + channel.send_message(MessagePayload::Data(Box::from(data)), destination)?; + } + } + + Ok(()) + }) + } + + pub(crate) fn receive_message( + fd: RawFd, + metadata: &mut MaybeUninit, + buf: &mut [u8], + from: &mut MaybeUninit, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = io.files.file(fd)?; + let channel = file.as_message_channel()?; + + let message = channel.receive_message()?; + + from.write(message.source); + + match &message.payload { + MessagePayload::Data(data) => { + // TODO allow truncated messages? + let len = data.len(); + if buf.len() < len { + return Err(Error::MissingData); + } + + metadata.write(ReceivedMessageMetadata::Data(len)); + buf[..len].copy_from_slice(data); + + Ok(()) + } + MessagePayload::File(file) => { + let fd = io.files.place_file(file.clone(), true)?; + + metadata.write(ReceivedMessageMetadata::File(fd)); + + Ok(()) + } + } + }) + } + + // Network + pub(crate) fn connect_socket( + socket_fd: Option, + remote: &SocketAddr, + ty: SocketType, + local_result: &mut MaybeUninit, + ) -> Result { + assert!(socket_fd.is_none()); + + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let (local, file) = match ty { + SocketType::TcpStream => { + let (local, socket) = TcpSocket::connect((*remote).into())?; + (local, File::from_stream_socket(socket)) + } + _ => return Err(Error::InvalidArgument), + }; + let fd = io.files.place_file(file, true)?; + local_result.write(local.into()); + Ok(fd) + }) + } + + pub(crate) fn bind_socket(listen: &SocketAddr, ty: SocketType) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = match ty { + SocketType::UdpPacket => { + File::from_packet_socket(UdpSocket::bind((*listen).into())?) + } + SocketType::RawPacket => File::from_packet_socket(RawSocket::bind()?), + SocketType::TcpStream => { + File::from_listener_socket(TcpListener::bind((*listen).into())?) + } + }; + let fd = io.files.place_file(file, true)?; + Ok(fd) + }) + } + + pub(crate) fn accept( + socket_fd: RawFd, + remote_result: &mut MaybeUninit, + ) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |mut io| { + let file = io.files.file(socket_fd)?; + let mut remote = MaybeUninit::uninit(); + let accepted_file = file.accept(&mut remote)?; + let accepted_fd = io.files.place_file(accepted_file, true)?; + unsafe { + remote_result.write(remote.assume_init().into()); + } + Ok(accepted_fd) + }) + } + + pub(crate) fn send_to( + socket_fd: RawFd, + buffer: &[u8], + recepient: &Option, + ) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let file = io.files.file(socket_fd)?; + + file.send_to(buffer, recepient.map(Into::into)) + }) + } + + pub(crate) fn receive_from( + socket_fd: RawFd, + buffer: &mut [u8], + remote_result: &mut MaybeUninit, + ) -> Result { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let file = io.files.file(socket_fd)?; + let mut remote = MaybeUninit::uninit(); + let len = file.receive_from(buffer, &mut remote)?; + remote_result.write(unsafe { remote.assume_init() }.into()); + Ok(len) + }) + } + + pub(crate) fn set_socket_option(socket_fd: RawFd, option: &SocketOption) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let file = io.files.file(socket_fd)?; + let socket = file.as_socket()?; + socket.set_option(option) + }) + } + + pub(crate) fn get_socket_option( + socket_fd: RawFd, + option: &mut SocketOption, + ) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process::(); + + run_with_io(&process, |io| { + let file = io.files.file(socket_fd)?; + let socket = file.as_socket()?; + socket.get_option(option) + }) + } + + // Handled outside + pub(crate) fn exit_signal(_frame: &SignalEntryData) -> ! { + unreachable!() + } + + pub(crate) fn fork() -> Result { + unreachable!() + } + + pub(crate) fn execve(_options: &ExecveOptions<'_>) -> Result<(), Error> { + unreachable!() + } +} + +mod generated { + #![allow(unreachable_code)] + + use abi::SyscallFunction; + use abi_lib::SyscallRegister; + + use super::{ + arg, + impls::{self, *}, + }; + + include!(concat!(env!("OUT_DIR"), "/generated_dispatcher.rs")); } /// Entrypoint for system calls that takes raw argument values @@ -829,20 +851,14 @@ pub fn raw_syscall_handler(func: u64, args: &[u64]) -> u64 { let Ok(func) = SyscallFunction::try_from(func as usize) else { todo!("Undefined syscall: {}", func); }; + let args = unsafe { core::mem::transmute(args) }; - // let proc = Process::current(); - // debugln!( - // "{} {:?}: raw_syscall_handler {:?}, {:x?}", - // proc.id(), - // proc.name(), - // func, - // args - // ); - let result = syscall_handler(func, args); + let result = generated::handle_syscall(func, args); - if let &Err(error) = &result { - warnln!("{:?}: {:?}", func, error); - } + let value = match result { + Ok(value) => value, + Err(e) => (-(e as u32 as isize)) as usize, + }; - result.into_syscall_result() as u64 + value as _ } From 8e1c4a2befff33a6e06a25a5df1fcb550f84c567 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 12 Mar 2024 13:47:42 +0200 Subject: [PATCH 210/211] refactor: remove old comments --- src/arch/aarch64/mod.rs | 1 - src/arch/x86_64/syscall.rs | 1 - src/main.rs | 3 +-- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index e358a2fb..5144d879 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -93,7 +93,6 @@ impl Platform for AArch64 { } unsafe fn reset(&self) -> ! { - loop {} if let Some(reset) = self.reset.try_get() { reset.reset() } else { diff --git a/src/arch/x86_64/syscall.rs b/src/arch/x86_64/syscall.rs index 7685e33e..b1db7314 100644 --- a/src/arch/x86_64/syscall.rs +++ b/src/arch/x86_64/syscall.rs @@ -23,7 +23,6 @@ fn syscall_inner(frame: &mut SyscallFrame) { return; } } - // FIXME: Fork is temporarily disabled if frame.rax == usize::from(SyscallFunction::Fork) as u64 { unsafe { ProcessImpl::raw_fork(frame); diff --git a/src/main.rs b/src/main.rs index df05203a..23818470 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,8 +25,7 @@ inline_const, maybe_uninit_uninit_array, const_maybe_uninit_uninit_array, - never_type, - trace_macros + never_type )] #![allow( clippy::new_without_default, From 7f1f6b73377367db17f98a740316b904c37ce3b1 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 12 Mar 2024 14:46:54 +0200 Subject: [PATCH 211/211] refactor: wrap IDs in proper wrappers --- driver/net/core/src/config.rs | 12 +++++----- lib/vfs/src/channel.rs | 9 +++++--- lib/vfs/src/pty.rs | 4 ++-- libk/libk-thread/src/lib.rs | 8 +++---- libk/libk-thread/src/process.rs | 11 ++++----- libk/libk-thread/src/thread.rs | 4 ++-- libk/libk-thread/src/types.rs | 40 +++++++++------------------------ src/device/display/linear_fb.rs | 4 ++-- src/device/serial/pl011.rs | 3 +-- src/device/tty.rs | 7 ++---- src/syscall/mod.rs | 30 ++++++++++++------------- src/task/process.rs | 8 +++---- 12 files changed, 59 insertions(+), 81 deletions(-) diff --git a/driver/net/core/src/config.rs b/driver/net/core/src/config.rs index 8865d547..f5e66a59 100644 --- a/driver/net/core/src/config.rs +++ b/driver/net/core/src/config.rs @@ -3,7 +3,7 @@ use serde::Serialize; use vfs::{ChannelDescriptor, MessagePayload}; use yggdrasil_abi::{ error::Error, - io::MessageDestination, + io::{ChannelPublisherId, MessageDestination}, net::{ netconfig::{ InterfaceInfo, InterfaceQuery, NetConfigRequest, NetConfigResult, RouteInfo, @@ -18,7 +18,9 @@ use crate::{ l3::{arp, Route}, }; -async fn receive_request(channel: &ChannelDescriptor) -> Result<(u32, NetConfigRequest), Error> { +async fn receive_request( + channel: &ChannelDescriptor, +) -> Result<(ChannelPublisherId, NetConfigRequest), Error> { loop { let raw = channel.receive_message_async().await?; match &raw.payload { @@ -35,13 +37,13 @@ async fn receive_request(channel: &ChannelDescriptor) -> Result<(u32, NetConfigR fn send_reply( channel: &ChannelDescriptor, - recepient: u32, + recepient: ChannelPublisherId, message: NetConfigResult, ) -> Result<(), Error> { let data = serde_json::to_vec(&message).map_err(|_| Error::InvalidArgument)?; channel.send_message( MessagePayload::Data(data.into_boxed_slice()), - MessageDestination::Specific(recepient), + MessageDestination::Specific(recepient.into()), ) } @@ -81,7 +83,7 @@ fn query_route(destination: IpAddr) -> Option { Some(RoutingInfo { interface_name: interface.name.clone(), - interface_id: interface_id, + interface_id, destination, gateway, source, diff --git a/lib/vfs/src/channel.rs b/lib/vfs/src/channel.rs index 28b19f49..0feb212d 100644 --- a/lib/vfs/src/channel.rs +++ b/lib/vfs/src/channel.rs @@ -13,7 +13,10 @@ use alloc::{ use futures_util::{task::AtomicWaker, Future}; use libk_thread::{block, sync::Mutex}; use libk_util::sync::{IrqSafeSpinlock, LockMethod}; -use yggdrasil_abi::{error::Error, io::MessageDestination}; +use yggdrasil_abi::{ + error::Error, + io::{ChannelPublisherId, MessageDestination}, +}; use crate::{FileReadiness, FileRef}; @@ -34,7 +37,7 @@ pub enum MessagePayload { /// Describes a message sent over a channel pub struct Message { /// Channel descriptor ID from which the message came - pub source: u32, + pub source: ChannelPublisherId, /// Data of the message pub payload: MessagePayload, } @@ -89,7 +92,7 @@ impl ChannelDescriptor { dst: MessageDestination, ) -> Result<(), Error> { let message = Arc::new(Message { - source: self.id, + source: unsafe { ChannelPublisherId::from_raw(self.id) }, payload, }); diff --git a/lib/vfs/src/pty.rs b/lib/vfs/src/pty.rs index 8f3dd662..597360a5 100644 --- a/lib/vfs/src/pty.rs +++ b/lib/vfs/src/pty.rs @@ -19,7 +19,7 @@ use yggdrasil_abi::{ DeviceRequest, TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions, TerminalSize, }, - process::Signal, + process::{ProcessId, Signal}, }; const CAPACITY: usize = 8192; @@ -38,7 +38,7 @@ struct PtyMasterToSlaveHalf { // Actual data to be read by the slave buffer: IrqSafeSpinlock, ready_ring: LossyRingQueue, - signal_pgroup: IrqSafeRwLock>, + signal_pgroup: IrqSafeRwLock>, } /// Pseudo-terminal shared device diff --git a/libk/libk-thread/src/lib.rs b/libk/libk-thread/src/lib.rs index 6360d83e..fe3c5503 100644 --- a/libk/libk-thread/src/lib.rs +++ b/libk/libk-thread/src/lib.rs @@ -16,10 +16,10 @@ use kernel_arch::{Architecture, ArchitectureImpl, KernelTableManagerImpl}; use libk_mm::phys::GlobalPhysicalAllocator; pub(crate) mod api { - use yggdrasil_abi::process::Signal; + use yggdrasil_abi::process::{ProcessId, Signal}; extern "Rust" { - pub fn __signal_process_group(group_id: u32, signal: Signal); + pub fn __signal_process_group(group_id: ProcessId, signal: Signal); } } @@ -41,7 +41,7 @@ pub type TaskContextImpl = use sched::CpuQueue; pub use types::{AtomicThreadState, ThreadAffinity, ThreadId, ThreadState}; -use yggdrasil_abi::process::Signal; +use yggdrasil_abi::process::{ProcessId, Signal}; /// Returns local CPU index #[inline] @@ -53,6 +53,6 @@ pub fn cpu_count() -> usize { ArchitectureImpl::cpu_count() } -pub fn signal_process_group(group_id: u32, signal: Signal) { +pub fn signal_process_group(group_id: ProcessId, signal: Signal) { unsafe { __signal_process_group(group_id, signal) } } diff --git a/libk/libk-thread/src/process.rs b/libk/libk-thread/src/process.rs index fab3a36f..4b041253 100644 --- a/libk/libk-thread/src/process.rs +++ b/libk/libk-thread/src/process.rs @@ -19,13 +19,13 @@ use libk_util::{ }; use yggdrasil_abi::{ error::Error, - process::{ExitCode, Signal, ThreadSpawnOptions}, + process::{ExitCode, ProcessId, Signal, ThreadSpawnOptions}, }; use crate::{ futex::UserspaceMutex, thread::Thread, - types::{ProcessId, ProcessTlsInfo}, + types::{AllocateProcessId, ProcessTlsInfo}, TaskContextImpl, ThreadId, }; @@ -146,7 +146,7 @@ impl, IO: ProcessIo> ProcessImpl { image: Option, ) -> (Arc, Arc) { let name = name.into(); - let id = ProcessId::next(); + let id = ProcessId::new(); let process = Arc::new(Self { name, @@ -248,10 +248,7 @@ impl, IO: ProcessIo> ProcessImpl { let src_thread = Thread::current(); let src_process = src_thread.process::(); - let value = src_process - .fork_inner(frame) - .map(|ProcessId(p)| p as u32) - .into_syscall_register(); + let value = src_process.fork_inner(frame).into_syscall_register(); frame.set_return_value(value as _); } diff --git a/libk/libk-thread/src/thread.rs b/libk/libk-thread/src/thread.rs index 8157386c..8d259b87 100644 --- a/libk/libk-thread/src/thread.rs +++ b/libk/libk-thread/src/thread.rs @@ -18,14 +18,14 @@ use libk_util::{ }; use yggdrasil_abi::{ error::Error, - process::{ExitCode, Signal, SignalEntryData}, + process::{ExitCode, ProcessId, Signal, SignalEntryData}, }; use crate::{ mem::ForeignPointer, process::{Process, ProcessManager}, sched::CpuQueue, - types::{ProcessId, ThreadAffinity, ThreadId, ThreadState}, + types::{ThreadAffinity, ThreadId, ThreadState}, TaskContextImpl, }; diff --git a/libk/libk-thread/src/types.rs b/libk/libk-thread/src/types.rs index 1cc3c2cb..f07d62e5 100644 --- a/libk/libk-thread/src/types.rs +++ b/libk/libk-thread/src/types.rs @@ -1,10 +1,15 @@ use core::{ fmt, mem::size_of, - sync::atomic::{AtomicU64, Ordering}, + sync::atomic::{AtomicU32, AtomicU64, Ordering}, }; use atomic_enum::atomic_enum; +use yggdrasil_abi::process::ProcessId; + +pub trait AllocateProcessId { + fn new() -> Self; +} /// Represents the states a thread can be at some point in time #[atomic_enum] @@ -34,11 +39,6 @@ pub enum ThreadId { User(u64), } -/// Unique number assigned to each [Process] -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -#[repr(transparent)] -pub struct ProcessId(pub u64); - // TLS layout (x86-64): // | mem_size | uthread_size | // | Data .......| self, ??? | @@ -145,31 +145,11 @@ impl fmt::Display for ThreadId { } } -impl fmt::Display for ProcessId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "", self.0) - } -} - -// XXX TODO Remove this -impl From for u32 { - fn from(value: ProcessId) -> Self { - value.0 as _ - } -} - -impl From for ProcessId { - fn from(value: u32) -> Self { - Self(value as _) - } -} - -impl ProcessId { - /// Generates a new [ProcessId] - pub fn next() -> Self { - static COUNTER: AtomicU64 = AtomicU64::new(1); +impl AllocateProcessId for ProcessId { + fn new() -> Self { + static COUNTER: AtomicU32 = AtomicU32::new(1); let id = COUNTER.fetch_add(1, Ordering::SeqCst); - Self(id) + unsafe { Self::from_raw(id) } } } diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index f72b51eb..6cb50184 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -5,7 +5,7 @@ use core::{ task::{Context, Poll}, }; -use abi::{error::Error, io::DeviceRequest}; +use abi::{error::Error, io::DeviceRequest, process::ProcessId}; use device_api::Device; use libk_mm::{ address::{IntoRaw, PhysicalAddress}, @@ -17,7 +17,7 @@ use libk_thread::thread::Thread; use libk_util::sync::IrqSafeSpinlock; use ygg_driver_block::BlockDevice; -use crate::{arch::L3, task::process::ProcessId}; +use crate::arch::L3; use super::{DisplayDevice, DisplayDimensions}; diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 061d77b3..08f830cd 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -1,5 +1,5 @@ //! ARM PL011 driver -use abi::{error::Error, io::DeviceRequest}; +use abi::{error::Error, io::DeviceRequest, process::ProcessId}; use alloc::boxed::Box; use device_api::{ interrupt::{InterruptHandler, Irq}, @@ -25,7 +25,6 @@ use vfs::{CharDevice, FileReadiness}; use crate::{ debug::{self, DebugSink, LogLevel}, device::tty::{TtyContext, TtyDevice}, - task::process::ProcessId, }; register_bitfields! { diff --git a/src/device/tty.rs b/src/device/tty.rs index bb279942..c213adc3 100644 --- a/src/device/tty.rs +++ b/src/device/tty.rs @@ -8,17 +8,14 @@ use core::{ use abi::{ error::Error, io::{TerminalInputOptions, TerminalLineOptions, TerminalOptions, TerminalOutputOptions}, - process::Signal, + process::{ProcessId, Signal}, }; use device_api::serial::SerialDevice; use futures_util::Future; use libk_thread::process::ProcessImpl; use libk_util::{ring::RingBuffer, sync::IrqSafeSpinlock, waker::QueueWaker}; -use crate::{ - proc::io::ProcessIoImpl, - task::process::{ProcessId, ProcessManagerImpl}, -}; +use crate::{proc::io::ProcessIoImpl, task::process::ProcessManagerImpl}; struct TerminalRing { buffer: IrqSafeSpinlock>, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index 592a0000..4ba2eb5c 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -39,7 +39,6 @@ fn run_with_io_at< } mod impls { - use abi::process::{ExecveOptions, SpawnOption}; pub(crate) use abi::{ error::Error, io::{ @@ -53,6 +52,10 @@ mod impls { process::{ExitCode, MutexOperation, Signal, SignalEntryData, SpawnOptions}, system::SystemInfo, }; + use abi::{ + io::ChannelPublisherId, + process::{ExecveOptions, ProcessId, SpawnOption}, + }; use alloc::{boxed::Box, sync::Arc}; use libk::{block, runtime}; use vfs::{File, IoContext, MessagePayload, Read, Seek, Write}; @@ -71,7 +74,6 @@ mod impls { use libk_thread::{ process::{Process, ProcessManager}, thread::Thread, - types::ProcessId, }; use crate::{ @@ -178,7 +180,7 @@ mod impls { thread.exit_process::(code) } - pub(crate) fn spawn_process(options: &SpawnOptions<'_>) -> Result { + pub(crate) fn spawn_process(options: &SpawnOptions<'_>) -> Result { let thread = Thread::current(); let process = thread.process::(); @@ -191,7 +193,7 @@ mod impls { options.arguments, options.environment, )?; - let pid: u32 = child_process.id().into(); + let pid = child_process.id(); // Inherit group and session from the creator child_process.inherit(&process)?; @@ -210,10 +212,10 @@ mod impls { } } &SpawnOption::SetProcessGroup(pgroup) => { - let pgroup = if pgroup == 0 { + let pgroup = if pgroup.into_raw() == 0 { child_process.id() } else { - pgroup.into() + pgroup }; child_process.set_group_id(pgroup); } @@ -247,17 +249,16 @@ mod impls { }) } - pub(crate) fn wait_process(pid: u32, status: &mut ExitCode) -> Result<(), Error> { - let pid = ProcessId::from(pid); + pub(crate) fn wait_process(pid: ProcessId, status: &mut ExitCode) -> Result<(), Error> { let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; *status = block!(target.wait_for_exit().await)?; Ok(()) } - pub(crate) fn get_pid() -> u32 { + pub(crate) fn get_pid() -> ProcessId { let thread = Thread::current(); let process = thread.process::(); - process.id().into() + process.id() } pub(crate) fn nanosleep(duration: &Duration) { @@ -272,8 +273,7 @@ mod impls { thread.set_signal_entry(ip, sp); } - pub(crate) fn send_signal(target: u32, signal: Signal) -> Result<(), Error> { - let pid = ProcessId(target as _); + pub(crate) fn send_signal(pid: ProcessId, signal: Signal) -> Result<(), Error> { let target = ProcessManagerImpl::get(pid).ok_or(Error::DoesNotExist)?; target.raise_signal(signal); Ok(()) @@ -660,7 +660,7 @@ mod impls { fd: RawFd, metadata: &mut MaybeUninit, buf: &mut [u8], - from: &mut MaybeUninit, + from: &mut MaybeUninit, ) -> Result<(), Error> { let thread = Thread::current(); let process = thread.process::(); @@ -823,7 +823,7 @@ mod impls { unreachable!() } - pub(crate) fn fork() -> Result { + pub(crate) fn fork() -> Result { unreachable!() } @@ -835,7 +835,7 @@ mod impls { mod generated { #![allow(unreachable_code)] - use abi::SyscallFunction; + use abi::{io::ChannelPublisherId, process::ProcessId, SyscallFunction}; use abi_lib::SyscallRegister; use super::{ diff --git a/src/task/process.rs b/src/task/process.rs index e9015583..d069fe5b 100644 --- a/src/task/process.rs +++ b/src/task/process.rs @@ -1,6 +1,6 @@ //! Process data structures -use abi::process::Signal; +use abi::process::{ProcessId, Signal}; use alloc::{collections::BTreeMap, sync::Arc}; use libk_thread::process::{Process, ProcessManager}; use libk_util::sync::spin_rwlock::IrqSafeRwLock; @@ -9,7 +9,7 @@ use crate::proc::io::ProcessIoImpl; pub use libk_thread::{ process::ProcessImage, - types::{ProcessId, ProcessTlsInfo, ProcessTlsLayout}, + types::{ProcessTlsInfo, ProcessTlsLayout}, }; /// Process manager implementation @@ -40,6 +40,6 @@ impl ProcessManager for ProcessManagerImpl { } #[no_mangle] -fn __signal_process_group(group_id: u32, signal: Signal) { - ProcessImpl::signal_group(ProcessId(group_id as _), signal) +fn __signal_process_group(group_id: ProcessId, signal: Signal) { + ProcessImpl::signal_group(group_id, signal) }