pci: better capability handling (todo: split/merge into master

separately)

.gitignore

@@ -3,3 +3,6 @@
 /xtask.toml
 /qemu.toml
 /etc/boot/yboot.cfg
+/disk-*.img
+/tmp-*.txt
+/*.log

Cargo.lock

@@ -1145,6 +1145,7 @@ dependencies = [
 name = "kernel-arch-hosted"
 version = "0.1.0"
 dependencies = [
+ "device-api",
  "kernel-arch-interface",
  "libk-mm-interface",
  "yggdrasil-abi",
@@ -1291,6 +1292,7 @@ dependencies = [
 name = "libk-mm"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
  "kernel-arch",
  "libk-mm-interface",
  "libk-util",
@@ -1356,6 +1358,10 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "libutil"
+version = "0.1.0"
+
 [[package]]
 name = "libyalloc"
 version = "0.1.0"
@@ -2702,6 +2708,24 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
+[[package]]
+name = "ygg_driver_intel_hda"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytemuck",
+ "device-api",
+ "futures-util",
+ "libk",
+ "libk-mm",
+ "libk-util",
+ "log",
+ "tock-registers",
+ "ygg_driver_pci",
+ "ygg_driver_sound_core",
+ "yggdrasil-abi",
+]
+
 [[package]]
 name = "ygg_driver_net_core"
 version = "0.1.0"
@@ -2830,6 +2854,20 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
+[[package]]
+name = "ygg_driver_sound_core"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "device-api",
+ "futures-util",
+ "libk",
+ "libk-mm",
+ "libk-util",
+ "log",
+ "yggdrasil-abi",
+]
+
 [[package]]
 name = "ygg_driver_usb"
 version = "0.1.0"
@@ -2867,12 +2905,30 @@ dependencies = [
  "yggdrasil-abi",
 ]
 
+[[package]]
+name = "ygg_driver_virtio_blk"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "bytemuck",
+ "device-api",
+ "libk",
+ "libk-mm",
+ "libk-util",
+ "log",
+ "tock-registers",
+ "ygg_driver_pci",
+ "ygg_driver_virtio_core",
+ "yggdrasil-abi",
+]
+
 [[package]]
 name = "ygg_driver_virtio_core"
 version = "0.1.0"
 dependencies = [
  "bitflags 2.8.0",
  "device-api",
+ "kernel-arch-hosted",
  "libk",
  "libk-mm",
  "libk-util",
@@ -2886,6 +2942,7 @@ dependencies = [
 name = "ygg_driver_virtio_gpu"
 version = "0.1.0"
 dependencies = [
+ "async-trait",
  "bytemuck",
  "device-api",
  "libk",
@@ -2904,6 +2961,7 @@ dependencies = [
  "bitflags 2.8.0",
  "bytemuck",
  "device-api",
+ "futures-util",
  "libk",
  "libk-mm",
  "libk-util",
@@ -2974,6 +3032,7 @@ dependencies = [
  "ygg_driver_ahci",
  "ygg_driver_fat32",
  "ygg_driver_input",
+ "ygg_driver_intel_hda",
  "ygg_driver_net_core",
  "ygg_driver_net_igbe",
  "ygg_driver_net_loopback",
@@ -2981,8 +3040,10 @@ dependencies = [
  "ygg_driver_net_stmmac",
  "ygg_driver_nvme",
  "ygg_driver_pci",
+ "ygg_driver_sound_core",
  "ygg_driver_usb",
  "ygg_driver_usb_xhci",
+ "ygg_driver_virtio_blk",
  "ygg_driver_virtio_gpu",
  "ygg_driver_virtio_net",
  "yggdrasil-abi",

Cargo.toml

@@ -17,7 +17,8 @@ members = [
     "lib/libyalloc",
     "lib/runtime",
     "lib/qemu",
-    "lib/abi-serde"
+    "lib/abi-serde",
+    "lib/libutil"
 ]
 
 [workspace.dependencies]

LICENSE.txt

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 Mark Poliakov <mark@alnyan.me>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the " Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

etc/aarch64-unknown-none.json

@@ -3,7 +3,7 @@
   "os": "none",
   "abi": "softfloat",
   "llvm-target": "aarch64-unknown-none",
-  "data-layout": "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
+  "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32",
   "max-atomic-width": 128,
   "target-pointer-width": "64",
   "features": "+v8a,+strict-align,-neon,-fp-armv8",

etc/riscv64-unknown-none.json

@@ -14,7 +14,7 @@
   "panic-strategy": "abort",
   "dynamic-linking": true,
   "relocation-model": "pic",
-  "code-model": "large",
+  "code-model": "medium",
   "eh-frame-header": false,
 
   "crt-objects-fallback": "false",

etc/x86_64-unknown-none.json

@@ -3,11 +3,12 @@
   "cpu": "x86-64",
   "os": "none",
   "abi": "softfloat",
+  "rustc-abi": "x86-softfloat",
   "llvm-target": "x86_64-unknown-linux-gnu",
   "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128",
   "max-atomic-width": 64,
   "target-pointer-width": "64",
-  "features": "-avx,-sse,+soft-float",
+  "features": "-avx,-sse,-avx2,+soft-float",
 
   "disable-redzone": true,
   "executables": true,

kernel/Cargo.toml

@@ -30,11 +30,13 @@ ygg_driver_net_core = { path = "driver/net/core" }
 ygg_driver_net_loopback = { path = "driver/net/loopback" }
 ygg_driver_virtio_net = { path = "driver/virtio/net", features = ["pci"] }
 ygg_driver_virtio_gpu = { path = "driver/virtio/gpu", features = ["pci"] }
+ygg_driver_virtio_blk = { path = "driver/virtio/blk", features = ["pci"] }
 ygg_driver_nvme = { path = "driver/block/nvme" }
 ygg_driver_ahci = { path = "driver/block/ahci" }
 ygg_driver_input = { path = "driver/input" }
 ygg_driver_usb_xhci.path = "driver/usb/xhci"
 ygg_driver_net_rtl81xx.path = "driver/net/rtl81xx"
+ygg_driver_sound_core.path = "driver/sound/core"
 
 memfs = { path = "driver/fs/memfs" }
 ext2 = { path = "driver/fs/ext2" }
@@ -69,6 +71,7 @@ kernel-arch-x86.workspace = true
 
 ygg_driver_acpi.path = "driver/acpi"
 ygg_driver_net_igbe.path = "driver/net/igbe"
+ygg_driver_intel_hda.path = "driver/sound/intel-hda"
 
 acpi.workspace = true
 

kernel/arch/aarch64/src/lib.rs

@@ -1,5 +1,5 @@
 #![no_std]
-#![feature(naked_functions, trait_upcasting, decl_macro)]
+#![feature(naked_functions, decl_macro)]
 #![allow(clippy::new_without_default)]
 
 extern crate alloc;

kernel/arch/aarch64/src/mem/process.rs

@@ -7,7 +7,7 @@ use core::{
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::PhysicalRefMut,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
@@ -79,7 +79,15 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         )
     }
 
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error> {
+        self.update_l3_entry(address, |entry| entry.update(update))
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> {
         self.pop_l3_entry(address)
     }
 
@@ -120,7 +128,32 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         Ok(())
     }
 
-    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+    fn update_l3_entry<F: FnOnce(&mut PageEntry<L3>) -> Result<(), Error>>(
+        &mut self,
+        virt: usize,
+        mapper: F,
+    ) -> Result<(), Error> {
+        let l1i = virt.page_index::<L1>();
+        let l2i = virt.page_index::<L2>();
+        let l3i = virt.page_index::<L3>();
+
+        let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let entry = &mut l3[l3i];
+        if !entry.is_present() {
+            return Err(Error::DoesNotExist);
+        }
+
+        mapper(entry)?;
+        ic_iallu();
+        dc_cvac((&raw const l3[l3i]).addr());
+        tlb_flush_vaae1(virt);
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> {
         let l1i = virt.page_index::<L1>();
         let l2i = virt.page_index::<L2>();
         let l3i = virt.page_index::<L3>();
@@ -129,14 +162,16 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
         let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
 
-        let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+        let entry = l3[l3i];
+        let page = entry.as_page().ok_or(Error::DoesNotExist)?;
+        let dirty = entry.is_dirty();
 
         l3[l3i] = PageEntry::INVALID;
         ic_iallu();
         dc_cvac((&raw const l3[l3i]).addr());
         tlb_flush_vaae1(virt);
 
-        Ok(page)
+        Ok((page, dirty))
     }
 
     fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {

kernel/arch/aarch64/src/mem/table.rs

@@ -9,6 +9,7 @@ use kernel_arch_interface::KERNEL_VIRT_OFFSET;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
+    process::PageAttributeUpdate,
     table::{
         EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel,
         TableAllocator,
@@ -46,6 +47,7 @@ bitflags! {
 
         const NON_GLOBAL = 1 << 11;
 
+        const DIRTY = 1 << 51;
         const PXN = 1 << 53;
         const UXN = 1 << 54;
     }
@@ -136,8 +138,32 @@ impl<L: EntryLevel> PageTable<L> {
 }
 
 impl<L: EntryLevel> PageEntry<L> {
+    const ATTR_MASK: u64 = 0xFFF | (0xFFFF << 48);
     pub const INVALID: Self = Self(0, PhantomData);
 
+    pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> {
+        let mut attrs = PageAttributes::from_bits_retain(self.0);
+        if let Some(write) = update.user_write {
+            // Make writeable/non-writeable
+            if write {
+                attrs &= !PageAttributes::AP_ACCESS_MASK;
+                attrs |= PageAttributes::AP_BOTH_READWRITE;
+            } else {
+                todo!();
+            }
+        }
+        if let Some(dirty) = update.dirty {
+            if dirty {
+                attrs |= PageAttributes::DIRTY;
+            } else {
+                attrs &= !PageAttributes::DIRTY;
+            }
+        }
+        self.0 &= !Self::ATTR_MASK;
+        self.0 |= attrs.bits() & Self::ATTR_MASK;
+        Ok(())
+    }
+
     pub const fn is_present(self) -> bool {
         self.0 & PageAttributes::PRESENT.bits() != 0
     }
@@ -271,7 +297,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         if self.0 & PageAttributes::PRESENT.bits() != 0
             && self.0 & PageAttributes::BLOCK.bits() == 0
         {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
@@ -283,7 +309,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         } else if let Some(table) = self.as_table() {
             EntryType::Table(table)
         } else {
-            EntryType::Page(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            EntryType::Page(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         }
     }
 }
@@ -316,10 +342,14 @@ impl PageEntry<L3> {
         )
     }
 
+    pub fn is_dirty(&self) -> bool {
+        self.0 & PageAttributes::DIRTY.bits() != 0
+    }
+
     pub fn as_page(&self) -> Option<PhysicalAddress> {
         let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits();
         if self.0 & mask == mask {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
@@ -355,6 +385,10 @@ impl From<MapAttributes> for PageAttributes {
             out |= PageAttributes::AP_KERNEL_READONLY;
         }
 
+        if value.contains(MapAttributes::DIRTY) {
+            out |= PageAttributes::DIRTY;
+        }
+
         if value.contains(MapAttributes::NON_GLOBAL) {
             out |= PageAttributes::NON_GLOBAL;
         }
@@ -377,6 +411,10 @@ impl From<PageAttributes> for MapAttributes {
             _ => unreachable!(),
         };
 
+        if value.contains(PageAttributes::DIRTY) {
+            out |= MapAttributes::DIRTY;
+        }
+
         if value.contains(PageAttributes::NON_GLOBAL) {
             out |= MapAttributes::NON_GLOBAL;
         }

kernel/arch/hosted/Cargo.toml

@@ -7,3 +7,4 @@ edition = "2021"
 kernel-arch-interface.workspace = true
 yggdrasil-abi.workspace = true
 libk-mm-interface.workspace = true
+device-api.workspace = true

kernel/arch/hosted/src/lib.rs

@@ -1,9 +1,11 @@
-#![feature(never_type)]
+#![feature(never_type, allocator_api, slice_ptr_get)]
 use std::{
+    alloc::{Allocator, Global, Layout},
     marker::PhantomData,
     sync::atomic::{AtomicBool, Ordering},
 };
 
+use device_api::dma::{DmaAllocation, DmaAllocator};
 use kernel_arch_interface::{
     cpu::{CpuData, IpiQueue},
     mem::{
@@ -105,6 +107,14 @@ impl Architecture for ArchitectureImpl {
     fn ipi_queue(_cpu_id: u32) -> Option<&'static IpiQueue<Self>> {
         None
     }
+
+    fn load_barrier() {}
+
+    fn store_barrier() {}
+
+    fn memory_barrier() {}
+
+    fn flush_virtual_range(_range: std::ops::Range<usize>) {}
 }
 
 impl KernelTableManager for KernelTableManagerImpl {
@@ -150,7 +160,7 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         unimplemented!()
     }
 
-    unsafe fn unmap_page(&mut self, _address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn unmap_page(&mut self, _address: usize) -> Result<(PhysicalAddress, bool), Error> {
         unimplemented!()
     }
 
@@ -202,3 +212,19 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator> TaskContext<K, PA>
 extern "Rust" fn __signal_process_group(_group_id: ProcessGroupId, _signal: Signal) {
     unimplemented!()
 }
+
+pub struct HostedDmaAllocator;
+
+impl DmaAllocator for HostedDmaAllocator {
+    fn allocate(&self, layout: Layout) -> Result<DmaAllocation, Error> {
+        let ptr = Global.allocate(layout.align_to(0x1000).unwrap()).unwrap();
+        let base = ptr.as_non_null_ptr();
+        let addr: usize = base.addr().into();
+        Ok(DmaAllocation {
+            host_virtual: base.cast(),
+            host_physical: addr as _,
+            page_count: layout.size().div_ceil(0x1000),
+            bus_address: addr as _,
+        })
+    }
+}

kernel/arch/riscv64/src/mem/process.rs

@@ -6,7 +6,7 @@ use core::{
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::PhysicalRefMut,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
@@ -70,7 +70,15 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         Ok(())
     }
 
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error> {
+        self.update_l3_entry(address, |entry| entry.update(update))
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> {
         self.pop_l3_entry(address)
     }
 
@@ -118,7 +126,11 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         Ok(())
     }
 
-    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+    fn update_l3_entry<F: FnOnce(&mut PageEntry<L3>) -> Result<(), Error>>(
+        &mut self,
+        virt: usize,
+        mapper: F,
+    ) -> Result<(), Error> {
         let l1i = virt.page_index::<L1>();
         let l2i = virt.page_index::<L2>();
         let l3i = virt.page_index::<L3>();
@@ -127,12 +139,33 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
         let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
 
-        let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+        let entry = &mut l3[l3i];
+        if !entry.is_present() {
+            return Err(Error::DoesNotExist);
+        }
+        mapper(entry)?;
+        super::tlb_flush_va_asid(virt, self.asid as usize);
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> {
+        let l1i = virt.page_index::<L1>();
+        let l2i = virt.page_index::<L2>();
+        let l3i = virt.page_index::<L3>();
+
+        // TODO somehow drop tables if they're known to be empty?
+        let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let entry = l3[l3i];
+        let page = entry.as_page().ok_or(Error::DoesNotExist)?;
+        let dirty = entry.is_dirty();
 
         l3[l3i] = PageEntry::INVALID;
         super::tlb_flush_va_asid(virt, self.asid as usize);
 
-        Ok(page)
+        Ok((page, dirty))
     }
 
     fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {
@@ -178,6 +211,9 @@ fn to_page_attributes(src: MapAttributes) -> PageAttributes {
     if src.intersects(MapAttributes::USER_READ | MapAttributes::USER_WRITE) {
         result |= PageAttributes::U;
     }
+    if src.contains(MapAttributes::DIRTY) {
+        result |= PageAttributes::SW_DIRTY;
+    }
     result
 }
 
@@ -192,5 +228,9 @@ fn to_map_attributes(src: PageAttributes) -> MapAttributes {
         }
     }
 
+    if src.contains(PageAttributes::SW_DIRTY) {
+        result |= MapAttributes::DIRTY;
+    }
+
     result
 }

kernel/arch/riscv64/src/mem/table.rs

@@ -6,6 +6,7 @@ use core::{
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
+    process::PageAttributeUpdate,
     table::{
         page_index, EntryLevel, EntryLevelDrop, NextPageTable, NonTerminalEntryLevel,
         TableAllocator,
@@ -88,6 +89,8 @@ impl<L: EntryLevel> PageTable<L> {
 }
 
 impl<L: EntryLevel> PageEntry<L> {
+    // Upper + lower 10 bits
+    const ATTR_MASK: u64 = 0xFFC00000000003FF;
     pub const INVALID: Self = Self(0, PhantomData);
 
     /// Constructs a [PageEntry] from its raw representation.
@@ -103,6 +106,23 @@ impl<L: EntryLevel> PageEntry<L> {
         self.0 & PageAttributes::V.bits() != 0
     }
 
+    pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> {
+        let mut attrs = self.attributes();
+        if let Some(write) = update.user_write {
+            attrs.set(PageAttributes::W, write);
+        }
+        if let Some(dirty) = update.dirty {
+            attrs.set(PageAttributes::SW_DIRTY, dirty);
+        }
+        self.0 &= !Self::ATTR_MASK;
+        self.0 |= attrs.bits() & Self::ATTR_MASK;
+        Ok(())
+    }
+
+    pub const fn is_dirty(&self) -> bool {
+        self.0 & PageAttributes::SW_DIRTY.bits() != 0
+    }
+
     pub fn attributes(self) -> PageAttributes {
         PageAttributes::from_bits_retain(self.0)
     }
@@ -211,7 +231,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
             & (PageAttributes::R | PageAttributes::W | PageAttributes::X | PageAttributes::V)
                 .bits()
             == PageAttributes::V.bits())
-        .then_some((self.0 << 2) & !0xFFF)
+        .then_some((self.0 & !Self::ATTR_MASK) << 2)
         .map(PhysicalAddress::from_u64)
     }
 }
@@ -232,7 +252,7 @@ impl PageEntry<L3> {
 
     pub fn as_page(&self) -> Option<PhysicalAddress> {
         (self.0 & PageAttributes::V.bits() != 0)
-            .then_some((self.0 << 2) & !0xFFF)
+            .then_some((self.0 & !Self::ATTR_MASK) << 2)
             .map(PhysicalAddress::from_u64)
     }
 }

kernel/arch/x86_64/src/lib.rs

@@ -1,6 +1,6 @@
 #![no_std]
 #![allow(clippy::new_without_default)]
-#![feature(naked_functions, trait_upcasting)]
+#![feature(naked_functions)]
 
 extern crate alloc;
 

kernel/arch/x86_64/src/mem/process.rs

@@ -4,7 +4,7 @@ use core::marker::PhantomData;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::PhysicalRefMut,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{
         EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator,
     },
@@ -61,7 +61,15 @@ impl<TA: TableAllocator> ProcessAddressSpaceManager<TA> for ProcessAddressSpaceI
         self.write_l3_entry(address, PageEntry::page(physical, flags.into()), false)
     }
 
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error> {
+        self.update_l3_entry(address, |entry| entry.update(update))
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> {
         self.pop_l3_entry(address)
     }
 
@@ -111,7 +119,33 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         Ok(())
     }
 
-    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+    fn update_l3_entry<F: FnOnce(&mut PageEntry<L3>) -> Result<(), Error>>(
+        &mut self,
+        virt: usize,
+        mapper: F,
+    ) -> Result<(), Error> {
+        let l0i = virt.page_index::<L0>();
+        let l1i = virt.page_index::<L1>();
+        let l2i = virt.page_index::<L2>();
+        let l3i = virt.page_index::<L3>();
+
+        let mut l1 = self.l0.get_mut(l0i).ok_or(Error::DoesNotExist)?;
+        let mut l2 = l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let entry = &mut l3[l3i];
+        if !entry.is_present() {
+            return Err(Error::DoesNotExist);
+        }
+        mapper(entry)?;
+        unsafe {
+            flush_tlb_entry(virt);
+        }
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> {
         let l0i = virt.page_index::<L0>();
         let l1i = virt.page_index::<L1>();
         let l2i = virt.page_index::<L2>();
@@ -123,13 +157,14 @@ impl<TA: TableAllocator> ProcessAddressSpaceImpl<TA> {
         let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
 
         let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+        let dirty = l3[l3i].is_dirty();
 
         l3[l3i] = PageEntry::INVALID;
         unsafe {
             flush_tlb_entry(virt);
         }
 
-        Ok(page)
+        Ok((page, dirty))
     }
 
     fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {

kernel/arch/x86_64/src/mem/table.rs

@@ -8,6 +8,7 @@ use bitflags::bitflags;
 use libk_mm_interface::{
     address::{AsPhysicalAddress, PhysicalAddress},
     pointer::{PhysicalRef, PhysicalRefMut},
+    process::PageAttributeUpdate,
     table::{
         EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel,
         TableAllocator,
@@ -31,6 +32,8 @@ bitflags! {
         /// For tables, allows user access to further translation levels, for pages/blocks, allows
         /// user access to the region covered by the entry
         const USER = 1 << 2;
+        /// If set, the page has been written to
+        const DIRTY = 1 << 6;
     }
 }
 
@@ -98,11 +101,15 @@ impl PageEntry<L3> {
     /// not
     pub fn as_page(self) -> Option<PhysicalAddress> {
         if self.0 & PageAttributes::PRESENT.bits() != 0 {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
     }
+
+    pub fn is_dirty(&self) -> bool {
+        self.0 & PageAttributes::DIRTY.bits() != 0
+    }
 }
 
 impl PageEntry<L2> {
@@ -145,7 +152,7 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
         if self.0 & PageAttributes::PRESENT.bits() != 0
             && self.0 & PageAttributes::BLOCK.bits() == 0
         {
-            Some(PhysicalAddress::from_u64(self.0 & !0xFFF))
+            Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK))
         } else {
             None
         }
@@ -158,6 +165,8 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
 }
 
 impl<L: EntryLevel> PageEntry<L> {
+    const ATTR_MASK: u64 = 0xFFF | (1 << 63);
+
     /// An entry that is not mapped
     pub const INVALID: Self = Self(0, PhantomData);
 
@@ -179,6 +188,21 @@ impl<L: EntryLevel> PageEntry<L> {
     pub fn is_present(&self) -> bool {
         self.0 & PageAttributes::PRESENT.bits() != 0
     }
+
+    pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> {
+        let mut attrs = PageAttributes::from_bits_retain(self.0);
+        if let Some(write) = update.user_write {
+            if write {
+                attrs |= PageAttributes::WRITABLE;
+            } else {
+                attrs &= !PageAttributes::WRITABLE;
+            }
+        }
+        // Dirty is ignored, it's hardware-managed
+        self.0 &= !Self::ATTR_MASK;
+        self.0 |= attrs.bits() & Self::ATTR_MASK;
+        Ok(())
+    }
 }
 
 impl<L: EntryLevel> PageTable<L> {

kernel/driver/block/ahci/src/port.rs

@@ -16,7 +16,8 @@ use libk::{
     error::Error,
 };
 use libk_mm::{
-    address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, PageProvider,
+    address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, OnDemandPage,
+    PageProvider, VirtualPage,
 };
 use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker, OneTimeInit};
 use tock_registers::interfaces::{Readable, Writeable};
@@ -365,8 +366,21 @@ impl Device for AhciPort {
 }
 
 impl PageProvider for AhciPort {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        todo!()
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -375,10 +389,6 @@ impl PageProvider for AhciPort {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        todo!()
+        unimplemented!()
     }
 }

kernel/driver/block/nvme/src/drive.rs

@@ -11,7 +11,7 @@ use libk::{
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
     table::MapAttributes,
-    PageProvider, PageSlice,
+    OnDemandPage, PageProvider, PageSlice, VirtualPage,
 };
 
 use crate::{command::IdentifyNamespaceRequest, register_nvme_namespace, IoDirection};
@@ -151,8 +151,21 @@ impl BlockDevice for NvmeNamespace {
 }
 
 impl PageProvider for NvmeNamespace {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        todo!()
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -161,10 +174,6 @@ impl PageProvider for NvmeNamespace {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        todo!()
+        unimplemented!()
     }
 }

kernel/driver/block/scsi/src/lib.rs

@@ -22,7 +22,9 @@ use libk::{
     fs::devfs,
     task::{runtime, sync::AsyncMutex},
 };
-use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider};
+use libk_mm::{
+    address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage,
+};
 use libk_util::{
     sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
     OneTimeInit,
@@ -278,12 +280,21 @@ impl BlockDevice for ScsiUnit {
 }
 
 impl PageProvider for ScsiUnit {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        Err(Error::NotImplemented)
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
     }
 
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        Err(Error::NotImplemented)
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -292,7 +303,7 @@ impl PageProvider for ScsiUnit {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        Err(Error::NotImplemented)
+        unimplemented!()
     }
 }
 

kernel/driver/block/scsi/src/transport.rs

@@ -84,8 +84,7 @@ impl ScsiTransportWrapper {
                 response_buffer.slice_mut(0..R::RESPONSE_LEN),
             )
             .await?;
-        let response_bytes =
-            unsafe { MaybeUninit::slice_assume_init_ref(&response_buffer[..response_len]) };
+        let response_bytes = unsafe { response_buffer[..response_len].assume_init_ref() };
 
         R::parse_response(response_bytes)
     }

kernel/driver/bus/pci/src/capability/express.rs

@@ -0,0 +1,177 @@
+use core::time::Duration;
+
+use libk::{error::Error, task::runtime::psleep};
+use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy};
+
+use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+// bitflags! {
+//     pub struct PcieLinkControl: u16 {
+//         const ASPM_DISABLE = 0 << 0;
+//         // Active state power management control
+//         const ASPM_MASK = 0x3 << 0;
+//         // Enable clock power management
+//         const ECPM = 1 << 8;
+//     }
+// }
+
+register_bitfields! {
+    u32,
+    pub DeviceCapabilities [
+        MAX_PAYLOAD_SIZE            OFFSET(0)   NUMBITS(3)  [],
+        PHANTOM_FUNCTIONS           OFFSET(3)   NUMBITS(2)  [],
+        L0S_ACCEPTABLE_LATENCY      OFFSET(6)   NUMBITS(3)  [],
+        L1_ACCEPTABLE_LATENCY       OFFSET(9)   NUMBITS(3)  [],
+        ROLE_ERROR_REPORTING        OFFSET(15)  NUMBITS(1)  [],
+        CAPTURED_SLOT_POWER_LIMIT   OFFSET(18)  NUMBITS(8)  [],
+        CAPTURED_SLOT_POWER_SCALE   OFFSET(26)  NUMBITS(3)  [],
+        FUNCTION_LEVEL_RESET        OFFSET(28)  NUMBITS(1)  [],
+    ],
+    pub LinkCapabilities [
+        MAX_LINK_SPEED              OFFSET(0)   NUMBITS(4)  [],
+        MAX_LINK_WIDTH              OFFSET(4)   NUMBITS(6)  [],
+        ASPM                        OFFSET(10)  NUMBITS(2)  [],
+        L0S_EXIT_LATENCY            OFFSET(12)  NUMBITS(3)  [],
+        L1_EXIT_LATENCY             OFFSET(15)  NUMBITS(3)  [],
+        CLOCK_PM                    OFFSET(18)  NUMBITS(1)  [],
+        SURPRISE_DOWN_ERROR         OFFSET(19)  NUMBITS(1)  [],
+        DATA_LINK_ACTIVE_REPORTING  OFFSET(20)  NUMBITS(1)  [],
+        LINK_BANDWIDTH_NOTIFICATION OFFSET(21)  NUMBITS(1)  [],
+        ASPM_COMPLIANCE             OFFSET(22)  NUMBITS(1)  [],
+        PORT_NUMBER                 OFFSET(24)  NUMBITS(8)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+    pub DeviceControl [
+        CORRECTABLE_ERROR_REPORTING OFFSET(0)   NUMBITS(1)  [],
+        NONFATAL_ERROR_REPORTING    OFFSET(1)   NUMBITS(1)  [],
+        FATAL_ERROR_REPORTING       OFFSET(2)   NUMBITS(1)  [],
+        UNSUPPORTED_REQ_REPORTING   OFFSET(3)   NUMBITS(1)  [],
+        RELAXED_ORDERING            OFFSET(4)   NUMBITS(1)  [],
+        MAX_PAYLOAD_SIZE            OFFSET(5)   NUMBITS(3)  [],
+        EXTENDED_TAG_FIELD          OFFSET(8)   NUMBITS(1)  [],
+        PHANTOM_FUNCTIONS           OFFSET(9)   NUMBITS(1)  [],
+        AUX_POWER_PM_ENABLE         OFFSET(10)  NUMBITS(1)  [],
+        NO_SNOOP                    OFFSET(11)  NUMBITS(1)  [],
+        MAX_READ_REQ_SIZE           OFFSET(12)  NUMBITS(3)  [],
+        FUNCTION_LEVEL_RESET        OFFSET(15)  NUMBITS(1)  [],
+    ],
+    pub LinkControl [
+        ASPM                        OFFSET(0)   NUMBITS(2)  [
+            Disabled = 0,
+            L0sEntryEnabled = 1,
+            L1EntryEnabled = 2,
+            L0sL1EntryEnabled = 3,
+        ],
+        READ_COMPLETION_BOUNDARY    OFFSET(3)   NUMBITS(1)  [],
+        LINK_DISABLE                OFFSET(4)   NUMBITS(1)  [],
+        RETRAIN_LINK                OFFSET(5)   NUMBITS(1)  [],
+        COMMON_CLOCK_CONFIG         OFFSET(6)   NUMBITS(1)  [],
+        EXTENDED_SYNCH              OFFSET(7)   NUMBITS(1)  [],
+        CLOCK_PM                    OFFSET(8)   NUMBITS(1)  [],
+        HARDWARE_ABW_DISABLE        OFFSET(9)   NUMBITS(1)  [],
+        LINK_BW_MANAGEMENT_IRQ      OFFSET(10)  NUMBITS(1)  [],
+        LINK_ABW_IRQ                OFFSET(11)  NUMBITS(1)  [],
+    ],
+}
+
+/// PCIe capability
+pub struct PciExpressCapability;
+
+/// PCI Express capability data structure
+pub struct PciExpressData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl PciCapability for PciExpressCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PciExpressData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PciExpressData { space, offset }
+    }
+}
+
+macro_rules! reg_read {
+    ($self:expr, $offset:literal, u32) => {
+        $self.space.read_u32($self.offset + $offset)
+    };
+    ($self:expr, $offset:literal, u16) => {
+        $self.space.read_u16($self.offset + $offset)
+    };
+}
+macro_rules! reg_write {
+    ($self:expr, $offset:literal, u32, $value:expr) => {
+        $self.space.write_u32($self.offset + $offset, $value)
+    };
+    ($self:expr, $offset:literal, u16, $value:expr) => {
+        $self.space.write_u16($self.offset + $offset, $value)
+    };
+}
+macro_rules! make_register {
+    (
+        $reg:ident : $ty:ident @ $offset:literal {
+            $get:ident
+            $(, $set:ident, $modify:ident)?
+            $(,)?
+        }
+    ) => {
+        pub fn $get(&self) -> LocalRegisterCopy<$ty, $reg::Register> {
+            LocalRegisterCopy::new(reg_read!(self, $offset, $ty))
+        }
+
+        $(
+            pub fn $set(&mut self, value: $ty) {
+                reg_write!(self, $offset, $ty, value)
+            }
+
+            pub fn $modify(&mut self, field: FieldValue<$ty, $reg::Register>) {
+                let mut value = self.$get();
+                value.modify(field);
+                self.$set(value.get());
+            }
+        )?
+    };
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PciExpressData<'s, S> {
+    make_register!(DeviceCapabilities : u32 @ 0x04 { device_capabilities });
+    make_register!(DeviceControl : u16 @ 0x08 {
+        device_control,
+        set_device_control,
+        modify_device_control,
+    });
+    make_register!(LinkCapabilities : u32 @ 0x0C { link_capabilities });
+    make_register!(LinkControl : u16 @ 0x10 {
+        link_control,
+        set_link_control,
+        modify_link_control,
+    });
+
+    pub fn function_level_reset(&mut self) -> Result<(), Error> {
+        if self
+            .device_capabilities()
+            .matches_all(DeviceCapabilities::FUNCTION_LEVEL_RESET::SET)
+        {
+            self.modify_device_control(DeviceControl::FUNCTION_LEVEL_RESET::SET);
+            psleep(Duration::from_millis(10));
+            Ok(())
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
+    pub fn hot_link_reset(&mut self) {
+        self.modify_link_control(LinkControl::LINK_DISABLE::SET);
+        psleep(Duration::from_millis(10));
+        self.modify_link_control(LinkControl::LINK_DISABLE::CLEAR);
+        psleep(Duration::from_millis(100));
+    }
+}

kernel/driver/bus/pci/src/capability/mod.rs

@@ -0,0 +1,14 @@
+//! PCI capability structures and queries
+
+pub mod express;
+pub mod msi;
+pub mod power;
+pub mod virtio;
+
+pub use express::PciExpressCapability;
+pub use msi::{MsiCapability, MsiXCapability};
+pub use power::PowerManagementCapability;
+pub use virtio::{
+    VirtioCapability, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
+    VirtioInterruptStatusCapability, VirtioNotifyConfigCapability,
+};

kernel/driver/bus/pci/src/capability/msi.rs

@@ -1,91 +1,21 @@
-//! PCI capability structures and queries
-
 use alloc::{sync::Arc, vec, vec::Vec};
-use bitflags::bitflags;
+
 use device_api::interrupt::{
     InterruptAffinity, InterruptHandler, MessageInterruptController, MsiInfo,
 };
+use libk::error::Error;
 use libk_mm::{address::PhysicalAddress, device::DeviceMemoryIoMut};
 use tock_registers::{
     interfaces::{Readable, Writeable},
     registers::{ReadWrite, WriteOnly},
 };
-use yggdrasil_abi::error::Error;
-
-use crate::PciBaseAddress;
-
-use super::{PciCapability, PciCapabilityId, PciConfigurationSpace};
-
-bitflags! {
-    pub struct PcieLinkControl: u16 {
-        const ASPM_DISABLE = 0 << 0;
-        // Active state power management control
-        const ASPM_MASK = 0x3 << 0;
-        // Enable clock power management
-        const ECPM = 1 << 8;
-    }
-}
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
 use core::mem::offset_of;
 #[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
 use kernel_arch_x86::intrinsics;
 
-pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
-    fn from_space_offset(space: &'s S, offset: usize) -> Self;
-
-    fn space(&self) -> &'s S;
-    fn offset(&self) -> usize;
-
-    fn bar_index(&self) -> Option<usize> {
-        let value = self.space().read_u8(self.offset() + 4);
-        (value <= 0x5).then_some(value as _)
-    }
-
-    fn bar_offset(&self) -> usize {
-        let value = self.space().read_u32(self.offset() + 8);
-        value as _
-    }
-
-    fn length(&self) -> usize {
-        let value = self.space().read_u32(self.offset() + 12);
-        value as _
-    }
-}
-
-pub trait VirtioCapability {
-    const CFG_TYPE: u8;
-    const MIN_LEN: usize = 0;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
-}
-
-/// Power management capability entry
-pub struct PowerManagementCapability;
-/// MSI-X capability query
-pub struct MsiXCapability;
-/// MSI capability query
-pub struct MsiCapability;
-/// PCIe capability
-pub struct PciExpressCapability;
-
-// VirtIO-over-PCI capabilities
-/// VirtIO PCI configuration access
-pub struct VirtioDeviceConfigCapability;
-/// VirtIO common configuration
-pub struct VirtioCommonConfigCapability;
-/// VirtIO notify configuration
-pub struct VirtioNotifyConfigCapability;
-/// VirtIO interrupt status
-pub struct VirtioInterruptStatusCapability;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DevicePowerState {
-    D0,
-    D1,
-    D2,
-    D3Cold,
-    D3Hot,
-}
+use crate::{PciBaseAddress, PciCapability, PciCapabilityId, PciConfigurationSpace};
 
 /// Represents an entry in MSI-X vector table
 #[repr(C)]
@@ -109,11 +39,10 @@ pub struct MsiXVectorTable<'a> {
     len: usize,
 }
 
-/// PCI Power Management capability data structure
-pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
+/// MSI-X capability query
+pub struct MsiXCapability;
+/// MSI capability query
+pub struct MsiCapability;
 
 /// MSI-X capability data structure
 pub struct MsiXData<'s, S: PciConfigurationSpace + ?Sized + 's> {
@@ -127,63 +56,6 @@ pub struct MsiData<'s, S: PciConfigurationSpace + ?Sized + 's> {
     offset: usize,
 }
 
-/// PCI Express capability data structure
-pub struct PcieData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> {
-    space: &'s S,
-    offset: usize,
-}
-
-impl<T: VirtioCapability> PciCapability for T {
-    const ID: PciCapabilityId = PciCapabilityId::VendorSpecific;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>;
-
-    fn check<S: PciConfigurationSpace + ?Sized>(space: &S, offset: usize, len: usize) -> bool {
-        let cfg_type = space.read_u8(offset + 3);
-        cfg_type == T::CFG_TYPE && len >= T::MIN_LEN
-    }
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        T::Output::from_space_offset(space, offset)
-    }
-}
-
-impl PciCapability for PowerManagementCapability {
-    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        PowerManagementData { space, offset }
-    }
-}
-
 impl PciCapability for MsiXCapability {
     const ID: PciCapabilityId = PciCapabilityId::MsiX;
     type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = MsiXData<'a, S>;
@@ -210,167 +82,6 @@ impl PciCapability for MsiCapability {
     }
 }
 
-impl PciCapability for PciExpressCapability {
-    const ID: PciCapabilityId = PciCapabilityId::PciExpress;
-    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PcieData<'a, S>;
-
-    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
-        space: &'s S,
-        offset: usize,
-        _len: usize,
-    ) -> Self::CapabilityData<'s, S> {
-        PcieData { space, offset }
-    }
-}
-
-impl VirtioCapability for VirtioDeviceConfigCapability {
-    const CFG_TYPE: u8 = 0x04;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioDeviceConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioCommonConfigCapability {
-    const CFG_TYPE: u8 = 0x01;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioCommonConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioNotifyConfigCapability {
-    const CFG_TYPE: u8 = 0x02;
-    const MIN_LEN: usize = 0x14;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> {
-    pub fn offset_multiplier(&self) -> usize {
-        self.space.read_u32(self.offset + 16) as usize
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioNotifyConfigData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl VirtioCapability for VirtioInterruptStatusCapability {
-    const CFG_TYPE: u8 = 0x03;
-    const MIN_LEN: usize = 1;
-    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>;
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> {
-    pub fn read_status(&self) -> (bool, bool) {
-        todo!()
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
-    for VirtioInterruptStatusData<'s, S>
-{
-    fn from_space_offset(space: &'s S, offset: usize) -> Self {
-        Self { space, offset }
-    }
-
-    fn space(&self) -> &'s S {
-        self.space
-    }
-
-    fn offset(&self) -> usize {
-        self.offset
-    }
-}
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
-    pub fn set_device_power_state(&self, state: DevicePowerState) {
-        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
-        let current = self.get_device_power_state();
-
-        if state == current {
-            return;
-        }
-
-        log::info!("Set device power state: {state:?}");
-
-        match state {
-            DevicePowerState::D0 => {
-                // power = 0b00 | PME_EN
-                self.space.write_u16(self.offset + 4, pmcsr);
-            }
-            _ => {
-                log::warn!("TODO: {state:?} power state");
-            }
-        }
-    }
-
-    pub fn set_pme_en(&self, state: bool) {
-        let pmcsr = self.space.read_u16(self.offset + 4);
-        let new = if state {
-            pmcsr | (1 << 8)
-        } else {
-            pmcsr & !(1 << 8)
-        };
-        if pmcsr == new {
-            return;
-        }
-
-        log::info!("Set PMCSR.PME_En = {state}");
-
-        self.space.write_u16(self.offset + 4, new);
-    }
-
-    pub fn get_device_power_state(&self) -> DevicePowerState {
-        let pmcsr = self.space.read_u16(self.offset + 4);
-        match pmcsr & 0x3 {
-            0b00 => DevicePowerState::D0,
-            0b01 => DevicePowerState::D1,
-            0b10 => DevicePowerState::D2,
-            0b11 => DevicePowerState::D3Hot,
-            _ => unreachable!(),
-        }
-    }
-}
-
 impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiXData<'s, S> {
     // TODO use pending bits as well
     /// Maps and returns the vector table associated with the device's MSI-X capability
@@ -601,13 +312,3 @@ impl<'s, S: PciConfigurationSpace + ?Sized + 's> MsiData<'s, S> {
         Ok(info)
     }
 }
-
-impl<'s, S: PciConfigurationSpace + ?Sized + 's> PcieData<'s, S> {
-    pub fn link_control(&self) -> PcieLinkControl {
-        PcieLinkControl::from_bits_retain(self.space.read_u16(self.offset + 0x10))
-    }
-
-    pub fn set_link_control(&mut self, value: PcieLinkControl) {
-        self.space.write_u16(self.offset + 0x10, value.bits());
-    }
-}

kernel/driver/bus/pci/src/capability/power.rs

@@ -0,0 +1,82 @@
+use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DevicePowerState {
+    D0,
+    D1,
+    D2,
+    D3Cold,
+    D3Hot,
+}
+
+/// Power management capability entry
+pub struct PowerManagementCapability;
+
+/// PCI Power Management capability data structure
+pub struct PowerManagementData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl PciCapability for PowerManagementCapability {
+    const ID: PciCapabilityId = PciCapabilityId::PowerManagement;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = PowerManagementData<'a, S>;
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        PowerManagementData { space, offset }
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> PowerManagementData<'s, S> {
+    pub fn set_device_power_state(&self, state: DevicePowerState) {
+        let pmcsr = self.space.read_u16(self.offset + 4) & !0x3;
+        let current = self.get_device_power_state();
+
+        if state == current {
+            return;
+        }
+
+        log::info!("Set device power state: {state:?}");
+
+        match state {
+            DevicePowerState::D0 => {
+                // power = 0b00 | PME_EN
+                self.space.write_u16(self.offset + 4, pmcsr);
+            }
+            _ => {
+                log::warn!("TODO: {state:?} power state");
+            }
+        }
+    }
+
+    pub fn set_pme_en(&self, state: bool) {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        let new = if state {
+            pmcsr | (1 << 8)
+        } else {
+            pmcsr & !(1 << 8)
+        };
+        if pmcsr == new {
+            return;
+        }
+
+        log::info!("Set PMCSR.PME_En = {state}");
+
+        self.space.write_u16(self.offset + 4, new);
+    }
+
+    pub fn get_device_power_state(&self) -> DevicePowerState {
+        let pmcsr = self.space.read_u16(self.offset + 4);
+        match pmcsr & 0x3 {
+            0b00 => DevicePowerState::D0,
+            0b01 => DevicePowerState::D1,
+            0b10 => DevicePowerState::D2,
+            0b11 => DevicePowerState::D3Hot,
+            _ => unreachable!(),
+        }
+    }
+}

kernel/driver/bus/pci/src/capability/virtio.rs

@@ -0,0 +1,175 @@
+use crate::{PciCapability, PciCapabilityId, PciConfigurationSpace};
+
+pub trait VirtioCapabilityData<'s, S: PciConfigurationSpace + ?Sized + 's>: Sized {
+    fn from_space_offset(space: &'s S, offset: usize) -> Self;
+
+    fn space(&self) -> &'s S;
+    fn offset(&self) -> usize;
+
+    fn bar_index(&self) -> Option<usize> {
+        let value = self.space().read_u8(self.offset() + 4);
+        (value <= 0x5).then_some(value as _)
+    }
+
+    fn bar_offset(&self) -> usize {
+        let value = self.space().read_u32(self.offset() + 8);
+        value as _
+    }
+
+    fn length(&self) -> usize {
+        let value = self.space().read_u32(self.offset() + 12);
+        value as _
+    }
+}
+
+pub trait VirtioCapability {
+    const CFG_TYPE: u8;
+    const MIN_LEN: usize = 0;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a>: VirtioCapabilityData<'a, S>;
+}
+
+// VirtIO-over-PCI capabilities
+/// VirtIO PCI configuration access
+pub struct VirtioDeviceConfigCapability;
+/// VirtIO common configuration
+pub struct VirtioCommonConfigCapability;
+/// VirtIO notify configuration
+pub struct VirtioNotifyConfigCapability;
+/// VirtIO interrupt status
+pub struct VirtioInterruptStatusCapability;
+
+pub struct VirtioDeviceConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioCommonConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioNotifyConfigData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+pub struct VirtioInterruptStatusData<'s, S: PciConfigurationSpace + ?Sized + 's> {
+    space: &'s S,
+    offset: usize,
+}
+
+impl<T: VirtioCapability> PciCapability for T {
+    const ID: PciCapabilityId = PciCapabilityId::VendorSpecific;
+    type CapabilityData<'a, S: PciConfigurationSpace + ?Sized + 'a> = T::Output<'a, S>;
+
+    fn check<S: PciConfigurationSpace + ?Sized>(space: &S, offset: usize, len: usize) -> bool {
+        let cfg_type = space.read_u8(offset + 3);
+        cfg_type == T::CFG_TYPE && len >= T::MIN_LEN
+    }
+
+    fn data<'s, S: PciConfigurationSpace + ?Sized + 's>(
+        space: &'s S,
+        offset: usize,
+        _len: usize,
+    ) -> Self::CapabilityData<'s, S> {
+        T::Output::from_space_offset(space, offset)
+    }
+}
+
+impl VirtioCapability for VirtioDeviceConfigCapability {
+    const CFG_TYPE: u8 = 0x04;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioDeviceConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioDeviceConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioCommonConfigCapability {
+    const CFG_TYPE: u8 = 0x01;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioCommonConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioCommonConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioNotifyConfigCapability {
+    const CFG_TYPE: u8 = 0x02;
+    const MIN_LEN: usize = 0x14;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioNotifyConfigData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioNotifyConfigData<'s, S> {
+    pub fn offset_multiplier(&self) -> usize {
+        self.space.read_u32(self.offset + 16) as usize
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioNotifyConfigData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}
+
+impl VirtioCapability for VirtioInterruptStatusCapability {
+    const CFG_TYPE: u8 = 0x03;
+    const MIN_LEN: usize = 1;
+    type Output<'a, S: PciConfigurationSpace + ?Sized + 'a> = VirtioInterruptStatusData<'a, S>;
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioInterruptStatusData<'s, S> {
+    pub fn read_status(&self) -> (bool, bool) {
+        todo!()
+    }
+}
+
+impl<'s, S: PciConfigurationSpace + ?Sized + 's> VirtioCapabilityData<'s, S>
+    for VirtioInterruptStatusData<'s, S>
+{
+    fn from_space_offset(space: &'s S, offset: usize) -> Self {
+        Self { space, offset }
+    }
+
+    fn space(&self) -> &'s S {
+        self.space
+    }
+
+    fn offset(&self) -> usize {
+        self.offset
+    }
+}

kernel/driver/bus/pci/src/device.rs

@@ -13,7 +13,7 @@ use libk_util::{sync::spin_rwlock::IrqSafeRwLock, OneTimeInit};
 use yggdrasil_abi::error::Error;
 
 use crate::{
-    capability::{MsiCapability, MsiXCapability, MsiXVectorTable},
+    capability::{msi::MsiXVectorTable, MsiCapability, MsiXCapability, PciExpressCapability},
     driver::PciDriver,
     PciAddress, PciCommandRegister, PciConfigSpace, PciConfigurationSpace, PciSegmentInfo,
 };
@@ -142,6 +142,23 @@ impl PciDeviceInfo {
         }
     }
 
+    pub fn function_level_reset(&self) -> Result<(), Error> {
+        if let Some(mut pcie) = self.config_space.capability::<PciExpressCapability>() {
+            pcie.function_level_reset()
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
+    pub fn hot_link_reset(&self) -> Result<(), Error> {
+        if let Some(mut pcie) = self.config_space.capability::<PciExpressCapability>() {
+            pcie.hot_link_reset();
+            Ok(())
+        } else {
+            Err(Error::NotImplemented)
+        }
+    }
+
     pub fn init_interrupts(&self, preferred_mode: PreferredInterruptMode) -> Result<(), Error> {
         self.interrupt_config
             .try_init_with(|| {

kernel/driver/bus/usb/src/class_driver/hid_keyboard.rs

@@ -152,7 +152,7 @@ impl UsbDriver for UsbHidKeyboardDriver {
             event_count += state.retain(&buffer[2..], &mut events[event_count..]);
             event_count += state.press(&buffer[2..], &mut events[event_count..]);
 
-            let events = unsafe { MaybeUninit::slice_assume_init_ref(&events[..event_count]) };
+            let events = unsafe { events[..event_count].assume_init_ref() };
 
             for &event in events {
                 log::trace!("Generic Keyboard: {:?}", event);

kernel/driver/bus/usb/src/lib.rs

@@ -4,7 +4,6 @@
     iter_array_chunks,
     maybe_uninit_slice,
     maybe_uninit_as_bytes,
-    maybe_uninit_uninit_array,
     maybe_uninit_write_slice,
     maybe_uninit_fill
 )]

kernel/driver/bus/usb/src/pipe/control.rs

@@ -215,7 +215,7 @@ impl UsbControlPipeAccess {
     }
 
     pub async fn query_string(&self, index: u8) -> Result<String, UsbError> {
-        let mut buffer = MaybeUninit::uninit_array::<256>();
+        let mut buffer = [MaybeUninit::uninit(); 256];
 
         let len = self
             .control_transfer_in(
@@ -229,7 +229,7 @@ impl UsbControlPipeAccess {
                 &mut buffer[..],
             )
             .await?;
-        let data = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) };
+        let data = unsafe { buffer[..len].assume_init_ref() };
         let len = data[0] as usize;
 
         decode_usb_string(&data[2..len])

kernel/driver/bus/usb/src/pipe/normal.rs

@@ -15,7 +15,7 @@ pub trait UsbNormalPipeIn: UsbGenericPipe {
     async fn read(&self, buffer: &mut [u8]) -> Result<usize, UsbError> {
         let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?;
         let len = self.read_dma(dma_buffer.slice_mut(0..buffer.len())).await?;
-        let dma_slice = unsafe { MaybeUninit::slice_assume_init_ref(&dma_buffer[..len]) };
+        let dma_slice = unsafe { dma_buffer[..len].assume_init_ref() };
         buffer[..len].copy_from_slice(dma_slice);
         Ok(len)
     }
@@ -35,7 +35,7 @@ pub trait UsbNormalPipeOut: UsbGenericPipe {
 
     async fn write(&self, buffer: &[u8]) -> Result<usize, UsbError> {
         let mut dma_buffer = self.allocate_dma_buffer(buffer.len())?;
-        MaybeUninit::copy_from_slice(&mut dma_buffer, buffer);
+        dma_buffer[..].write_copy_of_slice(buffer);
         let dma_buffer = unsafe { DmaBuffer::assume_init_slice(dma_buffer) };
 
         self.write_dma(dma_buffer.slice(0..buffer.len())).await

kernel/driver/fs/ext2/src/access.rs

@@ -2,6 +2,22 @@ use libk::error::Error;
 
 use crate::{BlockGroupDescriptor, Ext2Fs, ExtendedSuperblock, Inode};
 
+pub enum InodeBlock<'a> {
+    Hole,
+    Data(&'a [u8]),
+}
+
+impl InodeBlock<'_> {
+    pub fn copy_to(&self, offset: usize, destination: &mut [u8]) {
+        match self {
+            Self::Hole => destination.fill(0),
+            Self::Data(data) => {
+                destination.copy_from_slice(&data[offset..offset + destination.len()])
+            }
+        }
+    }
+}
+
 impl Ext2Fs {
     pub async fn with_block<T, F: FnOnce(&[u8]) -> Result<T, Error>>(
         &self,
@@ -47,14 +63,19 @@ impl Ext2Fs {
             .await
     }
 
-    pub async fn with_inode_block<T, F: FnOnce(&[u8]) -> Result<T, Error>>(
+    pub async fn with_inode_block<T, F: FnOnce(InodeBlock) -> Result<T, Error>>(
         &self,
         inode: &Inode,
         block: u32,
         mapper: F,
     ) -> Result<T, Error> {
         let block_index = self.inode_block_index(inode, block).await?;
-        self.with_block(block_index, mapper).await
+        if block_index == 0 {
+            mapper(InodeBlock::Hole)
+        } else {
+            self.with_block(block_index, |block| mapper(InodeBlock::Data(block)))
+                .await
+        }
     }
 
     pub async fn with_inode_block_mut<T, F: FnOnce(&mut [u8]) -> Result<T, Error>>(

kernel/driver/fs/ext2/src/data.rs

@@ -5,6 +5,7 @@ use libk::vfs::Metadata;
 use yggdrasil_abi::{
     bitflags,
     io::{FileMode, FileType, GroupId, UserId},
+    time::SystemTime,
 };
 
 use crate::Ext2Fs;
@@ -253,8 +254,9 @@ impl Inode {
             inode: Some(ino),
             block_count: self.blocks(fs) as _,
             block_size: fs.block_size as _,
-            ctime: self.ctime as _,
-            mtime: self.mtime as _,
+            ctime: SystemTime::from_seconds(self.ctime as u64),
+            mtime: SystemTime::from_seconds(self.mtime as u64),
+            atime: SystemTime::from_seconds(self.atime as u64),
         }
     }
 }

kernel/driver/fs/ext2/src/dir/walk.rs

@@ -9,7 +9,7 @@ use yggdrasil_abi::{
     util::FixedString,
 };
 
-use crate::{data::FsRequiredFeatures, Dirent, Ext2Fs};
+use crate::{access::InodeBlock, data::FsRequiredFeatures, Dirent, Ext2Fs};
 
 use super::DirentName;
 
@@ -110,8 +110,17 @@ impl<'a, D: DerefMut<Target = [u8]> + 'a> Record<'a, D> {
 }
 
 impl<'a> DirentIter<'a> {
-    pub fn new(fs: &'a Ext2Fs, block: &'a [u8], offset: usize) -> Self {
-        Self { fs, block, offset }
+    pub fn new(fs: &'a Ext2Fs, block: InodeBlock<'a>, offset: usize) -> Self {
+        let data = if let InodeBlock::Data(data) = block {
+            data
+        } else {
+            &[]
+        };
+        Self {
+            fs,
+            block: data,
+            offset,
+        }
     }
 
     pub fn offset(&self) -> usize {
@@ -126,8 +135,9 @@ impl<'a> DirentIter<'a> {
         let dirent_struct_end = self.offset + size_of::<Dirent>();
         let dirent: &Dirent = bytemuck::from_bytes(&self.block[self.offset..dirent_struct_end]);
         let record_size = (dirent.ent_size as usize).max(size_of::<Dirent>());
+        let end = (self.offset + record_size).min(self.block.len());
         let record = Record {
-            data: &self.block[self.offset..self.offset + record_size],
+            data: &self.block[self.offset..end],
             fs: self.fs,
             offset: self.offset,
         };

kernel/driver/fs/ext2/src/inode/cache.rs

@@ -15,7 +15,8 @@ use yggdrasil_abi::{
 };
 
 use crate::{
-    data::InodeMode, dir::DirectoryNode, file::RegularNode, symlink::SymlinkNode, Ext2Fs, Inode,
+    access::InodeBlock, data::InodeMode, dir::DirectoryNode, file::RegularNode,
+    symlink::SymlinkNode, Ext2Fs, Inode,
 };
 
 pub struct InodeHolder {
@@ -103,7 +104,7 @@ impl InodeAccess {
         result
     }
 
-    pub async fn map_blocks<T, F: Fn(&Inode, usize, &[u8]) -> Result<Option<T>, Error>>(
+    pub async fn map_blocks<T, F: Fn(&Inode, usize, InodeBlock) -> Result<Option<T>, Error>>(
         &self,
         mapper: F,
     ) -> Result<Option<T>, Error> {
@@ -175,9 +176,9 @@ impl InodeAccess {
             .map_err(|_| Error::InvalidArgument)?;
 
         self.map_mut(|inode| {
-            inode.mtime = metadata.mtime as _;
-            inode.atime = metadata.mtime as _;
-            inode.ctime = metadata.ctime as _;
+            inode.mtime = metadata.mtime.seconds() as u32;
+            inode.atime = metadata.mtime.seconds() as u32;
+            inode.ctime = metadata.ctime.seconds() as u32;
 
             inode.mode.update_permissions(metadata.mode);
             inode.uid = uid;

kernel/driver/fs/ext2/src/lib.rs

@@ -390,12 +390,17 @@ impl Ext2Fs {
             let block_offset = (pos % self.block_size as u64) as usize;
             let amount = remaining.min(self.block_size - block_offset);
 
-            self.with_inode_block(inode, block_index as u32, |block| {
-                buffer[offset..offset + amount]
-                    .copy_from_slice(&block[block_offset..block_offset + amount]);
-                Ok(())
-            })
-            .await?;
+            let block = self.inode_block_index(inode, block_index as u32).await?;
+            if block == 0 {
+                buffer[offset..offset + amount].fill(0);
+            } else {
+                self.with_block(block, |block| {
+                    buffer[offset..offset + amount]
+                        .copy_from_slice(&block[block_offset..block_offset + amount]);
+                    Ok(())
+                })
+                .await?;
+            }
 
             pos += amount as u64;
             offset += amount;

kernel/driver/fs/ext2/src/symlink.rs

@@ -54,8 +54,9 @@ impl SymlinkNode {
                 } else {
                     self.fs
                         .with_inode_block(inode, 0, |block| {
-                            write.extend_from_slice(&block[..len]);
-                            buf[..len].copy_from_slice(&block[..len]);
+                            write.resize(len, 0);
+                            block.copy_to(0, &mut write[..]);
+                            block.copy_to(0, &mut buf[..len]);
                             Ok(())
                         })
                         .await?;

kernel/driver/fs/fat32/src/directory.rs

@@ -15,6 +15,7 @@ use libk_util::{
 };
 use yggdrasil_abi::{
     io::{DirectoryEntry, FileMode, GroupId, UserId},
+    time::SystemTime,
     util::FixedString,
 };
 
@@ -254,8 +255,10 @@ impl DirectoryNode {
                 FileMode::default_file()
             },
             inode: Some(cluster.0),
-            ctime: 0,
-            mtime: 0,
+            // TODO
+            ctime: SystemTime::ZERO,
+            mtime: SystemTime::ZERO,
+            atime: SystemTime::ZERO,
             block_count: (size.div_ceil(self.fs.layout.bytes_per_sector as u32)) as u64,
             block_size: self.fs.layout.bytes_per_sector as u64,
         };

kernel/driver/fs/fat32/src/lib.rs

@@ -10,7 +10,10 @@ use libk::{
     vfs::{Filesystem, FilesystemMountOption, Metadata, Node, NodeFlags, NodeRef},
 };
 use libk_util::get_le_u32;
-use yggdrasil_abi::io::{FileMode, GroupId, UserId};
+use yggdrasil_abi::{
+    io::{FileMode, GroupId, UserId},
+    time::SystemTime,
+};
 
 extern crate alloc;
 
@@ -64,8 +67,10 @@ impl Fat32Fs {
             uid: UserId::root(),
             gid: GroupId::root(),
             mode: FileMode::default_dir(),
-            ctime: 0,
-            mtime: 0,
+            // TODO
+            ctime: SystemTime::ZERO,
+            mtime: SystemTime::ZERO,
+            atime: SystemTime::ZERO,
             inode: Some(fs.layout.root_directory_cluster.0),
             block_size: fs.layout.bytes_per_sector as u64,
             block_count: 0,

kernel/driver/fs/memfs/src/bvec.rs

@@ -28,8 +28,8 @@ impl<'a, A: BlockAllocator> BVec<'a, A> {
     ///
     /// The function is guaranteed to make no allocations before the vector is actually written to.
     pub fn new() -> Self {
-        let mut l0 = MaybeUninit::uninit_array();
-        let mut l1 = MaybeUninit::uninit_array();
+        let mut l0 = [const { MaybeUninit::uninit() }; L0_BLOCKS];
+        let mut l1 = [const { MaybeUninit::uninit() }; L1_BLOCKS];
 
         for it in l0.iter_mut() {
             it.write(BlockData::null());
@@ -183,6 +183,7 @@ impl<'a, A: BlockAllocator> BVec<'a, A> {
                     let block = unsafe { self.index_unchecked_mut(index) };
                     assert!(block.is_null());
                     *block = BlockData::new()?;
+                    block.fill(0);
                 }
             }
 

kernel/driver/fs/memfs/src/dir.rs

@@ -1,3 +1,5 @@
+use core::sync::atomic::Ordering;
+
 use alloc::sync::Arc;
 
 use libk::vfs::{
@@ -6,7 +8,7 @@ use libk::vfs::{
 };
 use yggdrasil_abi::error::Error;
 
-use crate::{block::BlockAllocator, file::FileNode, MemoryFilesystem};
+use crate::{block::BlockAllocator, file::FileNode, MemoryFilesystem, INO_COUNTER};
 
 pub(crate) struct DirectoryNode<A: BlockAllocator> {
     fs: Arc<MemoryFilesystem<A>>,
@@ -31,7 +33,8 @@ impl<A: BlockAllocator> DirectoryImpl for DirectoryNode<A> {
     }
 
     fn create_node(&self, _parent: &NodeRef, info: &CreateInfo) -> Result<NodeRef, Error> {
-        let metadata = Metadata::now(info.uid, info.gid, info.mode);
+        let ino = INO_COUNTER.fetch_add(1, Ordering::Relaxed);
+        let metadata = Metadata::now(info.uid, info.gid, info.mode, ino);
         match info.ty {
             CreateFileType::File => Ok(FileNode::<A>::new(self.fs.clone(), metadata)),
             CreateFileType::Directory => Ok(DirectoryNode::<A>::new(self.fs.clone(), metadata)),

kernel/driver/fs/memfs/src/file.rs

@@ -2,13 +2,14 @@ use alloc::sync::Arc;
 use core::any::Any;
 
 use libk::vfs::{CommonImpl, InstanceData, Metadata, Node, NodeFlags, NodeRef, RegularImpl};
-use libk_util::sync::IrqSafeSpinlock;
+use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock};
 use yggdrasil_abi::{error::Error, io::OpenOptions};
 
 use crate::{block::BlockAllocator, bvec::BVec, MemoryFilesystem};
 
 pub(crate) struct FileNode<A: BlockAllocator> {
     pub(crate) data: IrqSafeSpinlock<BVec<'static, A>>,
+    pub(crate) metadata: IrqSafeRwLock<Metadata>,
 }
 
 impl<A: BlockAllocator> FileNode<A> {
@@ -16,9 +17,10 @@ impl<A: BlockAllocator> FileNode<A> {
         Node::regular(
             Self {
                 data: IrqSafeSpinlock::new(BVec::new()),
+                metadata: IrqSafeRwLock::new(metadata),
             },
-            NodeFlags::IN_MEMORY_PROPS,
-            Some(metadata),
+            NodeFlags::empty(),
+            None,
             Some(fs),
         )
     }
@@ -32,6 +34,15 @@ impl<A: BlockAllocator> CommonImpl for FileNode<A> {
     fn size(&self, _node: &NodeRef) -> Result<u64, Error> {
         Ok(self.data.lock().size() as u64)
     }
+
+    fn metadata(&self, _node: &NodeRef) -> Result<Metadata, Error> {
+        Ok(*self.metadata.read())
+    }
+
+    fn set_metadata(&self, _node: &NodeRef, metadata: &Metadata) -> Result<(), Error> {
+        *self.metadata.write() = *metadata;
+        Ok(())
+    }
 }
 
 impl<A: BlockAllocator> RegularImpl for FileNode<A> {
@@ -55,6 +66,7 @@ impl<A: BlockAllocator> RegularImpl for FileNode<A> {
         pos: u64,
         buf: &mut [u8],
     ) -> Result<usize, Error> {
+        self.metadata.write().set_atime_now();
         self.data.lock().read(pos, buf)
     }
 
@@ -65,10 +77,12 @@ impl<A: BlockAllocator> RegularImpl for FileNode<A> {
         pos: u64,
         buf: &[u8],
     ) -> Result<usize, Error> {
+        self.metadata.write().set_mtime_now();
         self.data.lock().write(pos, buf)
     }
 
     fn truncate(&self, _node: &NodeRef, new_size: u64) -> Result<(), Error> {
+        self.metadata.write().set_mtime_now();
         self.data.lock().truncate(new_size)
     }
 

kernel/driver/fs/memfs/src/lib.rs

@@ -2,9 +2,12 @@
 #![no_std]
 #![deny(missing_docs)]
 #![allow(clippy::new_without_default, clippy::new_ret_no_self)]
-#![feature(maybe_uninit_uninit_array, maybe_uninit_array_assume_init)]
+#![feature(maybe_uninit_array_assume_init)]
 
-use core::marker::PhantomData;
+use core::{
+    marker::PhantomData,
+    sync::atomic::{AtomicU32, Ordering},
+};
 
 use alloc::sync::Arc;
 use block::BlockAllocator;
@@ -57,6 +60,8 @@ mod dir;
 mod file;
 mod tar;
 
+static INO_COUNTER: AtomicU32 = AtomicU32::new(1);
+
 /// In-memory read/write filesystem
 pub struct MemoryFilesystem<A: BlockAllocator> {
     root: IrqSafeSpinlock<Option<NodeRef>>,
@@ -91,7 +96,8 @@ impl<A: BlockAllocator> MemoryFilesystem<A> {
                     return Err(Error::DoesNotExist);
                 }
 
-                let node = DirectoryNode::<A>::new(self.clone(), Metadata::now_root(mode));
+                let ino = INO_COUNTER.fetch_add(1, Ordering::Relaxed);
+                let node = DirectoryNode::<A>::new(self.clone(), Metadata::now_root(mode, ino));
                 at.add_child(filename, node.clone())?;
 
                 node
@@ -114,11 +120,15 @@ impl<A: BlockAllocator> MemoryFilesystem<A> {
         let kind = hdr.node_kind();
         let mode = usize::from(&hdr.mode);
         let mode = FileMode::new(0o777 & (mode as u32));
+        let ino = INO_COUNTER.fetch_add(1, Ordering::Relaxed);
         match kind {
-            FileType::File => Ok(FileNode::<A>::new(self.clone(), Metadata::now_root(mode))),
+            FileType::File => Ok(FileNode::<A>::new(
+                self.clone(),
+                Metadata::now_root(mode, ino),
+            )),
             FileType::Directory => Ok(DirectoryNode::<A>::new(
                 self.clone(),
-                Metadata::now_root(mode),
+                Metadata::now_root(mode, ino),
             )),
             FileType::Symlink => {
                 let target = hdr.symlink_target()?;
@@ -129,7 +139,8 @@ impl<A: BlockAllocator> MemoryFilesystem<A> {
     }
 
     fn from_slice_internal(self: &Arc<Self>, tar_data: &'static [u8]) -> Result<NodeRef, Error> {
-        let root = DirectoryNode::<A>::new(self.clone(), Metadata::now_root(FileMode::new(0o755)));
+        let root =
+            DirectoryNode::<A>::new(self.clone(), Metadata::now_root(FileMode::new(0o755), 0));
 
         // 1. Create paths in tar
         for item in TarIterator::new(tar_data) {

kernel/driver/net/rtl81xx/src/lib.rs

@@ -7,7 +7,8 @@ use rtl8139::Rtl8139;
 use rtl8168::Rtl8168;
 use ygg_driver_pci::{
     capability::{
-        DevicePowerState, PciExpressCapability, PcieLinkControl, PowerManagementCapability,
+        express::LinkControl, power::DevicePowerState, PciExpressCapability,
+        PowerManagementCapability,
     },
     device::{PciDeviceInfo, PreferredInterruptMode},
     macros::pci_driver,
@@ -30,9 +31,7 @@ pci_driver! {
             info.init_interrupts(PreferredInterruptMode::Msi(false))?;
 
             if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
-                let mut lcr = pcie.link_control();
-                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
-                pcie.set_link_control(lcr);
+                pcie.modify_link_control(LinkControl::ASPM::CLEAR + LinkControl::CLOCK_PM::CLEAR);
             }
 
             // Enable MMIO + interrupts + bus mastering
@@ -76,9 +75,7 @@ pci_driver! {
             }
 
             if let Some(mut pcie) = info.config_space.capability::<PciExpressCapability>() {
-                let mut lcr = pcie.link_control();
-                lcr.remove(PcieLinkControl::ASPM_MASK | PcieLinkControl::ECPM);
-                pcie.set_link_control(lcr);
+                pcie.modify_link_control(LinkControl::ASPM::CLEAR + LinkControl::CLOCK_PM::CLEAR);
             }
 
             let device = Rtl8168::new(dma.clone(), base, info.clone())?;

kernel/driver/net/stmmac/src/lib.rs

@@ -1,4 +1,5 @@
 #![no_std]
+#![recursion_limit = "256"]
 
 use core::{mem::MaybeUninit, time::Duration};
 

kernel/driver/sound/core/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "ygg_driver_sound_core"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+log.workspace = true
+async-trait.workspace = true
+futures-util.workspace = true

kernel/driver/sound/core/src/lib.rs

@@ -0,0 +1,122 @@
+#![no_std]
+
+use core::{
+    sync::atomic::{AtomicU32, Ordering},
+    task::{Context, Poll},
+};
+
+use alloc::{boxed::Box, collections::btree_map::BTreeMap, format, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use device_api::device::Device;
+use libk::{block, device::char::CharDevice, error::Error, fs::devfs, vfs::FileReadiness};
+use libk_util::sync::spin_rwlock::IrqSafeRwLock;
+use yggdrasil_abi::io::FileMode;
+
+extern crate alloc;
+
+#[derive(Debug, Clone)]
+pub enum SampleFormat {
+    S8,
+    S16Le,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(usize)]
+pub enum SampleRate {
+    Rate8000 = 8000,
+    Rate11025 = 11025,
+    Rate16000 = 16000,
+    Rate22050 = 22050,
+    Rate32000 = 32000,
+    Rate44100 = 44100,
+    Rate48000 = 48000,
+    Rate88200 = 88200,
+    Rate96000 = 96000,
+    Rate176400 = 176400,
+    Rate192000 = 192000,
+    Rate384000 = 384000,
+}
+
+#[derive(Debug, Clone)]
+pub struct SinkFormat {
+    pub sample_rate: SampleRate,
+    pub sample_format: SampleFormat,
+    pub channels: usize,
+}
+
+#[derive(Debug, Clone)]
+pub struct SinkSupportedFormats {
+    pub rates: Vec<usize>,
+    pub formats: Vec<SampleFormat>,
+    pub channels: usize,
+}
+
+#[async_trait]
+pub trait AudioSink: Sync + Send {
+    async fn set_format(&self, format: SinkFormat) -> Result<(), Error>;
+    async fn current_format(&self) -> Result<SinkFormat, Error>;
+    async fn supported_formats(&self) -> Result<SinkSupportedFormats, Error>;
+
+    async fn write(&self, data: &[u8]) -> Result<usize, Error>;
+
+    async fn start(&self) -> Result<(), Error>;
+    fn stop(&self) -> Result<(), Error>;
+
+    fn display_name(&self) -> &str {
+        "Audio Sink"
+    }
+}
+
+pub struct AudioSinkWrapper(Arc<dyn AudioSink>);
+
+impl Device for AudioSinkWrapper {
+    fn display_name(&self) -> &str {
+        self.0.display_name()
+    }
+}
+
+impl FileReadiness for AudioSinkWrapper {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        todo!()
+    }
+}
+
+#[async_trait]
+impl CharDevice for AudioSinkWrapper {
+    fn close(&self) -> Result<(), Error> {
+        self.0.stop()
+    }
+
+    async fn write(&self, buffer: &[u8]) -> Result<usize, Error> {
+        self.0.write(buffer).await
+    }
+
+    fn device_request(&self, option: u32, buffer: &mut [u8], len: usize) -> Result<usize, Error> {
+        let _ = option;
+        let _ = buffer;
+        let _ = len;
+        log::warn!("device_request unimplemented: {option:#x}");
+        Err(Error::InvalidOperation)
+    }
+}
+
+static SINKS: IrqSafeRwLock<BTreeMap<u32, Arc<AudioSinkWrapper>>> =
+    IrqSafeRwLock::new(BTreeMap::new());
+static LAST_SINK_ID: AtomicU32 = AtomicU32::new(0);
+
+pub fn register_audio_sink(sink: Arc<dyn AudioSink>) {
+    let id = LAST_SINK_ID.fetch_add(1, Ordering::Relaxed);
+    let name = format!("snd{id}");
+    let sink = Arc::new(AudioSinkWrapper(sink));
+    SINKS.write().insert(id, sink.clone());
+    devfs::add_named_char_device(sink, &name, FileMode::new(0o222)).ok();
+}
+
+impl SampleFormat {
+    pub fn sample_size(&self) -> usize {
+        match self {
+            Self::S8 => 1,
+            Self::S16Le => 2,
+        }
+    }
+}

kernel/driver/sound/intel-hda/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "ygg_driver_intel_hda"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+device-api.workspace = true
+yggdrasil-abi.workspace = true
+libk-mm.workspace = true
+libk-util.workspace = true
+libk.workspace = true
+
+ygg_driver_pci.path = "../../bus/pci"
+ygg_driver_sound_core.path = "../core"
+
+log.workspace = true
+tock-registers.workspace = true
+futures-util.workspace = true
+async-trait.workspace = true
+bytemuck.workspace = true

kernel/driver/sound/intel-hda/src/codec/mod.rs

@@ -0,0 +1,527 @@
+use core::fmt;
+
+use alloc::{sync::Arc, vec::Vec};
+use libk::error::Error;
+use tock_registers::{fields::FieldValue, register_bitfields, LocalRegisterCopy};
+use yggdrasil_abi::bitflags;
+
+use crate::{
+    ring::{Command, PinControl, PinDefaultConfig, PinDevice, Verb},
+    HdAudio,
+};
+
+pub mod parameter;
+pub use parameter::*;
+
+#[derive(Debug, Clone, Copy)]
+pub struct Node {
+    pub codec: u8,
+    pub nid: u8,
+}
+
+pub struct WidgetConnectionChain<'a> {
+    afg: &'a AudioNode,
+    current: Option<u8>,
+}
+
+#[derive(Debug)]
+pub struct AudioOutputWidget {
+    channel_count: usize,
+}
+#[derive(Debug)]
+pub struct AudioInputWidget {
+    channel_count: usize,
+}
+#[derive(Debug)]
+pub struct PinComplexWidget {
+    default_config: PinDefaultConfig,
+}
+#[derive(Debug)]
+pub struct MixerWidget {}
+#[derive(Debug)]
+pub enum AudioWidget {
+    AudioOutput(AudioOutputWidget),
+    AudioInput(AudioInputWidget),
+    PinComplex(PinComplexWidget),
+    Mixer(MixerWidget),
+    Other,
+}
+
+#[derive(Debug)]
+pub enum AudioFormat {
+    Pcm { rates: Vec<usize>, bits: Vec<u8> },
+    Other,
+}
+
+#[derive(Debug)]
+pub struct AudioWidgetNode {
+    parent_nid: u8,
+    node: Node,
+
+    output_amp: Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>,
+    input_amp: Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>,
+    connection: Option<u8>,
+    formats: Vec<AudioFormat>,
+
+    widget: AudioWidget,
+}
+
+pub struct AudioNode {
+    node: Node,
+    widget_connection: Vec<(u8, u8)>,
+    widgets: Vec<AudioWidgetNode>,
+}
+
+pub struct Codec {
+    hda: Arc<HdAudio>,
+
+    root: Node,
+    nodes: Vec<AudioNode>,
+}
+
+impl AudioFormat {
+    async fn query_from_node(hda: &HdAudio, node: Node) -> Result<Vec<Self>, Error> {
+        let supported_formats = node
+            .get_parameter::<SupportedStreamFormats::Register>(hda)
+            .await?;
+
+        let mut formats = Vec::new();
+
+        if supported_formats.matches_all(SupportedStreamFormats::PCM::SET) {
+            formats.push(Self::query_pcm(hda, node).await?);
+        }
+        if supported_formats.matches_all(SupportedStreamFormats::F32::SET) {
+            formats.push(Self::Other);
+        }
+        if supported_formats.matches_all(SupportedStreamFormats::AC3::SET) {
+            formats.push(Self::Other);
+        }
+
+        Ok(formats)
+    }
+
+    async fn query_pcm(hda: &HdAudio, node: Node) -> Result<Self, Error> {
+        // Conversion tables
+        const RATES: &[(usize, FieldValue<u32, SupportedPcmFormats::Register>)] = &[
+            (8000, SupportedPcmFormats::RATE_8::SET),
+            (11024, SupportedPcmFormats::RATE_11_025::SET),
+            (16000, SupportedPcmFormats::RATE_16::SET),
+            (22050, SupportedPcmFormats::RATE_22_05::SET),
+            (32000, SupportedPcmFormats::RATE_32::SET),
+            (44100, SupportedPcmFormats::RATE_44_1::SET),
+            (48000, SupportedPcmFormats::RATE_48::SET),
+            (88200, SupportedPcmFormats::RATE_88_2::SET),
+            (96000, SupportedPcmFormats::RATE_96::SET),
+            (176400, SupportedPcmFormats::RATE_176_4::SET),
+            (192000, SupportedPcmFormats::RATE_192::SET),
+            (384000, SupportedPcmFormats::RATE_384::SET),
+        ];
+        const BITS: &[(u8, FieldValue<u32, SupportedPcmFormats::Register>)] = &[
+            (8, SupportedPcmFormats::BITS_8::SET),
+            (16, SupportedPcmFormats::BITS_16::SET),
+            (20, SupportedPcmFormats::BITS_20::SET),
+            (24, SupportedPcmFormats::BITS_24::SET),
+            (32, SupportedPcmFormats::BITS_32::SET),
+        ];
+
+        let supported_pcm_formats = node
+            .get_parameter::<SupportedPcmFormats::Register>(hda)
+            .await?;
+
+        let mut supported_rates = Vec::new();
+        let mut supported_bits = Vec::new();
+
+        for &(rate, bit) in RATES {
+            if supported_pcm_formats.matches_all(bit) {
+                supported_rates.push(rate);
+            }
+        }
+        for &(bits, bit) in BITS {
+            if supported_pcm_formats.matches_all(bit) {
+                supported_bits.push(bits);
+            }
+        }
+
+        Ok(Self::Pcm {
+            rates: supported_rates,
+            bits: supported_bits,
+        })
+    }
+}
+
+impl Node {
+    pub async fn perform_command(
+        &self,
+        hda: &HdAudio,
+        verb: Verb,
+        parameter: u32,
+    ) -> Result<u32, Error> {
+        hda.perform_command(Command::new(self.codec, self.nid, verb, parameter))
+            .await
+    }
+
+    pub async fn get_parameter<P: NodeParameter>(&self, hda: &HdAudio) -> Result<P::Value, Error> {
+        self.perform_command(hda, Verb::GetParameter, P::NUMBER as u32)
+            .await
+            .map(P::from_response)
+    }
+
+    pub async fn get_connection_list_entry(
+        &self,
+        hda: &HdAudio,
+        index: usize,
+    ) -> Result<u32, Error> {
+        self.perform_command(hda, Verb::GetConnectionListEntry, index as u32)
+            .await
+    }
+
+    pub async fn set_stream(&self, hda: &HdAudio, stream: u8, channel: u8) -> Result<(), Error> {
+        self.perform_command(
+            hda,
+            Verb::SetConverterStreamChannel,
+            ((stream as u32) << 4) | (channel as u32),
+        )
+        .await?;
+        Ok(())
+    }
+
+    pub async fn get_pin_default_config(&self, hda: &HdAudio) -> Result<PinDefaultConfig, Error> {
+        self.perform_command(hda, Verb::GetPinWidgetDefaultConfig, 0)
+            .await
+            .map(PinDefaultConfig::from)
+    }
+
+    pub async fn set_power_state(&self, hda: &HdAudio, power: bool) -> Result<(), Error> {
+        let ps_set = if power { 0x0 } else { 0x4 };
+        self.perform_command(hda, Verb::SetPowerState, ps_set)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn set_unsolicited_response(
+        &self,
+        hda: &HdAudio,
+        tag: Option<u8>,
+    ) -> Result<(), Error> {
+        let value = match tag {
+            Some(tag) => (tag as u32) | (1 << 7),
+            None => 0,
+        };
+        self.perform_command(hda, Verb::SetUnsolicitedResponse, value)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn set_pin_control(&self, hda: &HdAudio, pin_control: u32) -> Result<(), Error> {
+        self.perform_command(hda, Verb::SetPinWidgetControl, pin_control & 0xFF)
+            .await?;
+        Ok(())
+    }
+
+    pub async fn get_pin_control(
+        &self,
+        hda: &HdAudio,
+    ) -> Result<LocalRegisterCopy<u32, PinControl::Register>, Error> {
+        self.perform_command(hda, Verb::GetPinWidgetControl, 0)
+            .await
+            .map(LocalRegisterCopy::new)
+    }
+
+    pub async fn modify_pin_control(
+        &self,
+        hda: &HdAudio,
+        change: FieldValue<u32, PinControl::Register>,
+    ) -> Result<(), Error> {
+        let mut value = self.get_pin_control(hda).await?;
+        value.modify(change);
+        self.set_pin_control(hda, value.get()).await
+    }
+}
+
+impl AudioWidgetNode {
+    pub async fn probe(hda: &HdAudio, parent_nid: u8, node: Node) -> Result<Self, Error> {
+        let capabilities = node
+            .get_parameter::<AudioWidgetCapabilities::Register>(hda)
+            .await?;
+        let channel_count = ((capabilities.read(AudioWidgetCapabilities::EXT_CHANNEL_COUNT) << 1)
+            | capabilities.read(AudioWidgetCapabilities::CHANNEL_COUNT))
+            + 1;
+        let in_amp = capabilities.matches_all(AudioWidgetCapabilities::IN_AMP::SET);
+        let out_amp = capabilities.matches_all(AudioWidgetCapabilities::OUT_AMP::SET);
+
+        let in_amp =
+            if in_amp && capabilities.matches_all(AudioWidgetCapabilities::AMP_OVERRIDE::SET) {
+                node.get_parameter::<InputAmpCapabilities>(hda).await?
+            } else {
+                None
+            };
+        let out_amp =
+            if out_amp && capabilities.matches_all(AudioWidgetCapabilities::AMP_OVERRIDE::SET) {
+                node.get_parameter::<OutputAmpCapabilities>(hda).await?
+            } else {
+                None
+            };
+
+        let connection = if capabilities.matches_all(AudioWidgetCapabilities::CONN_LIST::SET) {
+            let (length, long) = node.get_parameter::<ConnectionListLength>(hda).await?;
+            // if length > 1 || long {
+            //     todo!()
+            // }
+            let connection = node.get_connection_list_entry(hda, 0).await?;
+            Some(connection as u8)
+        } else {
+            None
+        };
+
+        let formats = if capabilities.matches_all(AudioWidgetCapabilities::FORMAT_OVERRIDE::SET) {
+            AudioFormat::query_from_node(hda, node).await?
+        } else {
+            Vec::new()
+        };
+
+        let widget = match capabilities.read_as_enum(AudioWidgetCapabilities::TYPE) {
+            Some(AudioWidgetCapabilities::TYPE::Value::AudioOutput) => {
+                // Disable audio outputs immediately
+                node.set_stream(hda, 0x00, 0x00).await?;
+
+                AudioWidget::AudioOutput(AudioOutputWidget {
+                    channel_count: channel_count as usize,
+                })
+            }
+            Some(AudioWidgetCapabilities::TYPE::Value::AudioInput) => {
+                AudioWidget::AudioInput(AudioInputWidget {
+                    channel_count: channel_count as usize,
+                })
+            }
+            Some(AudioWidgetCapabilities::TYPE::Value::PinComplex) => {
+                let default_config = node.get_pin_default_config(hda).await?;
+                AudioWidget::PinComplex(PinComplexWidget { default_config })
+            }
+            Some(AudioWidgetCapabilities::TYPE::Value::AudioMixer) => {
+                AudioWidget::Mixer(MixerWidget {})
+            }
+            v => {
+                log::warn!("hda: unknown audio widget: {v:?}");
+                AudioWidget::Other
+            }
+        };
+
+        Ok(Self {
+            parent_nid,
+            node,
+            widget,
+
+            output_amp: out_amp,
+            input_amp: in_amp,
+            formats,
+            connection,
+        })
+    }
+}
+
+impl AudioNode {
+    pub async fn probe(hda: &HdAudio, node: Node) -> Result<Self, Error> {
+        // Query widgets
+        let subnodes = node.get_parameter::<SubNodeInfo>(hda).await?;
+
+        let mut widgets = Vec::new();
+        for wid in subnodes.iter() {
+            let widget_node = Node {
+                codec: node.codec,
+                nid: wid,
+            };
+            match AudioWidgetNode::probe(hda, node.nid, widget_node).await {
+                Ok(widget) => {
+                    widgets.push(widget);
+                }
+                Err(error) => {
+                    log::warn!(
+                        "hda: could not enumerate {}.{}.{}: {error:?}",
+                        node.codec,
+                        node.nid,
+                        wid
+                    );
+                }
+            }
+        }
+
+        let mut widget_connection = Vec::new();
+        for widget in &widgets {
+            if let Some(connection) = widget.connection {
+                widget_connection.push((widget.node.nid, connection));
+            }
+        }
+
+        Ok(Self {
+            node,
+            widgets,
+            widget_connection,
+        })
+    }
+
+    pub fn widget(&self, wid: u8) -> Option<&AudioWidgetNode> {
+        self.widgets.iter().find(|w| w.node.nid == wid)
+    }
+
+    pub fn widget_supported_formats<'a>(&self, widget: &'a AudioWidgetNode) -> &'a [AudioFormat] {
+        if !widget.formats.is_empty() {
+            &widget.formats
+        } else {
+            // TODO node supported formats
+            &[]
+        }
+    }
+
+    // Returns (pin, kind) pair
+    pub fn default_output_pin(&self) -> Option<(Node, PinDevice)> {
+        let mut best: Option<(Node, PinDevice)> = None;
+        for widget in &self.widgets {
+            if let AudioWidget::PinComplex(pin) = &widget.widget {
+                let Some(connection) = widget.connection else {
+                    continue;
+                };
+                let dev = pin.default_config.default_device;
+
+                let better = match best {
+                    None => true,
+                    Some((_, other)) => dev.output_score() > other.output_score(),
+                };
+
+                if better {
+                    best = Some((widget.node, dev));
+                }
+            }
+        }
+
+        best
+    }
+
+    pub fn connection_chain(&self, source: u8) -> WidgetConnectionChain<'_> {
+        WidgetConnectionChain {
+            afg: self,
+            current: Some(source),
+        }
+    }
+}
+
+impl fmt::Debug for AudioNode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("AudioNode")
+            .field("id", &self.node.nid)
+            .field("widgets", &self.widgets)
+            .finish()
+    }
+}
+
+impl Codec {
+    pub async fn probe(hda: Arc<HdAudio>, codec: u8) -> Result<Self, Error> {
+        let root_node = Node { codec, nid: 0 };
+        let id = root_node.get_parameter::<DeviceId>(&*hda).await?;
+        if (id.vendor_id == 0 && id.device_id == 0)
+            || (id.vendor_id == 0xFFFF && id.device_id == 0xFFFF)
+        {
+            return Err(Error::DoesNotExist);
+        }
+
+        let root_subnodes = root_node.get_parameter::<SubNodeInfo>(&*hda).await?;
+
+        let mut nodes = Vec::new();
+
+        for nid in root_subnodes.iter() {
+            let node = Node { codec, nid };
+            let node_type = node.get_parameter::<NodeType>(&*hda).await?;
+
+            // Ignore non-audio nodes
+            if node_type != 0x01 {
+                continue;
+            }
+
+            match AudioNode::probe(&*hda, node).await {
+                Ok(node) => nodes.push(node),
+                Err(error) => {
+                    log::warn!("hda: could not enumerate {}.{}: {error:?}", codec, nid);
+                }
+            }
+        }
+
+        Ok(Self {
+            hda,
+            root: root_node,
+            nodes,
+        })
+    }
+
+    pub fn node(&self, nid: u8) -> Option<&AudioNode> {
+        self.nodes.iter().find(|n| n.node.nid == nid)
+    }
+
+    pub async fn setup_default_output(&self) -> Result<Node, Error> {
+        let (pin_id, dev, node_id) = self
+            .nodes
+            .iter()
+            .find_map(|node| {
+                node.default_output_pin()
+                    .map(|(pin, dev)| (pin, dev, node.node))
+            })
+            .ok_or(Error::DoesNotExist)?;
+
+        log::info!(
+            "hda#{}: default output {pin_id:?}, {dev:?}",
+            self.root.codec
+        );
+        let audio_node = self.node(node_id.nid).ok_or(Error::DoesNotExist)?;
+        let mut audio_widget = None;
+
+        for widget in audio_node.connection_chain(pin_id.nid) {
+            let wid = widget.node;
+
+            match &widget.widget {
+                AudioWidget::PinComplex(_) => {
+                    log::info!("  Configure pin @ {wid:?}");
+                    wid.set_power_state(&*self.hda, true).await?;
+                    // TODO only set for jacks
+                    wid.set_unsolicited_response(&*self.hda, Some(wid.nid))
+                        .await?;
+                    wid.modify_pin_control(
+                        &*self.hda,
+                        PinControl::OutEnable::SET + PinControl::HPhnEnable::SET,
+                    )
+                    .await?;
+                }
+                AudioWidget::AudioOutput(out) => {
+                    log::info!("  Configure audio output @ {wid:?}");
+                    audio_widget = Some(widget.node);
+                    wid.set_power_state(&*self.hda, true).await?;
+                    wid.set_unsolicited_response(&*self.hda, None).await?;
+                }
+                AudioWidget::Mixer(_) => {
+                    log::info!("  Configure mixer @ {wid:?}");
+                    wid.set_power_state(&*self.hda, true).await?;
+                }
+                _ => (),
+            }
+        }
+
+        Ok(audio_widget.unwrap())
+    }
+}
+
+impl fmt::Debug for Codec {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Codec")
+            .field("id", &self.root.codec)
+            .field("nodes", &self.nodes)
+            .finish()
+    }
+}
+
+impl<'a> Iterator for WidgetConnectionChain<'a> {
+    type Item = &'a AudioWidgetNode;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let current = self.current?;
+        let widget = self.afg.widget(current)?;
+        self.current = widget.connection;
+        Some(widget)
+    }
+}

kernel/driver/sound/intel-hda/src/codec/parameter.rs

@@ -0,0 +1,198 @@
+use libk::error::Error;
+use tock_registers::{register_bitfields, LocalRegisterCopy};
+
+use crate::ring::NodeParameterNumber;
+
+pub trait NodeParameter {
+    const NUMBER: NodeParameterNumber;
+    type Value;
+
+    fn data(&self) -> u32 {
+        0
+    }
+    fn from_response(response: u32) -> Self::Value;
+}
+
+#[derive(Debug)]
+pub struct DeviceId {
+    pub vendor_id: u16,
+    pub device_id: u16,
+}
+#[derive(Debug)]
+pub struct SubNodeInfo {
+    pub start_number: u8,
+    pub total_number: u8,
+}
+pub struct NodeType;
+register_bitfields! {
+    u32,
+    pub AudioWidgetCapabilities [
+        CHANNEL_COUNT       OFFSET(0)   NUMBITS(1)  [],
+        IN_AMP              OFFSET(1)   NUMBITS(1)  [],
+        OUT_AMP             OFFSET(2)   NUMBITS(1)  [],
+        AMP_OVERRIDE        OFFSET(3)   NUMBITS(1)  [],
+        FORMAT_OVERRIDE     OFFSET(4)   NUMBITS(1)  [],
+        STRIPE              OFFSET(5)   NUMBITS(1)  [],
+        PROC_WIDGET         OFFSET(6)   NUMBITS(1)  [],
+        UNSOL_CAPABLE       OFFSET(7)   NUMBITS(1)  [],
+        CONN_LIST           OFFSET(8)   NUMBITS(1)  [],
+        DIGITAL             OFFSET(9)   NUMBITS(1)  [],
+        POWER_CONTROL       OFFSET(10)  NUMBITS(1)  [],
+        LR_SWAP             OFFSET(11)  NUMBITS(1)  [],
+        CP_CAPS             OFFSET(12)  NUMBITS(1)  [],
+        EXT_CHANNEL_COUNT   OFFSET(13)  NUMBITS(3)  [],
+        DELAY               OFFSET(16)  NUMBITS(4)  [],
+        TYPE                OFFSET(20)  NUMBITS(4)  [
+            AudioOutput     = 0x0,
+            AudioInput      = 0x1,
+            AudioMixer      = 0x2,
+            AudioSelector   = 0x3,
+            PinComplex      = 0x4,
+            Power           = 0x5,
+            VolumeKnob      = 0x6,
+            BeepGenerator   = 0x7,
+            VendorDefined   = 0xF,
+        ],
+    ],
+}
+register_bitfields! {
+    u32,
+    pub AmpCapabilities [
+        OFFSET          OFFSET(0)   NUMBITS(7)  [],
+        NUM_STEPS       OFFSET(8)   NUMBITS(7)  [],
+        STEP_SIZE       OFFSET(16)  NUMBITS(7)  [],
+        MUTE_CAPABLE    OFFSET(31)  NUMBITS(1)  [],
+    ],
+}
+pub struct InputAmpCapabilities;
+pub struct OutputAmpCapabilities;
+pub struct ConnectionListLength;
+register_bitfields! {
+    u32,
+    pub SupportedStreamFormats [
+        PCM OFFSET(0)   NUMBITS(1)  [],
+        F32 OFFSET(1)   NUMBITS(1)  [],
+        AC3 OFFSET(2)   NUMBITS(1)  [],
+    ],
+}
+register_bitfields! {
+    u32,
+    pub SupportedPcmFormats [
+        RATE_8    OFFSET(0)   NUMBITS(1)  [],
+        RATE_11_025 OFFSET(1)   NUMBITS(1)  [],
+        RATE_16   OFFSET(2)   NUMBITS(1)  [],
+        RATE_22_05  OFFSET(3)   NUMBITS(1)  [],
+        RATE_32     OFFSET(4)   NUMBITS(1)  [],
+        RATE_44_1   OFFSET(5)   NUMBITS(1)  [],
+        RATE_48     OFFSET(6)   NUMBITS(1)  [],
+        RATE_88_2   OFFSET(7)   NUMBITS(1)  [],
+        RATE_96     OFFSET(8)   NUMBITS(1)  [],
+        RATE_176_4  OFFSET(9)   NUMBITS(1)  [],
+        RATE_192    OFFSET(10)  NUMBITS(1)  [],
+        RATE_384    OFFSET(11)  NUMBITS(1)  [],
+
+        BITS_8      OFFSET(16)  NUMBITS(1)  [],
+        BITS_16     OFFSET(17)  NUMBITS(1)  [],
+        BITS_20     OFFSET(18)  NUMBITS(1)  [],
+        BITS_24     OFFSET(19)  NUMBITS(1)  [],
+        BITS_32     OFFSET(20)  NUMBITS(1)  [],
+    ],
+}
+
+impl NodeParameter for DeviceId {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::DeviceId;
+    type Value = Self;
+
+    fn from_response(response: u32) -> Self::Value {
+        Self {
+            vendor_id: (response >> 16) as u16,
+            device_id: response as u16,
+        }
+    }
+}
+impl NodeParameter for SubNodeInfo {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::NodeCount;
+    type Value = Self;
+
+    fn from_response(response: u32) -> Self::Value {
+        Self {
+            start_number: (response >> 16) as u8,
+            total_number: response as u8,
+        }
+    }
+}
+impl NodeParameter for NodeType {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::FunctionGroupType;
+    type Value = u8;
+
+    fn from_response(response: u32) -> Self::Value {
+        response as u8
+    }
+}
+impl NodeParameter for AudioWidgetCapabilities::Register {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::AudioWidgetCapabilities;
+    type Value = LocalRegisterCopy<u32, Self>;
+
+    fn from_response(response: u32) -> Self::Value {
+        LocalRegisterCopy::new(response)
+    }
+}
+impl NodeParameter for InputAmpCapabilities {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::InputAmplifierCapabilities;
+    type Value = Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>;
+
+    fn from_response(response: u32) -> Self::Value {
+        if response != 0 {
+            Some(LocalRegisterCopy::new(response))
+        } else {
+            None
+        }
+    }
+}
+impl NodeParameter for OutputAmpCapabilities {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::OutputAmplifierCapabilities;
+    type Value = Option<LocalRegisterCopy<u32, AmpCapabilities::Register>>;
+
+    fn from_response(response: u32) -> Self::Value {
+        if response != 0 {
+            Some(LocalRegisterCopy::new(response))
+        } else {
+            None
+        }
+    }
+}
+impl NodeParameter for ConnectionListLength {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::ConnectionListLength;
+    type Value = (usize, bool);
+
+    fn from_response(response: u32) -> Self::Value {
+        let long_form = response & (1 << 7) != 0;
+        let length = (response & 0x3F) as usize;
+        (length, long_form)
+    }
+}
+impl NodeParameter for SupportedStreamFormats::Register {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::SupportedFormats;
+    type Value = LocalRegisterCopy<u32, Self>;
+
+    fn from_response(response: u32) -> Self::Value {
+        LocalRegisterCopy::new(response)
+    }
+}
+impl NodeParameter for SupportedPcmFormats::Register {
+    const NUMBER: NodeParameterNumber = NodeParameterNumber::SupportedPcmRates;
+    type Value = LocalRegisterCopy<u32, Self>;
+
+    fn from_response(response: u32) -> Self::Value {
+        LocalRegisterCopy::new(response)
+    }
+}
+
+impl SubNodeInfo {
+    pub fn iter(&self) -> impl Iterator<Item = u8> {
+        // Exclude root
+        let start_subnode = self.start_number.max(1);
+        let end_subnode = self.start_number.saturating_add(self.total_number);
+        start_subnode..end_subnode
+    }
+}

kernel/driver/sound/intel-hda/src/lib.rs

@@ -0,0 +1,344 @@
+#![no_std]
+#![feature(let_chains)]
+
+use core::{
+    sync::atomic::{AtomicU8, Ordering},
+    time::Duration,
+};
+
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use codec::{AudioFormat, Codec, Node};
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
+};
+use futures_util::task::AtomicWaker;
+use libk::{
+    dma::DmaBuffer,
+    error::Error,
+    task::runtime::{self, psleep, with_timeout},
+    time::monotonic_time,
+};
+use libk_mm::device::DeviceMemoryIo;
+use libk_util::{
+    event::{BitmapEvent, BoolEvent, CounterEvent},
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+    OneTimeInit,
+};
+use regs::{Regs, SDxCTL0, CORBRP, GCAP, RIRBWP};
+use ring::{Command, CommandRing, NodeParameterNumber, Verb};
+use sink::HdAudioSink;
+use stream::{BufferDescriptorList, OutputStream};
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
+use ygg_driver_pci::{
+    capability::{power::DevicePowerState, PowerManagementCapability},
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+    PciBaseAddress, PciConfigurationSpace,
+};
+
+extern crate alloc;
+
+mod codec;
+mod regs;
+mod ring;
+mod sink;
+mod stream;
+
+struct HdAudio {
+    regs: IrqSafeSpinlock<DeviceMemoryIo<'static, Regs>>,
+    dma: Arc<dyn DmaAllocator>,
+    pci: PciDeviceInfo,
+
+    input_stream_count: usize,
+    output_stream_count: usize,
+    bidi_stream_count: usize,
+
+    sinks: Vec<IrqSafeSpinlock<Option<Arc<HdAudioSink>>>>,
+    last_stream_tag: AtomicU8,
+
+    softirq_event: BitmapEvent<AtomicWaker>,
+
+    cring: OneTimeInit<CommandRing>,
+}
+
+impl HdAudio {
+    fn new(
+        pci: PciDeviceInfo,
+        dma: Arc<dyn DmaAllocator>,
+        regs: DeviceMemoryIo<'static, Regs>,
+        input_stream_count: usize,
+        output_stream_count: usize,
+        bidi_stream_count: usize,
+    ) -> Self {
+        Self {
+            pci,
+            dma,
+            regs: IrqSafeSpinlock::new(regs),
+
+            input_stream_count,
+            output_stream_count,
+            bidi_stream_count,
+
+            sinks: (0..output_stream_count)
+                .map(|_| IrqSafeSpinlock::new(None))
+                .collect(),
+            last_stream_tag: AtomicU8::new(1),
+
+            softirq_event: BitmapEvent::new(AtomicWaker::new()),
+
+            cring: OneTimeInit::new(),
+        }
+    }
+}
+
+impl HdAudio {
+    async fn late_init(self: &Arc<Self>) -> Result<(), Error> {
+        // Interrogate present codecs
+        let codecs = self.regs.lock().take_attached_codecs();
+
+        for index in (0..16).filter(|&i| codecs & (1 << i) != 0) {
+            match Codec::probe(self.clone(), index).await {
+                Ok(codec) => {
+                    let codec = Arc::new(codec);
+                    log::info!("{codec:#?}");
+
+                    match codec.setup_default_output().await {
+                        Ok(id) => match self.add_sink(codec, id).await {
+                            Ok(_) => {
+                                log::info!("hda#{}: sink added", id.codec);
+                            }
+                            Err(error) => {
+                                log::info!("hda#{}: could not add sink: {error:?}", id.codec);
+                            }
+                        },
+                        Err(error) => {
+                            log::error!("Default output setup error: {error:?}");
+                        }
+                    }
+                }
+                Err(error) => {
+                    log::error!("hda: codec #{index} setup error: {error:?}");
+                }
+            }
+        }
+
+        loop {
+            runtime::sleep(Duration::from_secs(1)).await;
+        }
+    }
+
+    async fn add_sink(
+        self: &Arc<Self>,
+        codec: Arc<Codec>,
+        audio_widget: Node,
+    ) -> Result<Arc<HdAudioSink>, Error> {
+        for (index, slot) in self.sinks.iter().enumerate() {
+            let mut slot = slot.lock();
+            if slot.is_none() {
+                log::info!("hda: add sink #{index}");
+                let stream_tag = self.last_stream_tag.fetch_add(1, Ordering::Relaxed);
+                let sink = HdAudioSink::create(
+                    self.clone(),
+                    index + self.input_stream_count,
+                    stream_tag,
+                    codec.clone(),
+                    audio_widget,
+                )
+                .await?;
+
+                *slot = Some(sink.clone());
+                return Ok(sink);
+            }
+        }
+        log::warn!("hda: too many sinks");
+        Err(Error::InvalidOperation)
+    }
+
+    async fn softirq(&self) -> Result<(), Error> {
+        let cring = self.cring.get();
+
+        loop {
+            match with_timeout(self.softirq_event.wait(), Duration::from_millis(100)).await {
+                // IRQs happened
+                Ok(events) => {
+                    log::info!("softirq()");
+                    for sink_index in
+                        self.input_stream_count..self.input_stream_count + self.output_stream_count
+                    {
+                        // Handle sink events
+                        if events & (1 << sink_index) != 0
+                            && let Some(sink) = self.sink(sink_index - self.input_stream_count)
+                        {
+                            let position = self.regs.lock().STREAMS[sink_index].SDxLPIB.get();
+                            sink.handle_softirq(position).await;
+                        }
+                    }
+                    //
+                }
+                // Just poll
+                Err(_) => {}
+            }
+
+            let regs = self.regs.lock();
+            let rirb_head = regs.RIRBWP.read(RIRBWP::RIRBWP) as u8;
+            let corb_tail = regs.CORBRP.read(CORBRP::CORBRP) as u8;
+
+            cring.process_completions(corb_tail, rirb_head);
+        }
+    }
+
+    async fn perform_command(&self, command: Command) -> Result<u32, Error> {
+        let cring = self.cring.get();
+        let codec = command.codec();
+        let head = cring.submit_command(command).await?;
+
+        {
+            let regs = self.regs.lock();
+            regs.CORBWP.set(head as u16);
+        }
+
+        with_timeout(cring.wait_codec(codec), Duration::from_secs(3)).await
+    }
+
+    fn sink(&self, index: usize) -> Option<Arc<HdAudioSink>> {
+        self.sinks[index].lock().clone()
+    }
+}
+
+impl InterruptHandler for HdAudio {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        let regs = self.regs.lock();
+        let rirbsts = regs.RIRBSTS.extract();
+        let status = regs.INTSTS.get() & 0x3FFFFFFF;
+
+        if rirbsts.get() != 0 {
+            regs.RIRBSTS.set(rirbsts.get());
+            self.softirq_event.signal(1 << 30);
+        }
+
+        if status != 0 {
+            for stream in 0..30 {
+                if status & (1 << stream) != 0 {
+                    let stream_status = regs.STREAMS[stream].SDxSTS.extract();
+                    regs.STREAMS[stream].SDxSTS.set(stream_status.get());
+                }
+            }
+            self.softirq_event.signal(status as u64);
+        }
+
+        true
+    }
+}
+
+impl Device for HdAudio {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        // FLR if capable
+        self.pci.function_level_reset().ok();
+        self.pci
+            .map_interrupt(InterruptAffinity::Any, self.clone())?;
+
+        let regs = self.regs.lock();
+
+        let vmaj = regs.VMAJ.get();
+        let vmin = regs.VMIN.get();
+
+        log::info!(
+            "hda: version={}.{}, iss={}, oss={}, bss={}",
+            vmaj,
+            vmin,
+            self.input_stream_count,
+            self.output_stream_count,
+            self.bidi_stream_count
+        );
+
+        regs.reset(
+            Duration::from_millis(100),
+            self.input_stream_count + self.output_stream_count + self.bidi_stream_count,
+        )?;
+
+        regs.disable_interrupts();
+        regs.disable_control();
+        let corb_size = regs.set_max_corb_size()?;
+        let rirb_size = regs.set_max_rirb_size()?;
+        log::info!("hda: corb size = {corb_size}, rirb size = {rirb_size}");
+
+        let (cring, corb_base, rirb_base) =
+            CommandRing::with_capacity(&*self.dma, corb_size, rirb_size)?;
+        regs.initialize_corb_rirb(corb_base, rirb_base)?;
+        regs.start_corb_rirb();
+        regs.enable_interrupts();
+
+        self.cring.init(cring);
+
+        drop(regs);
+
+        // Spawn softirq worker
+        let this = self.clone();
+        runtime::spawn(async move { this.softirq().await })?;
+
+        // Do the main initialization in background
+        let this = self.clone();
+        runtime::spawn(async move { this.late_init().await })?;
+
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "High Definition Audio"
+    }
+}
+
+pci_driver! {
+    matches: [
+        device (0x8086:0x2668), // Intel ICH6 82801FB/FBM/FR/FW/FRW HDA
+        device (0x8086:0x293E), // Intel ICH9 82801I HDA
+
+        device (0x1022:0x1487), // AMD Starship/Matisse HDA
+    ],
+    driver: {
+        fn probe(
+            &self,
+            info: &PciDeviceInfo,
+            dma: &Arc<dyn DmaAllocator>,
+        ) -> Result<Arc<dyn Device>, Error> {
+            info.set_command(true, true, false, true);
+
+            let base = info
+                .config_space
+                .bar(0)
+                .and_then(PciBaseAddress::as_memory)
+                .ok_or(Error::InvalidArgument)?;
+
+            info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+            let regs = unsafe { DeviceMemoryIo::<Regs>::map(base, Default::default()) }?;
+
+            let gcap = regs.GCAP.extract();
+            if gcap.matches_all(GCAP::OK64::CLEAR) {
+                log::error!("Non 64-bit HD Audio not supported yet");
+                return Err(Error::NotImplemented);
+            }
+            let input_stream_count = gcap.read(GCAP::ISS) as usize;
+            let output_stream_count = gcap.read(GCAP::OSS).min(30) as usize;
+            let bidi_stream_count = gcap.read(GCAP::BSS) as usize;
+
+            let hda = Arc::new(HdAudio::new(
+                info.clone(),
+                dma.clone(),
+                regs,
+                input_stream_count,
+                output_stream_count,
+                bidi_stream_count,
+            ));
+
+            Ok(hda)
+        }
+
+        fn driver_name(&self) -> &str {
+            "intel-hda"
+        }
+    }
+}

kernel/driver/sound/intel-hda/src/regs.rs

@@ -0,0 +1,453 @@
+use core::time::Duration;
+
+use libk::{
+    dma::BusAddress,
+    error::Error,
+    task::runtime::{psleep, pwait},
+};
+use tock_registers::{
+    fields::FieldValue,
+    interfaces::{ReadWriteable, Readable, Writeable},
+    register_bitfields, register_structs,
+    registers::{ReadOnly, ReadWrite},
+};
+use ygg_driver_sound_core::{SampleFormat, SampleRate, SinkFormat};
+
+use crate::stream::OutputStream;
+
+register_bitfields! {
+    u8,
+    pub CORBCTL [
+        /// Enable CORB DMA engine
+        CORBRUN OFFSET(1)   NUMBITS(1)  [],
+        /// CORB memory error interrupt enable
+        CMEIE   OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub CORBSTS [
+        /// CORB memory error
+        CMEI    OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub CORBSIZE [
+        CORBSZCAP   OFFSET(4)   NUMBITS(4)  [],
+        CORBSIZE    OFFSET(0)   NUMBITS(2)  [
+            Size2   = 0b00,
+            Size16  = 0b01,
+            Size256 = 0b10,
+        ],
+    ],
+    pub RIRBCTL [
+        /// RIRB overrun interrupt control
+        RURBOIC     OFFSET(2)   NUMBITS(1)  [],
+        /// Enable RIRB DMA engine
+        RIRBDMAEN   OFFSET(1)   NUMBITS(1)  [],
+        /// Response interrupt control
+        RINTCTL     OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub RIRBSTS [
+        /// RIRB overrun interrupt
+        RIRBOIS     OFFSET(2)   NUMBITS(1)  [],
+        /// Response interrupt
+        RINTFL      OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub RIRBSIZE [
+        /// RIRB size capability
+        RIRBSZCAP   OFFSET(4)   NUMBITS(4)  [],
+        RIRBSIZE    OFFSET(0)   NUMBITS(2)  [
+            Size2   = 0b00,
+            Size16  = 0b01,
+            Size256 = 0b10,
+        ],
+    ],
+
+    pub SDxCTL0 [
+        SRST    OFFSET(0)   NUMBITS(1)  [],
+        RUN     OFFSET(1)   NUMBITS(1)  [],
+        IOCE    OFFSET(2)   NUMBITS(1)  [],
+        FEIE    OFFSET(3)   NUMBITS(1)  [],
+        DEIE    OFFSET(4)   NUMBITS(1)  [],
+    ],
+    pub SDxCTL2 [
+        STRIPE  OFFSET(0)   NUMBITS(2)  [],
+        TP      OFFSET(2)   NUMBITS(1)  [],
+        DIR     OFFSET(3)   NUMBITS(1)  [],
+        STRM    OFFSET(4)   NUMBITS(4)  [],
+    ],
+    pub SDxSTS [
+        BCIS    OFFSET(2)   NUMBITS(1)  [],
+        FIFOE   OFFSET(3)   NUMBITS(1)  [],
+        DESE    OFFSET(4)   NUMBITS(1)  [],
+        FIFORDY OFFSET(5)   NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u16,
+    pub GCAP [
+        /// Number of output streams supported
+        OSS     OFFSET(12)  NUMBITS(4)  [],
+        /// Number of input streams supported
+        ISS     OFFSET(8)   NUMBITS(4)  [],
+        /// Number of bidirectional streams supported
+        BSS     OFFSET(3)   NUMBITS(5)  [],
+        /// Number of serial data out signals
+        NSDO    OFFSET(1)   NUMBITS(2)  [],
+        /// 64-bit addresses supported
+        OK64    OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub GSTS [
+        /// Flush status bit
+        FSTS    OFFSET(1)   NUMBITS(1)  [],
+    ],
+    pub CORBRP [
+        /// CORB read pointer reset
+        CORBRPRST   OFFSET(15)  NUMBITS(1)  [],
+        /// CORB read pointer
+        CORBRP      OFFSET(0)   NUMBITS(8)  [],
+    ],
+    pub RIRBWP [
+        /// RIRB write pointer reset
+        RIRBWPRST   OFFSET(15)  NUMBITS(1)  [],
+        /// RIRB write pointer
+        RIRBWP      OFFSET(0)   NUMBITS(8)  [],
+    ],
+
+    pub SDxFMT [
+        CHAN        OFFSET(0)   NUMBITS(4)  [],
+        BITS        OFFSET(4)   NUMBITS(3)  [
+            Bits8   = 0b000,
+            Bits16  = 0b001,
+            Bits20  = 0b010,
+            Bits24  = 0b011,
+            Bits32  = 0b100,
+        ],
+        DIV         OFFSET(8)   NUMBITS(3)  [],
+        MULT        OFFSET(11)  NUMBITS(3)  [],
+        BASE        OFFSET(14)  NUMBITS(1)  [],
+    ],
+}
+
+register_bitfields! {
+    u32,
+    pub GCTL [
+        /// Accept unsolicited response enable
+        UNSOL   OFFSET(8)   NUMBITS(1)  [],
+        /// Flush control
+        FCNTRL  OFFSET(1)   NUMBITS(1)  [],
+        /// Controller reset
+        CRST    OFFSET(0)   NUMBITS(1)  [],
+    ],
+    pub INTCTL [
+        /// Global interrupt enable
+        GIE     OFFSET(31)  NUMBITS(1)  [],
+        /// Controller interrupt enable
+        CIE     OFFSET(30)  NUMBITS(1)  [],
+        // Stream interrupt enable bits are based on stream counts
+    ],
+    pub INTSTS [
+        /// Global interrupt status
+        GIS     OFFSET(31)  NUMBITS(1)  [],
+        /// Controller interrupt status
+        CIS     OFFSET(30)  NUMBITS(1)  [],
+        // Stream interrupt status bits are based on stream counts
+    ],
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub Regs {
+        (0x0000 => pub GCAP: ReadOnly<u16, GCAP::Register>),
+        (0x0002 => pub VMIN: ReadOnly<u8>),
+        (0x0003 => pub VMAJ: ReadOnly<u8>),
+        (0x0004 => pub OUTPAY: ReadOnly<u16>),
+        (0x0006 => pub INPAY: ReadOnly<u16>),
+        (0x0008 => pub GCTL: ReadWrite<u32, GCTL::Register>),
+        (0x000C => pub WAKEEN: ReadWrite<u16>),
+        (0x000E => pub WAKESTS: ReadWrite<u16>),
+        (0x0010 => pub GSTS: ReadWrite<u16, GSTS::Register>),
+        (0x0012 => _0),
+        (0x0018 => pub OUTSTRMPAY: ReadOnly<u16>),
+        (0x001A => pub INSTRMPAY: ReadOnly<u16>),
+        (0x001C => _1),
+        (0x0020 => pub INTCTL: ReadWrite<u32, INTCTL::Register>),
+        (0x0024 => pub INTSTS: ReadOnly<u32, INTSTS::Register>),
+        (0x0028 => _2),
+        (0x0030 => pub WALCLK: ReadOnly<u32>),
+        (0x0034 => _3),
+        (0x0038 => pub SSYNC: ReadWrite<u32>),
+        (0x003C => _4),
+        (0x0040 => pub CORBLBASE: ReadWrite<u32>),
+        (0x0044 => pub CORBUBASE: ReadWrite<u32>),
+        (0x0048 => pub CORBWP: ReadWrite<u16>),
+        (0x004A => pub CORBRP: ReadWrite<u16, CORBRP::Register>),
+        (0x004C => pub CORBCTL: ReadWrite<u8, CORBCTL::Register>),
+        (0x004D => pub CORBSTS: ReadWrite<u8, CORBSTS::Register>),
+        (0x004E => pub CORBSIZE: ReadWrite<u8, CORBSIZE::Register>),
+        (0x004F => _5),
+        (0x0050 => pub RIRBLBASE: ReadWrite<u32>),
+        (0x0054 => pub RIRBUBASE: ReadWrite<u32>),
+        (0x0058 => pub RIRBWP: ReadWrite<u16, RIRBWP::Register>),
+        (0x005A => pub RINTCNT: ReadWrite<u16>),
+        (0x005C => pub RIRBCTL: ReadWrite<u8, RIRBCTL::Register>),
+        (0x005D => pub RIRBSTS: ReadWrite<u8, RIRBSTS::Register>),
+        (0x005E => pub RIRBSIZE: ReadWrite<u8, RIRBSIZE::Register>),
+        (0x005F => _6),
+        (0x0060 => pub ICOI: ReadWrite<u32>),
+        (0x0064 => pub ICII: ReadWrite<u32>),
+        (0x0068 => pub ICIS: ReadOnly<u16>),
+        (0x006A => _7),
+        (0x0070 => pub DPLBASE: ReadWrite<u32>),
+        (0x0074 => pub DPUBASE: ReadWrite<u32>),
+        (0x0078 => _8),
+        (0x0080 => pub STREAMS: [StreamRegs; 128]),
+        (0x1080 => _9),
+        (0x2030 => pub WALCLKA: ReadOnly<u32>),
+        (0x2034 => _10),
+        (0x2080 => pub SDxLPIBA: [StreamLinkRegs; 128]),
+        (0x3080 => _11),
+        (0x4000 => @END),
+    }
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub StreamRegs {
+        (0x00 => pub SDxCTL0: ReadWrite<u8, SDxCTL0::Register>),
+        (0x01 => pub SDxCTL1: ReadWrite<u8>),
+        (0x02 => pub SDxCTL2: ReadWrite<u8, SDxCTL2::Register>),
+        (0x03 => pub SDxSTS: ReadWrite<u8, SDxSTS::Register>),
+        (0x04 => pub SDxLPIB: ReadWrite<u32>),
+        (0x08 => pub SDxCBL: ReadWrite<u32>),
+        (0x0C => pub SDxLVI: ReadWrite<u16>),
+        (0x0E => _0),
+        (0x10 => pub SDxFIFOD: ReadWrite<u16>),
+        (0x12 => pub SDxFMT: ReadWrite<u16, SDxFMT::Register>),
+        (0x14 => _1),
+        (0x18 => pub SDxBDPL: ReadWrite<u32>),
+        (0x1C => pub SDxBDPU: ReadWrite<u32>),
+        (0x20 => @END),
+    }
+}
+
+register_structs! {
+    #[allow(non_snake_case)]
+    pub StreamLinkRegs {
+        (0x00 => _0),
+        (0x04 => pub SDxLPIBA: ReadOnly<u32>),
+        (0x08 => _1),
+        (0x20 => @END),
+    }
+}
+
+impl Regs {
+    pub fn reset(&self, timeout: Duration, stream_count: usize) -> Result<(), Error> {
+        // Go through all the streams, disable their DMA
+        for i in 0..stream_count {
+            let stream = &self.STREAMS[i];
+            stream
+                .SDxCTL0
+                .write(SDxCTL0::RUN::CLEAR + SDxCTL0::SRST::SET);
+        }
+
+        self.CORBCTL.write(CORBCTL::CORBRUN::CLEAR);
+        self.RIRBCTL.write(RIRBCTL::RIRBDMAEN::CLEAR);
+
+        // Flush the FIFO
+        self.GCTL.modify(GCTL::FCNTRL::SET);
+        psleep(Duration::from_millis(10));
+        pwait(timeout, Duration::from_millis(10), || {
+            self.GSTS.matches_all(GSTS::FSTS::SET)
+        })?;
+        self.GCTL.modify(GCTL::FCNTRL::CLEAR);
+        self.GSTS.write(GSTS::FSTS::SET);
+
+        // TODO does FIFO need to be flushed here as well?
+        // Begin controller reset
+        self.GCTL.write(GCTL::CRST::CLEAR);
+        psleep(Duration::from_millis(10));
+        // End controller reset
+        self.GCTL.write(GCTL::CRST::SET);
+        psleep(Duration::from_millis(10));
+        pwait(timeout, Duration::from_millis(10), || {
+            self.GCTL.matches_all(GCTL::CRST::SET)
+        })?;
+
+        Ok(())
+    }
+
+    pub fn disable_interrupts(&self) {
+        self.INTCTL.set(0);
+    }
+
+    pub fn enable_interrupts(&self) {
+        // Also accept unsolicited responses from codecs
+        self.GCTL.modify(GCTL::UNSOL::SET);
+        self.INTCTL.write(INTCTL::GIE::SET + INTCTL::CIE::SET);
+    }
+
+    pub fn enable_stream_interrupts(&self, stream: usize) {
+        self.INTCTL.set(self.INTCTL.get() | (1 << stream));
+    }
+
+    pub fn disable_control(&self) {
+        self.DPLBASE.set(0);
+        self.DPUBASE.set(0);
+
+        self.CORBCTL.write(CORBCTL::CORBRUN::CLEAR);
+        self.RIRBCTL.write(RIRBCTL::RIRBDMAEN::CLEAR);
+    }
+
+    pub fn set_max_corb_size(&self) -> Result<usize, Error> {
+        const SIZES: &[(usize, u8, FieldValue<u8, CORBSIZE::Register>)] = &[
+            (256, 0b0100, CORBSIZE::CORBSIZE::Size256),
+            (16, 0b0010, CORBSIZE::CORBSIZE::Size16),
+            (2, 0b0001, CORBSIZE::CORBSIZE::Size2),
+        ];
+
+        let corbsize = self.CORBSIZE.extract();
+        let cap = corbsize.read(CORBSIZE::CORBSZCAP);
+        for &(size, bit, cfg) in SIZES {
+            if cap & bit != 0 {
+                self.CORBSIZE.modify_no_read(corbsize, cfg);
+                return Ok(size);
+            }
+        }
+
+        log::error!("hda: no supported CORB size");
+        Err(Error::InvalidOperation)
+    }
+
+    pub fn set_max_rirb_size(&self) -> Result<usize, Error> {
+        const SIZES: &[(usize, u8, FieldValue<u8, RIRBSIZE::Register>)] = &[
+            (256, 0b0100, RIRBSIZE::RIRBSIZE::Size256),
+            (16, 0b0010, RIRBSIZE::RIRBSIZE::Size16),
+            (2, 0b0001, RIRBSIZE::RIRBSIZE::Size2),
+        ];
+
+        let rirbsize = self.RIRBSIZE.extract();
+        let cap = rirbsize.read(RIRBSIZE::RIRBSZCAP);
+        for &(size, bit, cfg) in SIZES {
+            if cap & bit != 0 {
+                self.RIRBSIZE.modify_no_read(rirbsize, cfg);
+                return Ok(size);
+            }
+        }
+
+        log::error!("hda: no supported RIRB size");
+        Err(Error::InvalidOperation)
+    }
+
+    pub fn initialize_corb_rirb(
+        &self,
+        corb_base: BusAddress,
+        rirb_base: BusAddress,
+    ) -> Result<(), Error> {
+        let corb_base = corb_base.into_u64();
+        let rirb_base = rirb_base.into_u64();
+
+        self.CORBUBASE.set((corb_base >> 32) as u32);
+        self.CORBLBASE.set(corb_base as u32);
+
+        self.RIRBUBASE.set((rirb_base >> 32) as u32);
+        self.RIRBLBASE.set(rirb_base as u32);
+
+        // Reset write/read pointers
+        self.CORBWP.set(0);
+        self.CORBRP.write(CORBRP::CORBRPRST::SET);
+        self.RIRBSTS
+            .write(RIRBSTS::RIRBOIS::SET + RIRBSTS::RINTFL::SET);
+        self.RINTCNT.set(255);
+
+        // Ensure CORBRP is reset properly
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.CORBRP.matches_all(CORBRP::CORBRPRST::SET),
+        )?;
+        self.CORBRP.write(CORBRP::CORBRPRST::CLEAR);
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || self.CORBRP.matches_all(CORBRP::CORBRPRST::CLEAR),
+        )?;
+
+        Ok(())
+    }
+
+    pub fn start_corb_rirb(&self) {
+        self.CORBCTL
+            .modify(CORBCTL::CORBRUN::SET + CORBCTL::CMEIE::SET);
+        self.RIRBCTL
+            .modify(RIRBCTL::RIRBDMAEN::SET + RIRBCTL::RINTCTL::SET);
+    }
+
+    pub fn take_attached_codecs(&self) -> u16 {
+        let state = self.WAKESTS.get();
+        self.WAKESTS.set(0xFFFF);
+        state
+    }
+
+    pub fn configure_stream(
+        &self,
+        index: usize,
+        config: &SinkFormat,
+        bdl_base: BusAddress,
+        buffer_size: usize,
+        buffer_count: usize,
+        number: u8,
+    ) -> Result<(), Error> {
+        log::info!("Configure stream: #{index}");
+        let bdl_base = bdl_base.into_u64();
+
+        // TODO don't use pwait in async context
+        let regs = &self.STREAMS[index];
+
+        // Stop and reset the stream
+        regs.SDxCTL0.write(SDxCTL0::SRST::SET);
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || regs.SDxCTL0.matches_all(SDxCTL0::SRST::SET),
+        )?;
+        regs.SDxCTL0.write(SDxCTL0::SRST::CLEAR);
+        pwait(
+            Duration::from_millis(100),
+            Duration::from_millis(10),
+            || regs.SDxCTL0.matches_all(SDxCTL0::SRST::CLEAR),
+        )?;
+
+        regs.SDxCTL2.write(SDxCTL2::STRM.val(number));
+        regs.SDxSTS.write(SDxSTS::BCIS::SET);
+
+        regs.SDxLVI.set(buffer_count as u16 - 1);
+        regs.SDxCBL.set((buffer_count * buffer_size) as _);
+        regs.SDxBDPU.set((bdl_base >> 32) as u32);
+        regs.SDxBDPL.set(bdl_base as u32);
+
+        let mut format = SDxFMT::CHAN.val(config.channels as u16 - 1);
+
+        match config.sample_format {
+            SampleFormat::S8 => format += SDxFMT::BITS::Bits8,
+            SampleFormat::S16Le => format += SDxFMT::BITS::Bits16,
+        }
+        match config.sample_rate {
+            SampleRate::Rate8000 => format += SDxFMT::DIV.val(5),
+            SampleRate::Rate11025 => format += SDxFMT::DIV.val(3) + SDxFMT::BASE::SET,
+            SampleRate::Rate16000 => format += SDxFMT::DIV.val(2),
+            SampleRate::Rate22050 => format += SDxFMT::DIV.val(1) + SDxFMT::BASE::SET,
+            SampleRate::Rate32000 => format += SDxFMT::DIV.val(2) + SDxFMT::MULT.val(1),
+            SampleRate::Rate44100 => format += SDxFMT::BASE::SET,
+            SampleRate::Rate48000 => (),
+            SampleRate::Rate88200 => format += SDxFMT::MULT.val(1) + SDxFMT::BASE::SET,
+            SampleRate::Rate96000 => format += SDxFMT::MULT.val(1),
+            SampleRate::Rate176400 => format += SDxFMT::MULT.val(3) + SDxFMT::BASE::SET,
+            SampleRate::Rate192000 => format += SDxFMT::MULT.val(3),
+            SampleRate::Rate384000 => todo!(),
+        }
+
+        regs.SDxFMT.write(format);
+
+        regs.SDxCTL0.modify(SDxCTL0::IOCE::SET);
+
+        Ok(())
+    }
+}

kernel/driver/sound/intel-hda/src/ring.rs

@@ -0,0 +1,396 @@
+use core::{future::poll_fn, task::Poll};
+
+use device_api::dma::DmaAllocator;
+use futures_util::task::AtomicWaker;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+};
+use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker};
+use tock_registers::register_bitfields;
+use yggdrasil_abi::primitive_enum;
+
+// 4 bit address
+const MAX_CODEC: usize = 16;
+
+// Combines CORB and RIRB
+struct Inner {
+    corb: DmaBuffer<[u32]>,
+    rirb: DmaBuffer<[(u32, u32)]>,
+
+    codec_responses: [Option<u32>; MAX_CODEC],
+
+    corb_head: u32,
+    corb_tail: u32,
+    rirb_tail: u32,
+}
+
+pub struct CommandRing {
+    inner: IrqSafeSpinlock<Inner>,
+
+    // 16 possible codec addresses
+    codec_notify: [AtomicWaker; MAX_CODEC],
+    // Unsolicited response notify
+    unsol_notify: QueueWaker,
+}
+
+primitive_enum! {
+    pub enum Verb: u32 {
+        GetParameter = 0xF00,
+        GetConnectionListEntry = 0xF02,
+        SetSelectedInput = 0x701,
+        SetPowerState = 0x705,
+        SetConverterStreamChannel = 0x706,
+        SetPinWidgetControl = 0x707,
+        GetPinWidgetControl = 0xF07,
+        GetPinWidgetSense = 0xF09,
+        SetUnsolicitedResponse = 0x708,
+        GetUnsolicitedResponse = 0xF08,
+        SetEapdBtl = 0x70C,
+        GetPinWidgetDefaultConfig = 0xF1C,
+        SetOutputConverterChannelCount = 0x72D,
+        AudioFunctionNodeReset = 0x7FF,
+        SetAmplifierGain = 0x003,
+        SetStreamConverterFormat = 0x002,
+        SetStreamFormat = 0x200,
+    }
+}
+
+primitive_enum! {
+    pub enum NodeParameterNumber: u32 {
+        DeviceId = 0x00,
+        RevisionId = 0x02,
+        NodeCount = 0x04,
+        FunctionGroupType = 0x05,
+        AudioGroupCapabilities = 0x08,
+        AudioWidgetCapabilities = 0x09,
+        SupportedPcmRates = 0x0A,
+        SupportedFormats = 0x0B,
+        PinCapabilities = 0x0C,
+        InputAmplifierCapabilities = 0x0D,
+        OutputAmplifierCapabilities = 0x12,
+        ConnectionListLength = 0x0E,
+        SupportedPowerStates = 0x0F,
+        ProcessingCapabilities = 0x10,
+        GpioCount = 0x11,
+        VolumeCapabilities = 0x13,
+    }
+}
+
+register_bitfields! {
+    u32,
+    pub PinControl [
+        VRefEn      OFFSET(0)   NUMBITS(3)  [],
+        InEnable    OFFSET(5)   NUMBITS(1)  [],
+        OutEnable   OFFSET(6)   NUMBITS(1)  [],
+        HPhnEnable  OFFSET(7)   NUMBITS(1)  [],
+    ],
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinColor {
+    Black,
+    Gray,
+    Blue,
+    Green,
+    Red,
+    Orange,
+    Yellow,
+    Purple,
+    Pink,
+    White,
+    Other,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinType {
+    I1_8,
+    I1_4,
+    AtapiInternal,
+    Rca,
+    Optical,
+    OtherDigital,
+    OtherAnalog,
+    MultichannelAnalog,
+    Xlr,
+    Rj11,
+    Combination,
+    Other,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinDevice {
+    LineOut,
+    Speaker,
+    HeadphoneOut,
+    Cd,
+    SpdifOut,
+    OtherDigitalOut,
+    ModemLineSide,
+    ModemHandsetSide,
+    LineIn,
+    Aux,
+    MicIn,
+    Telephony,
+    SpdifIn,
+    OtherDigitalIn,
+    Other,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PinConnectivity {
+    None,
+    Jack,
+    Fixed,
+    Both,
+}
+
+#[derive(Debug)]
+pub struct PinDefaultConfig {
+    pub sequence: u8,
+    pub default_association: u8,
+    pub color: PinColor,
+    pub connection_type: PinType,
+    pub default_device: PinDevice,
+    pub connectivity: PinConnectivity,
+}
+
+#[derive(Clone, Copy)]
+pub struct Command(u32);
+
+impl Inner {
+    // Submit a command, returns new head and command token
+    pub fn push(&mut self, command: Command) -> Option<u8> {
+        if (self.corb_head + 1) % (self.corb.len() as u32) == self.corb_tail {
+            return None;
+        }
+        self.corb_head = (self.corb_head + 1) % self.corb.len() as u32;
+        let index = self.corb_head as usize % self.corb.len();
+        self.corb[index] = command.0;
+        self.corb.cache_flush_element(index, true);
+        Some(index as u8)
+    }
+
+    pub fn process_responses<U: Fn(u8, u32), C: Fn(u8)>(
+        &mut self,
+        unsol_handler: U,
+        codec_handler: C,
+        corb_tail: u32,
+        rirb_head: u32,
+    ) -> usize {
+        self.corb_tail = corb_tail;
+
+        let mut count = 0;
+        while self.rirb_tail != rirb_head {
+            self.rirb_tail = (self.rirb_tail + 1) % self.rirb.len() as u32;
+
+            let index = self.rirb_tail as usize;
+            self.rirb.cache_flush_element(index, false);
+            let (w0, w1) = self.rirb[index];
+            let codec = (w1 & 0xF) as u8;
+            if w1 & (1 << 4) == 0 {
+                // Solicited
+                self.codec_responses[codec as usize] = Some(w0);
+                codec_handler(codec);
+            } else {
+                // Unsolicited
+                unsol_handler(codec, w0);
+            }
+
+            count += 1;
+        }
+        count
+    }
+
+    pub fn take_codec_response(&mut self, codec: u8) -> Option<u32> {
+        self.codec_responses[codec as usize].take()
+    }
+}
+
+impl CommandRing {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        corb_size: usize,
+        rirb_size: usize,
+    ) -> Result<(Self, BusAddress, BusAddress), Error> {
+        let corb = DmaBuffer::new_slice(dma, 0, corb_size)?;
+        let rirb = DmaBuffer::new_slice(dma, (0, 0), rirb_size)?;
+        let corb_base = corb.bus_address();
+        let rirb_base = rirb.bus_address();
+        Ok((
+            Self {
+                inner: IrqSafeSpinlock::new(Inner {
+                    corb,
+                    rirb,
+                    corb_head: 0,
+                    corb_tail: 0,
+                    rirb_tail: 0,
+
+                    codec_responses: [None; MAX_CODEC],
+                }),
+
+                codec_notify: [const { AtomicWaker::new() }; 16],
+                unsol_notify: QueueWaker::new(),
+            },
+            corb_base,
+            rirb_base,
+        ))
+    }
+
+    // Wait for a specific codec to send its response
+    pub async fn wait_codec(&self, codec: u8) -> u32 {
+        poll_fn(|cx| {
+            if let Some(response) = self.inner.lock().take_codec_response(codec) {
+                Poll::Ready(response)
+            } else {
+                self.codec_notify[codec as usize].register(cx.waker());
+                Poll::Pending
+            }
+        })
+        .await
+    }
+
+    pub async fn submit_command(&self, command: Command) -> Result<u8, Error> {
+        // TODO block and wait for free slots in CORB
+        let mut inner = self.inner.lock();
+        // Clear previous codec response
+        let head = inner.push(command).ok_or(Error::WouldBlock)?;
+        inner.codec_responses[command.codec() as usize] = None;
+        Ok(head)
+    }
+
+    pub(super) fn process_completions(&self, corb_tail: u8, rirb_head: u8) -> usize {
+        let mut inner = self.inner.lock();
+        inner.process_responses(
+            |codec, message| {
+                log::info!("Unsolicited message: {message:#x} from codec {codec:#x}");
+            },
+            |codec| self.codec_notify[codec as usize].wake(),
+            corb_tail as u32,
+            rirb_head as u32,
+        )
+    }
+}
+
+impl Command {
+    pub const fn new(codec: u8, node: u8, verb: Verb, payload: u32) -> Self {
+        Self(((codec as u32) << 28) | ((node as u32) << 20) | ((verb as u32) << 8) | payload)
+    }
+
+    pub const fn get_parameter(codec: u8, node: u8, parameter: NodeParameterNumber) -> Self {
+        Self::new(codec, node, Verb::GetParameter, parameter as u32)
+    }
+
+    pub const fn get_connection_list_entry(codec: u8, node: u8, index: usize) -> Self {
+        Self::new(codec, node, Verb::GetConnectionListEntry, index as u32)
+    }
+
+    pub const fn set_stream_number(codec: u8, node: u8, stream: usize) -> Self {
+        Self::new(codec, node, Verb::SetConverterStreamChannel, stream as u32)
+    }
+
+    pub fn codec(&self) -> u8 {
+        ((self.0 >> 28) & 0xF) as u8
+    }
+}
+
+impl From<u32> for PinConnectivity {
+    fn from(value: u32) -> Self {
+        match value {
+            0b00 => Self::Jack,
+            0b10 => Self::Fixed,
+            0b11 => Self::Both,
+            _ => Self::None,
+        }
+    }
+}
+
+impl From<u32> for PinColor {
+    fn from(value: u32) -> Self {
+        match value {
+            1 => Self::Black,
+            2 => Self::Gray,
+            3 => Self::Blue,
+            4 => Self::Green,
+            5 => Self::Red,
+            6 => Self::Orange,
+            7 => Self::Yellow,
+            8 => Self::Purple,
+            9 => Self::Pink,
+            14 => Self::White,
+            _ => Self::Other,
+        }
+    }
+}
+
+impl From<u32> for PinType {
+    fn from(value: u32) -> Self {
+        match value {
+            1 => Self::I1_8,
+            2 => Self::I1_4,
+            3 => Self::AtapiInternal,
+            4 => Self::Rca,
+            5 => Self::Optical,
+            6 => Self::OtherDigital,
+            7 => Self::OtherAnalog,
+            8 => Self::MultichannelAnalog,
+            9 => Self::Xlr,
+            10 => Self::Rj11,
+            11 => Self::Combination,
+            _ => Self::Other,
+        }
+    }
+}
+
+impl PinDevice {
+    pub fn output_score(&self) -> u32 {
+        match self {
+            Self::HeadphoneOut => 10,
+            Self::Speaker => 9,
+            Self::LineOut => 8,
+            Self::SpdifOut => 7,
+            _ => 0,
+        }
+    }
+}
+impl From<u32> for PinDevice {
+    fn from(value: u32) -> Self {
+        match value {
+            0 => Self::LineOut,
+            1 => Self::Speaker,
+            2 => Self::HeadphoneOut,
+            3 => Self::Cd,
+            4 => Self::SpdifOut,
+            5 => Self::OtherDigitalOut,
+            6 => Self::ModemLineSide,
+            7 => Self::ModemHandsetSide,
+            8 => Self::LineIn,
+            9 => Self::Aux,
+            10 => Self::MicIn,
+            11 => Self::Telephony,
+            12 => Self::SpdifIn,
+            13 => Self::OtherDigitalIn,
+            _ => Self::Other,
+        }
+    }
+}
+
+impl From<u32> for PinDefaultConfig {
+    fn from(value: u32) -> Self {
+        let sequence = (value & 0xF) as u8;
+        let default_association = ((value >> 4) & 0xF) as u8;
+        let color = PinColor::from((value >> 12) & 0xF);
+        let connection_type = PinType::from((value >> 16) & 0xF);
+        let default_device = PinDevice::from((value >> 20) & 0xF);
+        let connectivity = PinConnectivity::from(value >> 30);
+
+        Self {
+            sequence,
+            color,
+            connectivity,
+            connection_type,
+            default_device,
+            default_association,
+        }
+    }
+}

kernel/driver/sound/intel-hda/src/sink.rs

@@ -0,0 +1,171 @@
+use core::{
+    sync::atomic::{AtomicBool, Ordering},
+    task::{Context, Poll},
+    time::Duration,
+};
+
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use device_api::device::Device;
+use futures_util::task::AtomicWaker;
+use libk::{device::char::CharDevice, error::Error, fs::devfs, task::runtime, vfs::FileReadiness};
+use libk_mm::PageBox;
+use libk_util::{event::BoolEvent, sync::spin_rwlock::IrqSafeRwLock};
+use tock_registers::interfaces::ReadWriteable;
+use ygg_driver_sound_core::{
+    AudioSink, SampleFormat, SampleRate, SinkFormat, SinkSupportedFormats,
+};
+use yggdrasil_abi::io::FileMode;
+
+use crate::{
+    codec::{Codec, Node},
+    regs::SDxCTL0,
+    ring::Verb,
+    stream::OutputStream,
+    HdAudio,
+};
+
+pub struct HdAudioSink {
+    hda: Arc<HdAudio>,
+    codec: Arc<Codec>,
+    audio_widget: Node,
+    pub index: usize,
+    stream_tag: u8,
+    stream: OutputStream,
+    notify: BoolEvent,
+    config: IrqSafeRwLock<SinkFormat>,
+    playing: AtomicBool,
+}
+
+impl HdAudioSink {
+    pub async fn create(
+        hda: Arc<HdAudio>,
+        index: usize,
+        stream_tag: u8,
+        codec: Arc<Codec>,
+        audio_widget: Node,
+    ) -> Result<Arc<Self>, Error> {
+        // TODO use default/current stream format (if PCM), don't always set 48KHz S16 x2
+        log::info!("hda: create sink #{index}, tag {stream_tag}");
+        let config = SinkFormat {
+            sample_rate: SampleRate::Rate48000,
+            sample_format: SampleFormat::S16Le,
+            channels: 2,
+        };
+        let one_millisecond =
+            (config.sample_rate as usize * config.sample_format.sample_size() * config.channels)
+                / 1000;
+        let bdl_capacity = 32;
+        log::info!("buffer_count = {bdl_capacity}, buffer_size = {one_millisecond}");
+        let (stream, bdl_address) = OutputStream::new(hda.clone(), one_millisecond, bdl_capacity)?;
+
+        // TODO check if the widget actually supports such format
+        audio_widget
+            .perform_command(&*hda, Verb::SetStreamFormat, (1 << 4) | 1)
+            .await?;
+
+        {
+            let regs = hda.regs.lock();
+            regs.configure_stream(
+                index,
+                &config,
+                bdl_address,
+                one_millisecond,
+                bdl_capacity,
+                stream_tag,
+            )?;
+            regs.enable_stream_interrupts(index);
+        }
+
+        audio_widget.set_stream(&*hda, stream_tag, 0).await?;
+
+        let this = Arc::new(Self {
+            hda,
+            codec,
+            audio_widget,
+            index,
+            stream_tag,
+            stream,
+            notify: BoolEvent::new(),
+            playing: AtomicBool::new(false),
+            config: IrqSafeRwLock::new(config),
+        });
+
+        ygg_driver_sound_core::register_audio_sink(this.clone());
+
+        Ok(this)
+    }
+
+    pub async fn handle_softirq(&self, position: u32) {
+        self.stream.update_tail(position);
+        self.notify.signal_saturating();
+    }
+
+    async fn write_blocking(&self, data: &[u8]) -> Result<(), Error> {
+        let mut position = 0;
+        while position != data.len() {
+            let written = self.stream.write(&data[position..]);
+            position += written;
+            if written == 0 {
+                self.notify.wait_reset().await;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl AudioSink for HdAudioSink {
+    async fn set_format(&self, format: SinkFormat) -> Result<(), Error> {
+        todo!()
+    }
+    async fn current_format(&self) -> Result<SinkFormat, Error> {
+        todo!()
+    }
+    async fn supported_formats(&self) -> Result<SinkSupportedFormats, Error> {
+        todo!()
+    }
+
+    async fn write(&self, data: &[u8]) -> Result<usize, Error> {
+        let amount = data.len().min(48000 * 2);
+        if !self.playing.swap(true, Ordering::Acquire) {
+            self.start().await?;
+        }
+        self.write_blocking(&data[..amount]).await?;
+        Ok(data.len())
+    }
+    async fn start(&self) -> Result<(), Error> {
+        self.playing.store(true, Ordering::Release);
+        let regs = self.hda.regs.lock();
+        log::info!("hda: start stream #{}", self.index);
+        self.stream.reset();
+        regs.STREAMS[self.index].SDxCTL0.modify(SDxCTL0::RUN::SET);
+        Ok(())
+    }
+    fn stop(&self) -> Result<(), Error> {
+        self.playing.store(false, Ordering::Release);
+        let regs = self.hda.regs.lock();
+        log::info!("hda: stop stream #{}", self.index);
+        regs.STREAMS[self.index].SDxCTL0.modify(SDxCTL0::RUN::CLEAR);
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl CharDevice for HdAudioSink {
+    async fn write(&self, buffer: &[u8]) -> Result<usize, Error> {
+        AudioSink::write(self, buffer).await
+    }
+}
+
+impl FileReadiness for HdAudioSink {
+    fn poll_read(&self, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        todo!()
+    }
+}
+
+impl Device for HdAudioSink {
+    fn display_name(&self) -> &str {
+        "HD Audio Sink"
+    }
+}

kernel/driver/sound/intel-hda/src/stream.rs

@@ -0,0 +1,135 @@
+use alloc::{sync::Arc, vec::Vec};
+use device_api::dma::DmaAllocator;
+use libk::{
+    dma::{BusAddress, DmaBuffer},
+    error::Error,
+};
+use libk_util::{
+    event::BoolEvent,
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
+};
+use ygg_driver_sound_core::SinkFormat;
+
+use crate::HdAudio;
+
+#[derive(Debug)]
+#[repr(C)]
+pub struct BufferDescriptor {
+    address: BusAddress,
+    length: u32,
+    flags: u32,
+}
+
+pub struct BufferDescriptorList {
+    entries: DmaBuffer<[BufferDescriptor]>,
+    periods: Vec<DmaBuffer<[u8]>>,
+    wr: usize,
+    rd: usize,
+    buffer_size: usize,
+}
+
+pub enum StreamKind {
+    Input,
+    Output,
+    Bidi,
+}
+
+pub struct OutputStream {
+    pub hda: Arc<HdAudio>,
+    pub bdl: IrqSafeSpinlock<BufferDescriptorList>,
+}
+
+impl BufferDescriptorList {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        capacity: usize,
+        frame_size: usize,
+    ) -> Result<Self, Error> {
+        let periods: Vec<DmaBuffer<[u8]>> = (0..capacity)
+            .map(|_| DmaBuffer::new_slice(dma, 0, frame_size))
+            .collect::<Result<_, _>>()?;
+        let entries =
+            DmaBuffer::new_slice_with(dma, |i| BufferDescriptor::new(&periods[i]), capacity)?;
+        Ok(Self {
+            entries,
+            wr: 0,
+            rd: 0,
+            periods,
+            buffer_size: frame_size,
+        })
+    }
+
+    pub fn write(&mut self, data: &[u8]) -> usize {
+        let capacity = self.periods.len() * self.buffer_size;
+
+        let mut position = 0;
+        while (self.wr + 1) % capacity != self.rd && position != data.len() {
+            let p = self.wr / self.buffer_size;
+            let o = self.wr % self.buffer_size;
+
+            self.periods[p][o] = data[position];
+
+            position += 1;
+            self.wr = (self.wr + 1) % capacity;
+        }
+
+        position
+    }
+
+    pub fn update_tail(&mut self, tail: u32) {
+        self.rd = tail as usize;
+    }
+
+    pub fn reset(&mut self) {
+        self.wr = self.rd;
+    }
+}
+
+impl BufferDescriptor {
+    pub fn new(buffer: &DmaBuffer<[u8]>) -> Self {
+        Self {
+            address: buffer.bus_address(),
+            length: buffer.len() as _,
+            flags: 1,
+        }
+    }
+}
+
+impl OutputStream {
+    pub fn new(
+        hda: Arc<HdAudio>,
+        buffer_size: usize,
+        bdl_capacity: usize,
+    ) -> Result<(Self, BusAddress), Error> {
+        let bdl = BufferDescriptorList::with_capacity(&*hda.dma, bdl_capacity, buffer_size)?;
+        let bdl_base = bdl.entries.bus_address();
+        Ok((
+            Self {
+                hda,
+                bdl: IrqSafeSpinlock::new(bdl),
+            },
+            bdl_base,
+        ))
+    }
+
+    pub fn params(&self) -> (BusAddress, usize, usize) {
+        let bdl = self.bdl.lock();
+        (
+            bdl.entries.bus_address(),
+            bdl.buffer_size,
+            bdl.entries.len(),
+        )
+    }
+
+    pub fn reset(&self) {
+        self.bdl.lock().reset()
+    }
+
+    pub fn write(&self, bytes: &[u8]) -> usize {
+        self.bdl.lock().write(bytes)
+    }
+
+    pub fn update_tail(&self, tail: u32) {
+        self.bdl.lock().update_tail(tail)
+    }
+}

kernel/driver/usb/xhci/src/lib.rs

@@ -9,7 +9,7 @@ use controller::Xhci;
 use device_api::{device::Device, dma::DmaAllocator, interrupt::InterruptAffinity};
 use regs::Regs;
 use ygg_driver_pci::{
-    capability::{DevicePowerState, PowerManagementCapability},
+    capability::{power::DevicePowerState, PowerManagementCapability},
     device::{PciDeviceInfo, PreferredInterruptMode},
     macros::pci_driver,
     PciCommandRegister, PciConfigurationSpace,

kernel/driver/virtio/blk/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "ygg_driver_virtio_blk"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+yggdrasil-abi.workspace = true
+libk-util.workspace = true
+libk-mm.workspace = true
+libk.workspace = true
+device-api = { workspace = true, features = ["derive"] }
+
+ygg_driver_virtio_core = { path = "../core" }
+ygg_driver_pci = { path = "../../bus/pci", optional = true }
+
+log.workspace = true
+bytemuck.workspace = true
+tock-registers.workspace = true
+async-trait.workspace = true
+
+[features]
+default = []
+pci = ["ygg_driver_pci", "ygg_driver_virtio_core/pci"]

kernel/driver/virtio/blk/src/lib.rs

@@ -0,0 +1,414 @@
+#![no_std]
+
+use core::mem::MaybeUninit;
+
+use alloc::{boxed::Box, format, sync::Arc, vec::Vec};
+use async_trait::async_trait;
+use bytemuck::{Pod, Zeroable};
+use device_api::{
+    device::{Device, DeviceInitContext},
+    dma::DmaAllocator,
+    interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
+};
+use libk::{
+    device::{block::BlockDevice, manager::probe_partitions},
+    dma::{DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+    fs::devfs,
+    task::runtime,
+};
+use libk_mm::{
+    address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage,
+};
+use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock};
+use ygg_driver_pci::{
+    device::{PciDeviceInfo, PreferredInterruptMode},
+    macros::pci_driver,
+};
+use ygg_driver_virtio_core::{
+    queue::VirtQueue,
+    transport::{pci::PciTransport, Transport},
+    DeviceStatus,
+};
+use yggdrasil_abi::{bitflags, io::FileMode};
+
+extern crate alloc;
+
+bitflags! {
+    pub struct Features: u64 {
+        const F_SIZE_MAX: bit 1;
+        const F_SEG_MAX: bit 2;
+        const F_RO: bit 5;
+    }
+}
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+struct DeviceConfig {
+    capacity: u64,
+    size_max: u32,
+    seg_max: u32,
+    // virtio_blk_geometry {
+    cylinders: u16,
+    heads: u8,
+    sectors: u8,
+    // }
+    blk_size: u32,
+    // virtio_blk_topology {
+    physical_block_exp: u8,
+    alignment_offset: u8,
+    min_io_size: u16,
+    opt_io_size: u32,
+    // }
+    writeback: u8,
+    _0: u8,
+    num_queues: u16,
+    max_discard_sectors: u32,
+    max_discard_seg: u32,
+    discard_sector_alignment: u32,
+    max_write_zeroes_sectors: u32,
+    max_write_zeroes_seg: u32,
+    write_zeroes_may_unmap: u8,
+    _1: [u8; 3],
+    max_secure_erase_sectors: u32,
+    max_secure_erase_seg: u32,
+    secure_erase_sector_alignment: u32,
+}
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+struct CommandHeader {
+    ty: u32,
+    _0: u32,
+    sector: u64,
+}
+
+pub struct VirtioBlk<T: Transport + 'static> {
+    transport: IrqSafeSpinlock<T>,
+    pci_device_info: PciDeviceInfo,
+    dma: Arc<dyn DmaAllocator>,
+
+    segment_size: usize,
+    read_only: bool,
+    capacity: u64,
+    request_queue: VirtQueue,
+}
+
+impl CommandHeader {
+    const TYPE_READ: u32 = 0;
+    const TYPE_WRITE: u32 = 1;
+
+    pub fn for_read(lba: u64) -> Self {
+        Self {
+            ty: Self::TYPE_READ,
+            _0: 0,
+            sector: lba,
+        }
+    }
+
+    pub fn for_write(lba: u64) -> Self {
+        Self {
+            ty: Self::TYPE_WRITE,
+            _0: 0,
+            sector: lba,
+        }
+    }
+}
+
+impl<T: Transport + 'static> VirtioBlk<T> {
+    // Only one VQ
+    const VQ_REQUEST_0: u16 = 0;
+
+    fn new(
+        dma: Arc<dyn DmaAllocator>,
+        mut transport: T,
+        pci_device_info: PciDeviceInfo,
+    ) -> Result<Self, Error> {
+        let features = Features::from(transport.read_device_features());
+        let device_cfg = transport
+            .device_cfg()
+            .ok_or(Error::InvalidArgument)
+            .inspect_err(|_| log::error!("virtio-blk does not expose device configuration"))?;
+        let device_cfg: &DeviceConfig =
+            bytemuck::from_bytes(&device_cfg[..size_of::<DeviceConfig>()]);
+
+        let read_only = features.contains(Features::F_RO);
+        let segment_size = if features.contains(Features::F_SIZE_MAX) {
+            device_cfg.size_max as usize
+        } else {
+            // I guess no limit then?
+            262144
+        };
+        let segment_limit = if features.contains(Features::F_SEG_MAX) {
+            device_cfg.seg_max as usize
+        } else {
+            // I guess no limit then?
+            8
+        };
+        let capacity = device_cfg.capacity;
+
+        if segment_limit < 3 {
+            // Won't be able to send header + data + status
+            log::error!("virtio-blk: allowed segment count too small");
+            return Err(Error::InvalidArgument);
+        }
+
+        let request_queue = VirtQueue::with_capacity(&*dma, Self::VQ_REQUEST_0, 256)?;
+
+        Ok(Self {
+            transport: IrqSafeSpinlock::new(transport),
+            dma,
+            pci_device_info,
+
+            segment_size,
+            read_only,
+            capacity,
+            request_queue,
+        })
+    }
+
+    fn begin_init(&self) -> Result<DeviceStatus, Error> {
+        let mut transport = self.transport.lock();
+        let mut status = DeviceStatus::RESET_VALUE;
+
+        log::debug!("Reset device");
+        transport.write_device_status(status);
+        status |= DeviceStatus::ACKNOWLEDGE;
+        transport.write_device_status(status);
+        status |= DeviceStatus::DRIVER;
+        transport.write_device_status(status);
+
+        let _device_features = transport.read_device_features();
+
+        // TODO blah blah blah
+
+        transport.write_driver_features(0);
+
+        status |= DeviceStatus::FEATURES_OK;
+        transport.write_device_status(status);
+
+        if !transport
+            .read_device_status()
+            .contains(DeviceStatus::FEATURES_OK)
+        {
+            return Err(Error::InvalidOperation);
+        }
+
+        Ok(status)
+    }
+
+    fn finish_init(&self, status: DeviceStatus) {
+        let mut transport = self.transport.lock();
+
+        transport.write_device_status(status | DeviceStatus::DRIVER_OK);
+    }
+
+    fn setup_queues(self: &Arc<Self>) -> Result<(), Error> {
+        self.pci_device_info
+            .init_interrupts(PreferredInterruptMode::Msi(true))?;
+        let msi_info = self
+            .pci_device_info
+            .map_interrupt(InterruptAffinity::Any, self.clone())?;
+        let vector = msi_info.map(|msi| msi.vector as u16);
+
+        let mut transport = self.transport.lock();
+
+        transport.set_queue(Self::VQ_REQUEST_0, &self.request_queue, vector);
+
+        Ok(())
+    }
+}
+
+impl<T: Transport + 'static> InterruptHandler for VirtioBlk<T> {
+    fn handle_irq(self: Arc<Self>, _vector: IrqVector) -> bool {
+        // Only one queue
+        self.request_queue.handle_notify();
+        true
+    }
+}
+
+impl<T: Transport + 'static> Device for VirtioBlk<T> {
+    unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
+        let status = self.begin_init()?;
+        self.setup_queues()?;
+        self.finish_init(status);
+        register_virtio_block_device(self.clone());
+        Ok(())
+    }
+
+    fn display_name(&self) -> &str {
+        "VirtIO Block Device"
+    }
+}
+
+#[async_trait]
+impl<T: Transport + 'static> BlockDevice for VirtioBlk<T> {
+    fn allocate_buffer(&self, size: usize) -> Result<DmaBuffer<[MaybeUninit<u8>]>, Error> {
+        DmaBuffer::new_uninit_slice(&*self.dma, size)
+    }
+
+    async fn read_aligned(
+        &self,
+        position: u64,
+        buffer: DmaSliceMut<'_, MaybeUninit<u8>>,
+    ) -> Result<(), Error> {
+        if position % 512 != 0 || buffer.len() % 512 != 0 {
+            return Err(Error::InvalidArgument);
+        }
+        let lba = position / 512;
+        let lba_count = buffer.len() / 512;
+        if lba + lba_count as u64 >= self.capacity {
+            return Err(Error::InvalidArgument);
+        }
+
+        let mut header = DmaBuffer::new_slice(&*self.dma, 0, size_of::<CommandHeader>())?;
+        *bytemuck::from_bytes_mut(&mut header[..]) = CommandHeader::for_read(lba);
+        let mut status = DmaBuffer::new_uninit_slice(&*self.dma, 1)?;
+
+        self.request_queue
+            .enqueue_wait(
+                &[header.slice(0..size_of::<CommandHeader>())],
+                &[buffer, status.slice_mut(0..1)],
+                || {
+                    self.transport.lock().notify(Self::VQ_REQUEST_0);
+                },
+            )
+            .await?;
+
+        let status = unsafe { DmaBuffer::assume_init_slice(status) }[0];
+        if status == 0 {
+            Ok(())
+        } else {
+            Err(Error::InvalidOperation)
+        }
+    }
+    async fn write_aligned(&self, position: u64, buffer: DmaSlice<'_, u8>) -> Result<(), Error> {
+        if self.read_only {
+            return Err(Error::ReadOnly);
+        }
+
+        if position % 512 != 0 || buffer.len() % 512 != 0 {
+            return Err(Error::InvalidArgument);
+        }
+        let lba = position / 512;
+        let lba_count = buffer.len() / 512;
+        if lba + lba_count as u64 >= self.capacity {
+            return Err(Error::InvalidArgument);
+        }
+
+        let mut header = DmaBuffer::new_slice(&*self.dma, 0, size_of::<CommandHeader>())?;
+        *bytemuck::from_bytes_mut(&mut header[..]) = CommandHeader::for_write(lba);
+        let mut status = DmaBuffer::new_uninit_slice(&*self.dma, 1)?;
+
+        self.request_queue
+            .enqueue_wait(
+                &[header.slice(0..size_of::<CommandHeader>()), buffer],
+                &[status.slice_mut(0..1)],
+                || {
+                    self.transport.lock().notify(Self::VQ_REQUEST_0);
+                },
+            )
+            .await?;
+
+        let status = unsafe { DmaBuffer::assume_init_slice(status) }[0];
+        if status == 0 {
+            Ok(())
+        } else {
+            Err(Error::InvalidOperation)
+        }
+    }
+
+    fn block_size(&self) -> usize {
+        512
+    }
+    fn block_count(&self) -> u64 {
+        self.capacity
+    }
+    fn max_blocks_per_request(&self) -> usize {
+        // TODO this limit can be bumped or scatter-gather operations
+        self.segment_size / 512
+    }
+}
+
+impl<T: Transport + 'static> PageProvider for VirtioBlk<T> {
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn clone_page(
+        &self,
+        _offset: u64,
+        _src_phys: PhysicalAddress,
+        _src_attrs: MapAttributes,
+    ) -> Result<PhysicalAddress, Error> {
+        unimplemented!()
+    }
+}
+
+static DEVICES: IrqSafeRwLock<Vec<Arc<dyn BlockDevice>>> = IrqSafeRwLock::new(Vec::new());
+
+fn register_virtio_block_device(device: Arc<dyn BlockDevice>) {
+    let index = {
+        let mut devices = DEVICES.write();
+        let index = devices.len();
+        devices.push(device.clone());
+        index
+    };
+
+    let name = format!("vb{index}");
+    devfs::add_named_block_device(device.clone(), name.clone(), FileMode::new(0o600)).ok();
+
+    runtime::spawn(async move {
+        let name = name;
+        log::info!("Probing partitions for {name}");
+        probe_partitions(device, |index, partition| {
+            let partition_name = format!("{name}p{}", index + 1);
+            devfs::add_named_block_device(
+                Arc::new(partition),
+                partition_name,
+                FileMode::new(0o600),
+            )
+            .ok();
+        })
+        .await
+        .ok();
+    })
+    .ok();
+}
+
+pci_driver! {
+    matches: [device (0x1AF4:0x1001)],
+    driver: {
+        fn probe(
+            &self,
+            info: &PciDeviceInfo,
+            dma: &Arc<dyn DmaAllocator>,
+        ) -> Result<Arc<dyn Device>, Error> {
+            let space = &info.config_space;
+
+            let transport = PciTransport::from_config_space(space).unwrap();
+            let device = VirtioBlk::new(dma.clone(), transport, info.clone())?;
+
+            let device = Arc::new(device);
+
+            Ok(device)
+        }
+
+        fn driver_name(&self) -> &str {
+            "virtio-blk"
+        }
+    }
+}

kernel/driver/virtio/core/Cargo.toml

@@ -16,6 +16,12 @@ log.workspace = true
 bitflags.workspace = true
 tock-registers.workspace = true
 
+[dev-dependencies]
+kernel-arch-hosted.path = "../../../arch/hosted"
+
 [features]
 default = []
 pci = ["ygg_driver_pci"]
+
+[lints]
+workspace = true

kernel/driver/virtio/core/src/queue.rs

@@ -1,361 +1,535 @@
 //! VirtIO queue implementation.
-//!
-//! # Note
-//!
-//! The code is poorly borrowed from `virtio-drivers` crate. I want to rewrite it properly myself.
-use core::{
-    mem::MaybeUninit,
-    sync::atomic::{fence, Ordering},
+
+use core::{future::poll_fn, mem::MaybeUninit, task::Poll};
+
+use alloc::{boxed::Box, sync::Arc};
+use device_api::dma::DmaAllocator;
+use libk::{
+    dma::{BusAddress, DmaBuffer, DmaSlice, DmaSliceMut},
+    error::Error,
+};
+use libk_util::{
+    event::OneTimeEvent, hash_table::DefaultHashTable, sync::IrqSafeSpinlock, waker::QueueWaker,
 };
 
-use device_api::dma::DmaAllocator;
-use libk::dma::{BusAddress, DmaBuffer};
-
-use crate::{error::Error, transport::Transport};
-
-#[derive(Debug)]
+#[derive(Clone, Copy, Debug)]
 #[repr(C)]
 struct Descriptor {
     address: BusAddress,
-    len: u32,
+    length: u32,
     flags: u16,
     next: u16,
 }
 
-// Layout:
-//  {
-//      flags: u16,
-//      idx: u16,
-//      ring: [u16; QUEUE_SIZE],
-//      used_event: u16
-//  }
 struct AvailableRing {
-    data: DmaBuffer<[MaybeUninit<u16>]>,
+    mapping: DmaBuffer<[u16]>,
+    capacity: usize,
 }
 
-// Layout:
-//  {
-//      flags: u16,
-//      idx: u16,
-//      ring: [UsedElem; QUEUE_SIZE],
-//      avail_event: u16,
-//      _pad: u16
-//  }
 struct UsedRing {
-    data: DmaBuffer<[MaybeUninit<u32>]>,
-    used_count: usize,
+    mapping: DmaBuffer<[u32]>,
+    last_seen_used: u16,
+    capacity: usize,
 }
 
-pub struct VirtQueue {
-    descriptor_table: DmaBuffer<[MaybeUninit<Descriptor>]>,
+pub struct DescriptorTable {
+    descriptors: DmaBuffer<[Descriptor]>,
+    free_count: usize,
+    first_free: Option<u16>,
+    last_free: Option<u16>,
+}
+
+struct VqInner {
+    descriptors: DescriptorTable,
     available: AvailableRing,
     used: UsedRing,
+}
 
+pub struct VirtQueue<N: VqNotificationMechanism = VqAsyncNotification> {
+    inner: IrqSafeSpinlock<VqInner>,
+    free_descriptor_notify: QueueWaker,
+
+    #[allow(unused)]
+    index: u16,
     capacity: usize,
 
-    queue_index: u16,
-    free_head: u16,
+    used_notify: N,
+}
 
-    avail_idx: u16,
-    last_used_idx: u16,
+pub trait VqNotificationMechanism {
+    type Token;
 
-    msix_vector: u16,
+    fn notify_used(&self, head: u16, length: u32);
+    fn create_token(&self, head: u16) -> Self::Token;
+}
+
+pub struct VqAsyncNotification {
+    completions: IrqSafeSpinlock<DefaultHashTable<u16, Arc<OneTimeEvent<u32>>>>,
+}
+
+pub struct VqManualNotification;
+
+pub struct VqCallbackNotification(Box<dyn Fn(u16, u32) + Sync + Send>);
+
+impl VqNotificationMechanism for VqAsyncNotification {
+    type Token = Arc<OneTimeEvent<u32>>;
+
+    fn notify_used(&self, head: u16, length: u32) {
+        let mut completions = self.completions.lock();
+        if let Some(completion) = completions.remove(&head) {
+            log::trace!("vq: completion #{head}");
+            completion.signal(length);
+        }
+    }
+
+    fn create_token(&self, head: u16) -> Self::Token {
+        let mut completions = self.completions.lock();
+        let token = Arc::new(OneTimeEvent::new());
+        completions.insert(head, token.clone());
+        token
+    }
+}
+
+impl VqNotificationMechanism for VqCallbackNotification {
+    type Token = u16;
+
+    fn create_token(&self, head: u16) -> Self::Token {
+        head
+    }
+
+    fn notify_used(&self, head: u16, length: u32) {
+        (self.0)(head, length);
+    }
+}
+
+impl VqNotificationMechanism for VqManualNotification {
+    type Token = u16;
+
+    fn create_token(&self, head: u16) -> Self::Token {
+        head
+    }
+
+    fn notify_used(&self, _head: u16, _length: u32) {
+        unreachable!()
+    }
 }
 
 impl AvailableRing {
+    const FLAGS: usize = 0;
+    const IDX: usize = 1;
+    const RING: usize = 2;
+
     pub fn with_capacity(
         dma: &dyn DmaAllocator,
-        no_irq: bool,
         capacity: usize,
+        no_interrupt: bool,
     ) -> Result<Self, Error> {
-        let mut data = DmaBuffer::new_zeroed_slice(dma, capacity + 3)?;
-
-        if no_irq {
-            data[0].write(1);
+        // flags + idx + [ring] + used_event
+        let mut mapping = DmaBuffer::new_slice(dma, 0u16, (capacity + 6) & !3)?;
+        if no_interrupt {
+            mapping[Self::FLAGS] |= 1 << 0;
         }
-
-        data[1].write(0);
-
-        Ok(Self { data })
+        Ok(Self { mapping, capacity })
     }
 
-    pub fn set_head(&mut self, slot: u16, head: u16) {
-        self.data[slot as usize + 2].write(head);
-    }
-
-    pub fn set_index(&mut self, index: u16) {
-        self.data[1].write(index);
+    pub fn push(&mut self, head: u16) -> u16 {
+        log::trace!("enqueue #{head}");
+        let idx = self.mapping[Self::IDX];
+        let index = idx as usize % self.capacity;
+        self.mapping[Self::RING + index] = head;
+        self.mapping.cache_flush_element(Self::RING + index, true);
+        let idx = idx.wrapping_add(1);
+        self.mapping[Self::IDX] = idx;
+        self.mapping.cache_flush_element(Self::IDX, true);
+        idx
     }
 }
 
 impl UsedRing {
-    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
-        let mut data = DmaBuffer::new_zeroed_slice(dma, capacity * 2 + 2)?;
-
-        data[0].write(0);
+    const FLAGS_IDX: usize = 0;
+    const RING: usize = 1;
 
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        capacity: usize,
+        no_notify: bool,
+    ) -> Result<Self, Error> {
+        // 2x u16 (flags + idx) + [ring x 2 x u32] + avail_event
+        let mut mapping = DmaBuffer::new_slice(dma, 0, capacity * 2 + 1)?;
+        if no_notify {
+            mapping[Self::FLAGS_IDX] |= 1 << 0;
+        }
+        mapping.cache_flush_element(Self::FLAGS_IDX, true);
         Ok(Self {
-            data,
-            used_count: 0,
+            mapping,
+            capacity,
+            last_seen_used: 0,
         })
     }
 
-    pub fn read_slot(&self, index: u16) -> (u32, u32) {
-        let index = unsafe { self.data[1 + index as usize * 2].assume_init() };
-        let len = unsafe { self.data[2 + index as usize * 2].assume_init() };
-        (index, len)
-    }
-
-    pub fn index(&self) -> u16 {
-        unsafe { (self.data[0].assume_init() >> 16) as u16 }
+    pub fn consume<F: FnMut(u16, u32)>(&mut self, mut handler: F) -> usize {
+        self.mapping.cache_flush_element(Self::FLAGS_IDX, false);
+        let idx = (self.mapping[Self::FLAGS_IDX] >> 16) as u16;
+        let mut count = 0;
+        while self.last_seen_used != idx {
+            let index = self.last_seen_used as usize % self.capacity;
+            self.mapping
+                .cache_flush_range(Self::RING + index..Self::RING + index + 1, false);
+            let head = self.mapping[Self::RING + index * 2] as u16;
+            let len = self.mapping[Self::RING + index * 2 + 1];
+            handler(head, len);
+            count += 1;
+            self.last_seen_used = self.last_seen_used.wrapping_add(1);
+        }
+        count
     }
 }
 
-impl VirtQueue {
-    pub fn with_capacity<T: Transport>(
-        transport: &mut T,
-        dma: &dyn DmaAllocator,
-        index: u16,
-        capacity: usize,
-        msix_vector: Option<u16>,
-        no_avail_irq: bool,
-    ) -> Result<Self, Error> {
-        // TODO check if queue is already set up
+impl Descriptor {
+    pub const EMPTY: Self = Self {
+        address: BusAddress::ZERO,
+        length: 0,
+        flags: 0,
+        next: 0,
+    };
 
-        let max_capacity = transport.max_queue_size(index);
+    pub const F_NEXT: u16 = 1 << 0;
+    pub const F_WRITE: u16 = 1 << 1;
+}
 
-        if !capacity.is_power_of_two() || capacity > u16::MAX.into() {
-            return Err(Error::InvalidQueueSize);
+impl DescriptorTable {
+    const FREE_CHAIN_END: u16 = u16::MAX;
+
+    pub fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        if capacity >= Self::FREE_CHAIN_END as usize - 1 {
+            return Err(Error::InvalidArgument);
         }
-
-        if capacity > max_capacity as usize {
-            return Err(Error::QueueTooLarge);
+        let mut descriptors = DmaBuffer::new_slice(dma, Descriptor::EMPTY, capacity)?;
+        for i in 0..capacity {
+            if i == capacity - 1 {
+                // Last descriptor of the free chain
+                descriptors[i].next = Self::FREE_CHAIN_END;
+            } else {
+                descriptors[i].next = (i + 1) as u16;
+            }
         }
 
-        let descriptor_table = DmaBuffer::new_zeroed_slice(dma, capacity)?;
-        let available = AvailableRing::with_capacity(dma, no_avail_irq, capacity)?;
-        let used = UsedRing::with_capacity(dma, capacity)?;
-
-        transport.set_queue(
-            index,
-            capacity as u16,
-            descriptor_table.bus_address(),
-            available.data.bus_address(),
-            used.data.bus_address(),
-            msix_vector,
-        );
-
         Ok(Self {
-            descriptor_table,
-            available,
-            used,
-
-            capacity,
-
-            queue_index: index,
-            free_head: 0,
-
-            avail_idx: 0,
-            last_used_idx: 0,
-
-            msix_vector: msix_vector.unwrap_or(0xFFFF),
+            descriptors,
+            free_count: capacity,
+            first_free: Some(0),
+            last_free: Some(capacity as u16 - 1),
         })
     }
 
-    pub fn capacity(&self) -> usize {
-        self.capacity
-    }
-
-    pub fn with_max_capacity<T: Transport>(
-        transport: &mut T,
-        dma: &dyn DmaAllocator,
-        index: u16,
-        capacity: usize,
-        msix_vector: Option<u16>,
-        no_avail_irq: bool,
-    ) -> Result<Self, Error> {
-        let max_capacity = transport.max_queue_size(index);
-        let capacity = capacity.min(max_capacity as usize);
-
-        Self::with_capacity(transport, dma, index, capacity, msix_vector, no_avail_irq)
+    // Allocate a chain of descriptors
+    fn alloc_descriptors(
+        &mut self,
+        h2d: &[DmaSlice<u8>],
+        d2h: &[DmaSliceMut<MaybeUninit<u8>>],
+    ) -> Result<u16, Error> {
+        if d2h.len() + h2d.len() == 0 {
+            // Empty transfer
+            return Err(Error::InvalidArgument);
+        }
+        if self.free_count < d2h.len() + h2d.len() {
+            // Not enough descriptor "slots" to place the buffers
+            return Err(Error::WouldBlock);
+        }
+        // Implied by free_count
+        debug_assert!(self.first_free.is_some());
+        debug_assert!(self.last_free.is_some());
+        let head = unsafe {
+            self.follow_descriptor_chain(h2d.len() + d2h.len(), |i, desc| {
+                if i < h2d.len() {
+                    desc.address = h2d[i].bus_address();
+                    desc.length = h2d[i].len() as u32;
+                    desc.flags = 0;
+                } else {
+                    let i = i - h2d.len();
+                    desc.address = d2h[i].bus_address();
+                    desc.length = d2h[i].len() as u32;
+                    desc.flags = Descriptor::F_WRITE;
+                }
+            })
+        };
+        Ok(head)
     }
 
     /// # Safety
     ///
-    /// Invariants: DmaBuffer remains valid and allocated until it is properly dequeued.
-    pub unsafe fn add<'a, 'b>(
+    /// The following invariants must hold:
+    ///
+    /// * **There must actually be `count` free descriptors**.
+    /// * **`count` should not be zero**.
+    /// * The VQ must be in a consistent state: first_free/last_free/free_count must be consistent
+    ///   in terms of tracking the current state of the descriptor allocation.
+    /// * Free descriptors in the table must be chained.
+    /// * Last free descriptor must have [Self::FREE_CHAIN_END] as its `next` field.
+    unsafe fn follow_descriptor_chain<F: Fn(usize, &mut Descriptor)>(
         &mut self,
-        input: &'a [&'b mut DmaBuffer<[MaybeUninit<u8>]>],
-        output: &'a [&'b DmaBuffer<[u8]>],
-    ) -> Result<u16, Error> {
-        if input.is_empty() && output.is_empty() {
-            return Err(Error::EmptyTransaction);
-        }
-        let n_desc = input.len() + output.len();
-
-        if self.used.used_count + 1 > self.capacity || self.used.used_count + n_desc > self.capacity
-        {
-            return Err(Error::QueueFull);
-        }
-
-        let head = self.add_direct(input, output);
-        let avail_slot = self.avail_idx % self.capacity as u16;
-
-        self.available.set_head(avail_slot, head);
-
-        fence(Ordering::SeqCst);
-
-        self.avail_idx = self.avail_idx.wrapping_add(1);
-        self.available.set_index(self.avail_idx);
-
-        fence(Ordering::SeqCst);
-
-        Ok(head)
-    }
-
-    unsafe fn add_direct<'a, 'b>(
-        &mut self,
-        input: &'a [&'b mut DmaBuffer<[MaybeUninit<u8>]>],
-        output: &'a [&'b DmaBuffer<[u8]>],
+        count: usize,
+        visitor: F,
     ) -> u16 {
-        let head = self.free_head;
-        let mut last = self.free_head;
-
-        for item in output {
-            assert_ne!(item.len(), 0);
-            let desc = &mut self.descriptor_table[usize::from(self.free_head)];
-            let next = (self.free_head + 1) % self.capacity as u16;
-
-            desc.write(Descriptor {
-                address: item.bus_address(),
-                len: item.len().try_into().unwrap(),
-                // TODO
-                flags: (1 << 0),
-                next,
-            });
-
-            last = self.free_head;
-            self.free_head = next;
+        debug_assert_ne!(count, 0);
+        let mut current = self.first_free.unwrap_unchecked();
+        let head = current;
+        for i in 0..count {
+            let descriptor = &mut self.descriptors[current as usize];
+            log::trace!("vq: alloc desc #{current}");
+            visitor(i, descriptor);
+            if i == count - 1 {
+                debug_assert_eq!(descriptor.flags & Descriptor::F_NEXT, 0);
+                current = descriptor.next;
+                descriptor.next = 0;
+            } else {
+                current = descriptor.next;
+                descriptor.flags |= Descriptor::F_NEXT;
+                debug_assert_ne!(current, Self::FREE_CHAIN_END);
+            };
         }
-
-        for item in input {
-            assert_ne!(item.len(), 0);
-            let desc = &mut self.descriptor_table[usize::from(self.free_head)];
-            let next = (self.free_head + 1) % self.capacity as u16;
-
-            desc.write(Descriptor {
-                address: item.bus_address(),
-                len: item.len().try_into().unwrap(),
-                // TODO MAGIC
-                flags: (1 << 0) | (1 << 1),
-                next,
-            });
-
-            last = self.free_head;
-            self.free_head = next;
+        if current == Self::FREE_CHAIN_END {
+            // No free descriptors left
+            debug_assert_eq!(self.free_count, count);
+            self.first_free = None;
+            self.last_free = None;
+        } else {
+            self.first_free = Some(current);
         }
-
-        {
-            let last_desc = self.descriptor_table[last as usize].assume_init_mut();
-
-            // TODO
-            last_desc.flags &= !(1 << 0);
-        }
-
-        self.used.used_count += input.len() + output.len();
-
-        fence(Ordering::SeqCst);
+        self.free_count -= count;
 
         head
     }
 
-    pub fn add_notify_wait_pop<'a, 'b, T: Transport>(
-        &mut self,
-        input: &'a [&'b mut DmaBuffer<[MaybeUninit<u8>]>],
-        output: &'a [&'b DmaBuffer<[u8]>],
-        transport: &mut T,
-    ) -> Result<u32, Error> {
-        let token = unsafe { self.add(input, output) }?;
+    fn add_free_descriptor(&mut self, idx: u16) {
+        log::trace!("vq: free descriptor #{idx}");
+        self.descriptors[idx as usize] = Descriptor {
+            next: Self::FREE_CHAIN_END,
+            ..Descriptor::EMPTY
+        };
 
-        transport.notify(self.queue_index);
-
-        while self.is_used_empty() {
-            core::hint::spin_loop();
-        }
-
-        fence(Ordering::SeqCst);
-
-        unsafe { self.pop_used(token) }
-    }
-
-    pub fn is_used_empty(&self) -> bool {
-        fence(Ordering::SeqCst);
-
-        self.last_used_idx == self.used.index()
-    }
-
-    pub fn pop_last_used(&mut self) -> Option<(u16, u32)> {
-        let token = self.peek_used()?;
-        let len = unsafe { self.pop_used(token) }.unwrap();
-
-        Some((token, len))
-    }
-
-    fn peek_used(&mut self) -> Option<u16> {
-        if !self.is_used_empty() {
-            let last_used = self.last_used_idx % self.capacity as u16;
-            Some(self.used.read_slot(last_used).0 as u16)
+        if let Some(last) = self.last_free {
+            // Implies first free is Some as well
+            self.descriptors[last as usize].next = idx;
+            // last -> idx
+            self.last_free = Some(idx);
         } else {
-            None
+            // Implies first free is None as well
+            self.first_free = Some(idx);
+            self.last_free = Some(idx);
         }
+
+        self.free_count += 1;
     }
 
-    unsafe fn pop_used(&mut self, token: u16) -> Result<u32, Error> {
-        if self.is_used_empty() {
-            return Err(Error::QueueEmpty);
-        }
-
-        let last_used_slot = self.last_used_idx % self.capacity as u16;
-        let (index, len) = self.used.read_slot(last_used_slot);
-
-        if index != token as u32 {
-            return Err(Error::WrongToken);
-        }
-
-        self.free_descriptor_chain(token);
-
-        fence(Ordering::SeqCst);
-
-        self.last_used_idx = self.last_used_idx.wrapping_add(1);
-
-        Ok(len)
-    }
-
-    unsafe fn free_descriptor_chain(&mut self, head: u16) -> usize {
-        let mut current_node = Some(self.descriptor_table[usize::from(head)].assume_init_mut());
+    fn free_descriptor_chain(&mut self, first: u16) -> usize {
+        let mut current = first;
         let mut count = 0;
+        loop {
+            let descriptor = &self.descriptors[current as usize];
+            let next = if descriptor.flags & Descriptor::F_NEXT != 0 {
+                Some(descriptor.next)
+            } else {
+                None
+            };
 
-        while let Some(current) = current_node {
-            assert_ne!(current.len, 0);
-            let next_head = (current.flags & (1 << 0) != 0).then_some(current.next);
-
-            current.address = BusAddress::ZERO;
-            current.flags = 0;
-            current.next = 0;
-            current.len = 0;
-
-            self.used.used_count -= 1;
             count += 1;
+            self.add_free_descriptor(current);
 
-            current_node =
-                next_head.map(|head| self.descriptor_table[usize::from(head)].assume_init_mut());
+            if let Some(next) = next {
+                current = next;
+            } else {
+                break;
+            }
         }
-
-        self.free_head = head;
         count
     }
+}
 
-    pub fn msix_vector(&self) -> u16 {
-        self.msix_vector
+impl VqInner {
+    fn with_capacity(dma: &dyn DmaAllocator, capacity: usize) -> Result<Self, Error> {
+        let descriptors = DescriptorTable::with_capacity(dma, capacity)?;
+        let available = AvailableRing::with_capacity(dma, capacity, false)?;
+        let used = UsedRing::with_capacity(dma, capacity, false)?;
+
+        Ok(Self {
+            descriptors,
+            available,
+            used,
+        })
+    }
+
+    fn try_enqueue(
+        &mut self,
+        h2d: &[DmaSlice<u8>],
+        d2h: &[DmaSliceMut<MaybeUninit<u8>>],
+    ) -> Result<u16, Error> {
+        let head = self.descriptors.alloc_descriptors(h2d, d2h)?;
+        self.available.push(head);
+        Ok(head)
+    }
+
+    fn consume<F: FnMut(u16, u32)>(&mut self, free_notify: &QueueWaker, mut handler: F) -> usize {
+        self.used.consume(|head, len| {
+            log::trace!("vq: used #{head}, len={len}");
+
+            self.descriptors.free_descriptor_chain(head);
+            free_notify.wake_all();
+
+            handler(head, len);
+        })
+    }
+}
+
+impl VirtQueue<VqAsyncNotification> {
+    pub fn with_capacity(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+    ) -> Result<Self, Error> {
+        let used_notify = VqAsyncNotification::new();
+        Self::with_capacity_and_notify(dma, index, capacity, used_notify)
+    }
+
+    pub async fn enqueue_wait<F: FnOnce()>(
+        &self,
+        h2d: &[DmaSlice<'_, u8>],
+        d2h: &[DmaSliceMut<'_, MaybeUninit<u8>>],
+        notify_queue: F,
+    ) -> Result<u32, Error> {
+        let completion = self.enqueue(h2d, d2h).await?;
+        notify_queue();
+        let result = completion.wait_copy().await;
+        Ok(result)
+    }
+}
+
+impl VirtQueue<VqCallbackNotification> {
+    pub fn with_capacity_and_callback(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+        used_callback: Box<dyn Fn(u16, u32) + Sync + Send>,
+    ) -> Result<Self, Error> {
+        let used_notify = VqCallbackNotification(used_callback);
+        Self::with_capacity_and_notify(dma, index, capacity, used_notify)
+    }
+}
+
+impl VirtQueue<VqManualNotification> {
+    pub fn with_capacity_manual(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+    ) -> Result<Self, Error> {
+        let used_notify = VqManualNotification;
+        Self::with_capacity_and_notify(dma, index, capacity, used_notify)
+    }
+
+    pub fn handle_notify_manual<F: FnMut(u16, u32)>(&self, handler: F) -> usize {
+        self.inner
+            .lock()
+            .consume(&self.free_descriptor_notify, handler)
+    }
+
+    // Used when queue does not support device-side used notification
+    pub fn enqueue_blocking<F: FnOnce()>(
+        &self,
+        h2d: &[DmaSlice<u8>],
+        d2h: &[DmaSliceMut<MaybeUninit<u8>>],
+        notify_queue: F,
+    ) -> Result<u32, Error> {
+        let token = self.try_enqueue(h2d, d2h)?;
+        let mut length = 0;
+        notify_queue();
+        loop {
+            self.handle_notify_manual(|head, len| {
+                assert_eq!(head, token);
+                length = len;
+            });
+            if length != 0 {
+                break;
+            }
+            core::hint::spin_loop();
+        }
+        Ok(length)
+    }
+}
+
+impl<N: VqNotificationMechanism> VirtQueue<N> {
+    pub fn with_capacity_and_notify(
+        dma: &dyn DmaAllocator,
+        index: u16,
+        capacity: usize,
+        used_notify: N,
+    ) -> Result<Self, Error> {
+        let inner = VqInner::with_capacity(dma, capacity)?;
+        Ok(Self {
+            inner: IrqSafeSpinlock::new(inner),
+
+            index,
+            capacity,
+            used_notify,
+            free_descriptor_notify: QueueWaker::new(),
+        })
+    }
+
+    pub fn descriptor_table_base(&self) -> BusAddress {
+        self.inner.lock().descriptors.descriptors.bus_address()
+    }
+
+    pub fn available_ring_base(&self) -> BusAddress {
+        self.inner.lock().available.mapping.bus_address()
+    }
+
+    pub fn used_ring_base(&self) -> BusAddress {
+        self.inner.lock().used.mapping.bus_address()
+    }
+
+    pub fn capacity(&self) -> u16 {
+        self.capacity as u16
+    }
+
+    pub fn handle_notify(&self) -> usize {
+        self.inner
+            .lock()
+            .consume(&self.free_descriptor_notify, |head, len| {
+                self.used_notify.notify_used(head, len);
+            })
+    }
+
+    pub fn try_enqueue(
+        &self,
+        h2d: &[DmaSlice<'_, u8>],
+        d2h: &[DmaSliceMut<'_, MaybeUninit<u8>>],
+    ) -> Result<N::Token, Error> {
+        let head = self.inner.lock().try_enqueue(h2d, d2h)?;
+        Ok(self.used_notify.create_token(head))
+    }
+
+    pub async fn enqueue(
+        &self,
+        h2d: &[DmaSlice<'_, u8>],
+        d2h: &[DmaSliceMut<'_, MaybeUninit<u8>>],
+    ) -> Result<N::Token, Error> {
+        poll_fn(|cx| match self.try_enqueue(h2d, d2h) {
+            Err(Error::WouldBlock) => {
+                self.free_descriptor_notify.register(cx.waker());
+                Poll::Pending
+            }
+            result => {
+                self.free_descriptor_notify.remove(cx.waker());
+                Poll::Ready(result)
+            }
+        })
+        .await
+    }
+}
+
+impl VqAsyncNotification {
+    pub fn new() -> Self {
+        Self {
+            completions: IrqSafeSpinlock::new(DefaultHashTable::new()),
+        }
     }
 }

kernel/driver/virtio/core/src/transport/mod.rs

@@ -7,8 +7,12 @@ use tock_registers::{
     registers::WriteOnly,
 };
 
-use crate::{CommonConfiguration, DeviceStatus};
+use crate::{
+    queue::{VirtQueue, VqNotificationMechanism},
+    CommonConfiguration, DeviceStatus,
+};
 
+#[cfg(any(feature = "pci", rust_analyzer))]
 pub mod pci;
 
 pub trait Transport: Send {
@@ -53,7 +57,23 @@ pub trait Transport: Send {
         cfg.queue_size.get().into()
     }
 
-    fn set_queue(
+    fn set_queue<N: VqNotificationMechanism>(
+        &mut self,
+        index: u16,
+        queue: &VirtQueue<N>,
+        msix_vector: Option<u16>,
+    ) {
+        self.set_queue_raw(
+            index,
+            queue.capacity(),
+            queue.descriptor_table_base(),
+            queue.available_ring_base(),
+            queue.used_ring_base(),
+            msix_vector,
+        );
+    }
+
+    fn set_queue_raw(
         &mut self,
         queue: u16,
         capacity: u16,

kernel/driver/virtio/core/src/transport/pci.rs

@@ -5,7 +5,7 @@ use tock_registers::{
 };
 use ygg_driver_pci::{
     capability::{
-        VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
+        virtio::VirtioCapabilityData, VirtioCommonConfigCapability, VirtioDeviceConfigCapability,
         VirtioInterruptStatusCapability, VirtioNotifyConfigCapability,
     },
     PciCommandRegister, PciConfigurationSpace,

kernel/driver/virtio/gpu/Cargo.toml

@@ -15,6 +15,7 @@ ygg_driver_pci = { path = "../../bus/pci", optional = true }
 
 log.workspace = true
 bytemuck.workspace = true
+async-trait.workspace = true
 
 [features]
 default = []

kernel/driver/virtio/gpu/src/command.rs

@@ -7,8 +7,11 @@ use libk::{
     dma::{BusAddress, DmaBuffer},
     error::Error,
 };
-use libk_util::sync::IrqSafeSpinlockGuard;
-use ygg_driver_virtio_core::{queue::VirtQueue, transport::Transport};
+use libk_util::sync::IrqSafeSpinlock;
+use ygg_driver_virtio_core::{
+    queue::{VirtQueue, VqManualNotification},
+    transport::Transport,
+};
 
 #[derive(Clone, Copy, Pod, Zeroable)]
 #[repr(C)]
@@ -93,19 +96,12 @@ pub struct TransferToHost2d {
     pub _0: u32,
 }
 
-pub struct ControlLock<'a, T: Transport> {
-    control: IrqSafeSpinlockGuard<'a, VirtQueue>,
-    transport: IrqSafeSpinlockGuard<'a, T>,
+pub struct CommandExecution<'a, T: Transport> {
+    pub(super) transport: &'a IrqSafeSpinlock<T>,
+    pub(super) control: &'a VirtQueue<VqManualNotification>,
 }
 
-impl<'a, T: Transport> ControlLock<'a, T> {
-    pub const fn new(
-        control: IrqSafeSpinlockGuard<'a, VirtQueue>,
-        transport: IrqSafeSpinlockGuard<'a, T>,
-    ) -> Self {
-        Self { control, transport }
-    }
-
+impl<'a, T: Transport> CommandExecution<'a, T> {
     fn send_recv<'r, Req: Pod>(
         &mut self,
         dma: &dyn DmaAllocator,
@@ -116,20 +112,18 @@ impl<'a, T: Transport> ControlLock<'a, T> {
         let mut request = unsafe { DmaBuffer::assume_init_slice(request) };
         request.copy_from_slice(bytemuck::bytes_of(req));
 
-        let len = self
-            .control
-            .add_notify_wait_pop(&[buffer], &[&request], &mut *self.transport)
-            .inspect_err(|error| {
-                log::warn!("virtio queue: {error:?}");
-            })
-            .map_err(|_| Error::InvalidArgument)? as usize;
+        let len = self.control.enqueue_blocking(
+            &[request.slice(0..size_of::<Req>())],
+            &[buffer.slice_mut(0..buffer.len())],
+            || self.transport.lock().notify(0),
+        )? as usize;
 
         if len < size_of::<ControlHeader>() {
             log::warn!("virtio-gpu: invalid device response length: {len}");
             return Err(Error::InvalidArgument);
         }
 
-        let payload = unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..len]) };
+        let payload = unsafe { buffer[..len].assume_init_ref() };
         let header = bytemuck::from_bytes(&payload[..size_of::<ControlHeader>()]);
         let data = &payload[size_of::<ControlHeader>()..len];
 

kernel/driver/virtio/gpu/src/lib.rs

@@ -6,7 +6,7 @@ extern crate alloc;
 use core::mem::MaybeUninit;
 
 use alloc::{sync::Arc, vec::Vec};
-use command::{ControlLock, ScanoutInfo};
+use command::{CommandExecution, ScanoutInfo};
 use device_api::{
     device::{Device, DeviceInitContext},
     dma::DmaAllocator,
@@ -23,7 +23,7 @@ use libk::{
 use libk_mm::{
     address::{AsPhysicalAddress, PhysicalAddress},
     table::MapAttributes,
-    PageProvider, L3_PAGE_SIZE,
+    OnDemandPage, PageProvider, VirtualPage, L3_PAGE_SIZE,
 };
 use libk_util::{
     sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
@@ -31,7 +31,7 @@ use libk_util::{
 };
 use ygg_driver_pci::{device::PciDeviceInfo, macros::pci_driver};
 use ygg_driver_virtio_core::{
-    queue::VirtQueue,
+    queue::{VirtQueue, VqManualNotification},
     transport::{pci::PciTransport, Transport},
     DeviceStatus,
 };
@@ -40,7 +40,7 @@ use yggdrasil_abi::error::Error;
 mod command;
 
 struct Queues {
-    control: IrqSafeSpinlock<VirtQueue>,
+    control: VirtQueue<VqManualNotification>,
 }
 
 struct Framebuffer {
@@ -64,7 +64,7 @@ struct Config {
 pub struct VirtioGpu<T: Transport> {
     transport: IrqSafeSpinlock<T>,
     #[allow(unused)]
-    pci_device_info: Option<PciDeviceInfo>,
+    pci_device_info: PciDeviceInfo,
 
     queues: OneTimeInit<Queues>,
     config: IrqSafeRwLock<Config>,
@@ -78,7 +78,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
     pub fn new(
         dma: Arc<dyn DmaAllocator>,
         transport: T,
-        info: Option<PciDeviceInfo>,
+        info: PciDeviceInfo,
     ) -> Result<Self, Error> {
         // Read num-scanouts from device config
         let Some(device_cfg) = transport.device_cfg() else {
@@ -149,34 +149,31 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         transport.write_device_status(status | DeviceStatus::DRIVER_OK);
     }
 
-    fn setup_queues(&self) -> Result<(), Error> {
+    fn setup_queues(self: &Arc<Self>) -> Result<(), Error> {
         // TODO cursorq
         let mut transport = self.transport.lock();
 
-        let control = VirtQueue::with_max_capacity(&mut *transport, &*self.dma, 0, 128, None, true)
-            .map_err(|_| Error::InvalidArgument)?;
+        let control = VirtQueue::with_capacity_manual(&*self.dma, 0, 128)?;
+        transport.set_queue(0, &control, None);
 
-        self.queues.init(Queues {
-            control: IrqSafeSpinlock::new(control),
-        });
+        self.queues.init(Queues { control });
 
         Ok(())
     }
 
-    fn control(&self) -> ControlLock<T> {
-        let queues = self.queues.get();
-        let control = queues.control.lock();
-        let transport = self.transport.lock();
-
-        ControlLock::new(control, transport)
+    fn begin_command(&self) -> CommandExecution<T> {
+        CommandExecution {
+            transport: &self.transport,
+            control: &self.queues.get().control,
+        }
     }
 
     fn setup_display(&self) -> Result<(), Error> {
-        let mut control = self.control();
         let mut config = self.config.write();
+        let mut command = self.begin_command();
 
         let scanouts =
-            control.query_scanouts(&*self.dma, self.num_scanouts, &mut config.response)?;
+            command.query_scanouts(&*self.dma, self.num_scanouts, &mut config.response)?;
         for (i, scanout) in scanouts.iter().enumerate() {
             log::info!(
                 "virtio-gpu: [{i}] {}x{} + {},{}",
@@ -214,23 +211,23 @@ impl<T: Transport + 'static> VirtioGpu<T> {
 
         let dma_buffer = DmaBuffer::new_uninit_slice(&*self.dma, size)?;
 
-        let mut control = self.control();
+        let mut command = self.begin_command();
 
-        let resource_id = control.create_resource_2d(
+        let resource_id = command.create_resource_2d(
             &*self.dma,
             &mut config.response,
             w,
             h,
             PixelFormat::R8G8B8A8,
         )?;
-        control.attach_backing(
+        command.attach_backing(
             &*self.dma,
             &mut config.response,
             resource_id,
             dma_buffer.bus_address(),
             size.try_into().unwrap(),
         )?;
-        control.set_scanout(
+        command.set_scanout(
             &*self.dma,
             &mut config.response,
             index as u32,
@@ -259,7 +256,7 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?;
         let r = config.scanouts[framebuffer.scanout_index].r;
 
-        let mut control = self.control();
+        let mut command = self.begin_command();
 
         if framebuffer.double {
             // Flip the buffer
@@ -267,8 +264,8 @@ impl<T: Transport + 'static> VirtioGpu<T> {
         } else {
             let resource_id = framebuffer.resource_id;
 
-            control.transfer_to_host_2d(&*self.dma, &mut config.response, resource_id, r)?;
-            control.resource_flush(&*self.dma, &mut config.response, resource_id, r)?;
+            command.transfer_to_host_2d(&*self.dma, &mut config.response, resource_id, r)?;
+            command.resource_flush(&*self.dma, &mut config.response, resource_id, r)?;
 
             Ok(())
         }
@@ -283,7 +280,6 @@ impl<T: Transport + 'static> Device for VirtioGpu<T> {
 
         // Set up some initial mode
         self.setup_display()?;
-
         self.setup_mode(0)?;
 
         DEVICE_REGISTRY.display.register(self.clone(), false)?;
@@ -297,7 +293,11 @@ impl<T: Transport + 'static> Device for VirtioGpu<T> {
 }
 
 impl<T: Transport + 'static> PageProvider for VirtioGpu<T> {
-    fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unreachable!()
+    }
+
+    fn get_page(&self, offset: u64) -> Result<VirtualPage, Error> {
         // TODO check that the page is mapped by framebuffer owner
         let config = self.config.read();
         let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?;
@@ -311,7 +311,16 @@ impl<T: Transport + 'static> PageProvider for VirtioGpu<T> {
         }
         let phys = unsafe { framebuffer.dma_buffer.as_physical_address() }.add(offset as usize);
 
-        Ok(phys)
+        Ok(VirtualPage::Immediate(phys))
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        Ok(())
     }
 
     fn clone_page(
@@ -320,11 +329,7 @@ impl<T: Transport + 'static> PageProvider for VirtioGpu<T> {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        Ok(())
+        unimplemented!()
     }
 }
 
@@ -427,7 +432,7 @@ pci_driver! {
                     log::error!("Couldn't set up PCI virtio transport: {error:?}");
                 })
                 .map_err(|_| Error::InvalidArgument)?;
-            let device = VirtioGpu::new(dma.clone(), transport, Some(info.clone()))?;
+            let device = VirtioGpu::new(dma.clone(), transport, info.clone())?;
             let device = Arc::new(device);
 
             Ok(device)

kernel/driver/virtio/net/Cargo.toml

@@ -18,6 +18,7 @@ log.workspace = true
 bitflags.workspace = true
 tock-registers.workspace = true
 bytemuck.workspace = true
+futures-util.workspace = true
 
 [features]
 default = []

kernel/driver/virtio/net/src/lib.rs

@@ -5,16 +5,19 @@ extern crate alloc;
 
 use core::mem::{size_of, MaybeUninit};
 
-use alloc::{collections::BTreeMap, sync::Arc};
+use alloc::{boxed::Box, sync::Arc};
 use bytemuck::{Pod, Zeroable};
 use device_api::{
     device::{Device, DeviceInitContext},
     dma::DmaAllocator,
     interrupt::{InterruptAffinity, InterruptHandler, IrqVector},
 };
-use libk::dma::DmaBuffer;
+use futures_util::task::AtomicWaker;
+use libk::{dma::DmaBuffer, task::runtime};
 use libk_util::{
-    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock, IrqSafeSpinlockGuard},
+    event::BitmapEvent,
+    hash_table::DefaultHashTable,
+    sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock},
     OneTimeInit,
 };
 use ygg_driver_net_core::{
@@ -26,28 +29,25 @@ use ygg_driver_pci::{
     macros::pci_driver,
 };
 use ygg_driver_virtio_core::{
-    queue::VirtQueue,
+    queue::{VirtQueue, VqCallbackNotification, VqManualNotification},
     transport::{pci::PciTransport, Transport},
     DeviceStatus,
 };
 use yggdrasil_abi::{error::Error, net::MacAddress};
 
-struct Queues {
-    receive: IrqSafeSpinlock<VirtQueue>,
-    transmit: IrqSafeSpinlock<VirtQueue>,
-}
-
 pub struct VirtioNet<T: Transport> {
     transport: IrqSafeSpinlock<T>,
-    queues: OneTimeInit<Queues>,
-    interface_id: OneTimeInit<u32>,
-
-    mac: IrqSafeRwLock<MacAddress>,
-
-    pending_packets: IrqSafeRwLock<BTreeMap<u16, DmaBuffer<[MaybeUninit<u8>]>>>,
+    pci_device_info: PciDeviceInfo,
     dma: Arc<dyn DmaAllocator>,
 
-    pci_device_info: Option<PciDeviceInfo>,
+    interface_id: OneTimeInit<u32>,
+    mac: IrqSafeRwLock<MacAddress>,
+
+    rx_queue: OneTimeInit<VirtQueue<VqManualNotification>>,
+    tx_queue: OneTimeInit<VirtQueue<VqCallbackNotification>>,
+    tx_in_flight: IrqSafeSpinlock<DefaultHashTable<u16, DmaBuffer<[u8]>>>,
+
+    softirq: BitmapEvent<AtomicWaker>,
 }
 
 #[derive(Clone, Copy, Debug, Pod, Zeroable)]
@@ -61,23 +61,15 @@ struct VirtioPacketHeader {
     csum_offset: u16,
 }
 
-impl Queues {
-    pub fn try_receive(&self, _index: usize) -> Option<(u16, IrqSafeSpinlockGuard<VirtQueue>)> {
-        let mut queue = self.receive.lock();
-        // TODO use len for packet size hint
-        let (token, _len) = queue.pop_last_used()?;
-        Some((token, queue))
-    }
-}
-
 impl<T: Transport + 'static> VirtioNet<T> {
-    const PACKET_SIZE: usize = 4096;
+    const VQ_RX: u16 = 0;
+    const VQ_TX: u16 = 1;
 
     pub fn new(
         dma: Arc<dyn DmaAllocator>,
         transport: T,
-        pci_device_info: Option<PciDeviceInfo>,
-    ) -> Self {
+        pci_device_info: PciDeviceInfo,
+    ) -> Result<Self, Error> {
         // Read MAC from device config
         let device_cfg = transport
             .device_cfg()
@@ -86,60 +78,21 @@ impl<T: Transport + 'static> VirtioNet<T> {
         mac_bytes.copy_from_slice(&device_cfg[..6]);
         let mac = MacAddress::from(mac_bytes);
 
-        Self {
+        pci_device_info.init_interrupts(PreferredInterruptMode::Msi(true))?;
+
+        Ok(Self {
             transport: IrqSafeSpinlock::new(transport),
-            queues: OneTimeInit::new(),
-            interface_id: OneTimeInit::new(),
-
-            mac: IrqSafeRwLock::new(mac),
-
-            pending_packets: IrqSafeRwLock::new(BTreeMap::new()),
-
             pci_device_info,
             dma,
-        }
-    }
 
-    pub fn listen(&self, buffers: usize) {
-        let queues = self.queues.get();
-        let mut queue = queues.receive.lock();
-        let mut packets = self.pending_packets.write();
+            interface_id: OneTimeInit::new(),
+            mac: IrqSafeRwLock::new(mac),
 
-        for _ in 0..buffers {
-            let mut packet = DmaBuffer::new_uninit_slice(&*self.dma, Self::PACKET_SIZE).unwrap();
-            let token = unsafe { queue.add(&[&mut packet], &[]).unwrap() };
-            packets.insert(token, packet);
-        }
-
-        let mut transport = self.transport.lock();
-        transport.notify(0);
-    }
-
-    fn handle_receive_interrupt(&self, queue: usize) -> bool {
-        let queues = self.queues.get();
-        let interface_id = *self.interface_id.get();
-        let mut count = 0;
-
-        while let Some((token, mut queue)) = queues.try_receive(queue) {
-            let mut pending_packets = self.pending_packets.write();
-            let packet = pending_packets.remove(&token).unwrap();
-
-            let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, Self::PACKET_SIZE).unwrap();
-
-            let token = unsafe { queue.add(&[&mut buffer], &[]).unwrap() };
-            pending_packets.insert(token, buffer);
-
-            let packet = unsafe { DmaBuffer::assume_init_slice(packet) };
-            let packet = RxPacket::new(packet, size_of::<VirtioPacketHeader>(), interface_id);
-            ygg_driver_net_core::receive_packet(packet).unwrap();
-            count += 1
-        }
-
-        if count != 0 {
-            self.transport.lock().notify(0);
-        }
-
-        count != 0
+            rx_queue: OneTimeInit::new(),
+            tx_queue: OneTimeInit::new(),
+            tx_in_flight: IrqSafeSpinlock::new(DefaultHashTable::new()),
+            softirq: BitmapEvent::new(AtomicWaker::new()),
+        })
     }
 
     fn begin_init(&self) -> Result<DeviceStatus, Error> {
@@ -178,46 +131,87 @@ impl<T: Transport + 'static> VirtioNet<T> {
         transport.write_device_status(status | DeviceStatus::DRIVER_OK);
     }
 
-    unsafe fn setup_queues(
-        self: Arc<Self>,
-        receive_count: usize,
-        transmit_count: usize,
-    ) -> Result<(), Error> {
-        let receive_vector = if let Some(pci) = self.pci_device_info.as_ref() {
-            pci.init_interrupts(PreferredInterruptMode::Msi(true))?;
-            let info = pci.map_interrupt(InterruptAffinity::Any, self.clone())?;
-
-            info.map(|info| info.vector as u16)
+    unsafe fn setup_queues(self: &Arc<Self>) -> Result<(), Error> {
+        let (rx_vector, tx_vector) = if let Ok(msis) =
+            self.pci_device_info
+                .map_interrupt_multiple(0..2, InterruptAffinity::Any, self.clone())
+        {
+            // Bound a MSI(-x) range, use per-queue vectors
+            (Some(msis[0].vector as u16), Some(msis[1].vector as u16))
         } else {
-            None
+            // TODO support non-MSI-x/non-multivec setups
+            self.pci_device_info
+                .map_interrupt(InterruptAffinity::Any, self.clone())?;
+            (None, None)
         };
 
-        // TODO multiqueue capability
-        assert_eq!(receive_count, 1);
-        assert_eq!(transmit_count, 1);
+        // Setup a callback to remove pending buffers and pass them to the network stack
+        let rx_queue = VirtQueue::with_capacity_manual(&*self.dma, Self::VQ_RX, 64)?;
+
+        let p = self.clone();
+        // Setup a callback to remove buffers from "in flight owned buffers" list
+        let tx_queue = VirtQueue::with_capacity_and_callback(
+            &*self.dma,
+            Self::VQ_TX,
+            64,
+            Box::new(move |head, _| {
+                p.tx_in_flight.lock().remove(&head);
+            }),
+        )?;
+
+        let rx_queue = self.rx_queue.init(rx_queue);
+        let tx_queue = self.tx_queue.init(tx_queue);
 
         let mut transport = self.transport.lock();
 
-        // Setup the virtqs
-        let rx = VirtQueue::with_max_capacity(
-            &mut *transport,
-            &*self.dma,
-            0,
-            128,
-            receive_vector,
-            false,
-        )
-        .map_err(cvt_error)?;
-        let tx = VirtQueue::with_max_capacity(&mut *transport, &*self.dma, 1, 128, None, true)
-            .map_err(cvt_error)?;
-
-        self.queues.init(Queues {
-            receive: IrqSafeSpinlock::new(rx),
-            transmit: IrqSafeSpinlock::new(tx),
-        });
+        transport.set_queue(Self::VQ_RX, rx_queue, rx_vector);
+        transport.set_queue(Self::VQ_TX, tx_queue, tx_vector);
 
         Ok(())
     }
+
+    async fn softirq(&self) -> Result<(), Error> {
+        const RX_SIZE: usize = 4096;
+        const RX_IN_FLIGHT: usize = 32;
+
+        let rx_queue = self.rx_queue.get();
+        let tx_queue = self.tx_queue.get();
+        let nic = *self.interface_id.get();
+
+        let mut rx_in_flight = DefaultHashTable::<u16, DmaBuffer<[MaybeUninit<u8>]>>::new();
+
+        // Setup initial Rx set
+        for _ in 0..RX_IN_FLIGHT {
+            let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, RX_SIZE)?;
+            let token = rx_queue.try_enqueue(&[], &[buffer.slice_mut(0..RX_SIZE)])?;
+            rx_in_flight.insert(token, buffer);
+        }
+
+        loop {
+            let events = self.softirq.wait().await;
+
+            if events & (1 << Self::VQ_RX) != 0 {
+                let refill_rx = rx_queue.handle_notify_manual(|head, _| {
+                    if let Some(packet) = rx_in_flight.remove(&head) {
+                        let packet = unsafe { DmaBuffer::assume_init_slice(packet) };
+                        let packet = RxPacket::new(packet, size_of::<VirtioPacketHeader>(), nic);
+                        ygg_driver_net_core::receive_packet(packet).ok();
+                    }
+                });
+
+                // Refill Rx buffers
+                for _ in 0..refill_rx {
+                    let mut buffer = DmaBuffer::new_uninit_slice(&*self.dma, RX_SIZE)?;
+                    let token = rx_queue.try_enqueue(&[], &[buffer.slice_mut(0..RX_SIZE)])?;
+                    rx_in_flight.insert(token, buffer);
+                }
+            }
+
+            if events & (1 << Self::VQ_TX) != 0 {
+                tx_queue.handle_notify();
+            }
+        }
+    }
 }
 
 impl<T: Transport + 'static> NetworkDevice for VirtioNet<T> {
@@ -226,13 +220,13 @@ impl<T: Transport + 'static> NetworkDevice for VirtioNet<T> {
     }
 
     fn transmit_buffer(&self, mut packet: DmaBuffer<[u8]>) -> Result<(), Error> {
-        let queues = self.queues.get();
-        let mut tx = queues.transmit.lock();
-        let mut transport = self.transport.lock();
+        let tx_queue = self.tx_queue.get();
         packet[..size_of::<VirtioPacketHeader>()].fill(0);
-        let _len = tx
-            .add_notify_wait_pop(&[], &[&packet], &mut *transport)
-            .unwrap();
+        let token = tx_queue.try_enqueue(&[packet.slice(0..packet.len())], &[])?;
+        // Add the packet to "in flight" list to make sure it doesn't get dropped and invalidated
+        // immediately after returning from this function
+        self.tx_in_flight.lock().insert(token, packet);
+        self.transport.lock().notify(Self::VQ_TX);
         Ok(())
     }
 
@@ -248,19 +242,14 @@ impl<T: Transport + 'static> NetworkDevice for VirtioNet<T> {
 impl<T: Transport + 'static> InterruptHandler for VirtioNet<T> {
     fn handle_irq(self: Arc<Self>, vector: IrqVector) -> bool {
         match vector {
-            IrqVector::Msi(_) => {
-                // MSI/MSI-X
-                self.handle_receive_interrupt(0)
+            IrqVector::Msi(vector) => {
+                self.softirq.signal(1 << vector);
+                true
             }
+            // TODO non-multivec/legacy IRQ setup
             IrqVector::Irq(_) => {
-                // Legacy IRQ
-                let (queue_irq, config_irq) = self.transport.lock().read_interrupt_status();
-
-                if queue_irq {
-                    self.handle_receive_interrupt(0);
-                }
-
-                queue_irq || config_irq
+                self.softirq.signal((1 << Self::VQ_RX) | (1 << Self::VQ_TX));
+                true
             }
         }
     }
@@ -274,15 +263,16 @@ impl<T: Transport + 'static> Device for VirtioNet<T> {
     unsafe fn init(self: Arc<Self>, _cx: DeviceInitContext) -> Result<(), Error> {
         let status = self.begin_init()?;
 
-        // TODO multiqueue
-        self.clone().setup_queues(1, 1)?;
+        self.setup_queues()?;
 
         self.finish_init(status);
 
         let iface =
             ygg_driver_net_core::register_interface(NetworkInterfaceType::Ethernet, self.clone());
         self.interface_id.init(iface.id());
-        self.listen(64);
+
+        let p = self.clone();
+        runtime::spawn(async move { p.softirq().await })?;
 
         Ok(())
     }
@@ -292,14 +282,6 @@ impl<T: Transport + 'static> Device for VirtioNet<T> {
     }
 }
 
-fn cvt_error(error: ygg_driver_virtio_core::error::Error) -> Error {
-    use ygg_driver_virtio_core::error::Error as VirtioError;
-    match error {
-        VirtioError::OsError(err) => err,
-        _ => Error::InvalidOperation,
-    }
-}
-
 pci_driver! {
     matches: [device (0x1AF4:0x1000)],
     driver: {
@@ -311,7 +293,7 @@ pci_driver! {
             let space = &info.config_space;
 
             let transport = PciTransport::from_config_space(space).unwrap();
-            let device = VirtioNet::new(dma.clone(), transport, Some(info.clone()));
+            let device = VirtioNet::new(dma.clone(), transport, info.clone())?;
 
             let device = Arc::new(device);
 

kernel/lib/memtables/src/riscv64.rs

@@ -11,6 +11,8 @@ bitflags! {
     #[derive(Clone, Copy, PartialEq, Eq, Debug)]
     pub struct PageAttributes: u64 {
         const N = 1 << 63;
+        /// Software-tracked dirty bit (RSW[0])
+        const SW_DIRTY = 1 << 9;
         /// Dirty bit
         const D = 1 << 7;
         /// Access bit

kernel/lib/vmalloc/src/lib.rs

@@ -72,6 +72,17 @@ impl<D: RangeData> VirtualMemoryAllocator<D> {
         Ok(start_pfn)
     }
 
+    /// Retrieves an entry corresponding to given PF#
+    pub fn get(&self, pfn: usize) -> Option<(Range<usize>, &D)> {
+        match self.map.get_entry_at_point(pfn as u64) {
+            Ok((range, data)) => {
+                let range = range.start() as usize..range.end() as usize + 1;
+                Some((range, data))
+            }
+            Err(_) => None,
+        }
+    }
+
     /// Tries to insert given PF# range with its associated metadata as allocated memory,
     /// returning [Error] if requested range overlaps any existing allocated ranges
     pub fn insert(&mut self, start_pfn: usize, page_count: usize, data: D) -> Result<(), Error> {

kernel/libk/libk-mm/Cargo.toml

@@ -12,6 +12,7 @@ vmalloc.workspace = true
 
 libyalloc = { path = "../../../lib/libyalloc", default-features = false, features = ["dep-of-kernel"] }
 
+async-trait.workspace = true
 log.workspace = true
 
 [lints.rust]

kernel/libk/libk-mm/interface/src/lib.rs

@@ -11,6 +11,15 @@ pub mod pointer;
 pub mod process;
 pub mod table;
 
+/// Describes which kind of a page fault occured
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PageFaultKind {
+    TranslationFault { write: bool },
+    WriteFault,
+    AccessFault,
+    Other,
+}
+
 /// Wrapper type to represent an object residing within the kernel
 #[repr(transparent)]
 pub struct KernelImageObject<T> {

kernel/libk/libk-mm/interface/src/process.rs

@@ -5,6 +5,11 @@ use crate::{
     table::{MapAttributes, TableAllocator},
 };
 
+pub struct PageAttributeUpdate {
+    pub user_write: Option<bool>,
+    pub dirty: Option<bool>,
+}
+
 /// Interface for virtual memory address space management
 pub trait ProcessAddressSpaceManager<TA: TableAllocator>: Sized {
     /// PFN of a minimum address allowed for virtual region allocation
@@ -27,13 +32,24 @@ pub trait ProcessAddressSpaceManager<TA: TableAllocator>: Sized {
         flags: MapAttributes,
     ) -> Result<(), Error>;
 
+    /// Adds/removes attributes from a page entry in the address space.
+    ///
+    /// # Safety
+    ///
+    /// The caller must ensure the validity of this update.
+    unsafe fn update_page_attributes(
+        &mut self,
+        address: usize,
+        update: &PageAttributeUpdate,
+    ) -> Result<(), Error>;
+
     /// Removes a single PAGE_SIZE mapping from the address space.
     ///
     /// # Safety
     ///
     /// The caller must ensure the process to which this address space belongs does not and
     /// will not access this page.
-    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error>;
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error>;
 
     /// Returns the [PhysicalAddress] and [MapAttributes] associated with given virtual `address`,
     /// if one is mapped

kernel/libk/libk-mm/interface/src/table.rs

@@ -37,6 +37,8 @@ bitflags! {
         const USER_WRITE = 1 << 1;
         /// The mapping is not global across the address spaces
         const NON_GLOBAL = 1 << 2;
+        /// The mapping is marked dirty by the OS
+        const DIRTY = 1 << 3;
     }
 }
 

kernel/libk/libk-mm/src/lib.rs

@@ -38,7 +38,7 @@ pub mod process;
 #[cfg(any(target_os = "none", rust_analyzer))]
 pub mod heap;
 
-pub use libk_mm_interface::table;
+pub use libk_mm_interface::{table, PageFaultKind};
 
 pub struct TableAllocatorImpl;
 
@@ -60,9 +60,23 @@ pub const L2_PAGE_SIZE: usize = 1 << 21;
 #[cfg(target_arch = "x86")]
 pub const L2_PAGE_SIZE: usize = 1 << 22;
 
+pub enum VirtualPage {
+    OnDemand,
+    Immediate(PhysicalAddress),
+}
+
+pub struct OnDemandPage {
+    pub physical: PhysicalAddress,
+    pub writeable: bool,
+}
+
 pub trait PageProvider: Send + Sync {
-    fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error>;
-    fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>;
+    fn ondemand_fetch(&self, offset: u64) -> Result<OnDemandPage, Error>;
+
+    fn get_page(&self, offset: u64) -> Result<VirtualPage, Error>;
+
+    fn release_page(&self, offset: u64, phys: PhysicalAddress, dirty: bool) -> Result<(), Error>;
+
     fn clone_page(
         &self,
         offset: u64,
@@ -130,7 +144,7 @@ impl<T> PageBox<T, GlobalPhysicalAllocator> {
         T: Copy,
     {
         let mut uninit = Self::new_uninit_slice(slice.len())?;
-        MaybeUninit::copy_from_slice(&mut uninit[..], slice);
+        uninit[..].write_copy_of_slice(slice);
         Ok(unsafe { uninit.assume_init_slice() })
     }
 }
@@ -343,7 +357,7 @@ impl<T, A: PhysicalMemoryAllocator<Address = PhysicalAddress>> PageBox<[MaybeUni
         // 1. MaybeUninit<T> is transparent
         // 2. self.value still points to the same memory and is not deallocated
         let page_count = self.page_count;
-        let value = MaybeUninit::slice_assume_init_mut(&mut *self.value);
+        let value = (&mut *self.value).assume_init_mut();
 
         core::mem::forget(self);
 
@@ -360,7 +374,7 @@ impl<T, A: PhysicalMemoryAllocator<Address = PhysicalAddress>> PageBox<[MaybeUni
     ///
     /// See [MaybeUninit::slice_assume_init_ref]
     pub unsafe fn assume_init_slice_ref(&self) -> &[T] {
-        MaybeUninit::slice_assume_init_ref(self.deref())
+        (&*self.value).assume_init_ref()
     }
 
     /// Returns a mutable reference to the slice data with [MaybeUninit] removed.
@@ -369,7 +383,7 @@ impl<T, A: PhysicalMemoryAllocator<Address = PhysicalAddress>> PageBox<[MaybeUni
     ///
     /// See [MaybeUninit::slice_assume_init_mut]
     pub unsafe fn assume_init_slice_mut(&mut self) -> &mut [T] {
-        MaybeUninit::slice_assume_init_mut(self.deref_mut())
+        (&mut *self.value).assume_init_mut()
     }
 
     /// Fills a slice of MaybeUninit<T> with zeroes.

kernel/libk/libk-mm/src/phys/mod.rs

@@ -219,7 +219,7 @@ pub unsafe fn init_from_iter<
     );
 
     let mut collected = 0;
-    const MAX_MEMORY: usize = 64 * 1024;
+    const MAX_MEMORY: usize = 256 * 1024;
 
     for (start, end) in it
         .into_iter()

kernel/libk/libk-mm/src/process.rs

@@ -4,8 +4,9 @@ use alloc::sync::Arc;
 use kernel_arch::ProcessAddressSpaceImpl;
 use libk_mm_interface::{
     address::PhysicalAddress,
-    process::ProcessAddressSpaceManager,
+    process::{PageAttributeUpdate, ProcessAddressSpaceManager},
     table::{MapAttributes, TableAllocator},
+    PageFaultKind,
 };
 use libk_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard};
 use vmalloc::{RangeData, VirtualMemoryAllocator};
@@ -14,7 +15,7 @@ use yggdrasil_abi::error::Error;
 use crate::{
     phys,
     pointer::{PhysicalRef, PhysicalRefMut},
-    PageProvider, TableAllocatorImpl, L3_PAGE_SIZE,
+    OnDemandPage, PageProvider, TableAllocatorImpl, VirtualPage, L3_PAGE_SIZE,
 };
 
 /// Describes how the physical memory is provided for the mapping
@@ -58,20 +59,27 @@ impl VirtualRangeBacking {
 }
 
 impl PageProvider for VirtualRangeBacking {
-    fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
+    fn ondemand_fetch(&self, opaque: u64) -> Result<OnDemandPage, Error> {
         match self {
-            Self::Anonymous => phys::alloc_page(),
+            Self::Anonymous => unreachable!(),
+            Self::File(f) => f.file.ondemand_fetch(f.offset + opaque),
+        }
+    }
+
+    fn get_page(&self, offset: u64) -> Result<VirtualPage, Error> {
+        match self {
+            Self::Anonymous => phys::alloc_page().map(VirtualPage::Immediate),
             Self::File(f) => f.file.get_page(f.offset + offset),
         }
     }
 
-    fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> {
+    fn release_page(&self, offset: u64, phys: PhysicalAddress, dirty: bool) -> Result<(), Error> {
         match self {
             Self::Anonymous => unsafe {
                 phys::free_page(phys);
                 Ok(())
             },
-            Self::File(f) => f.file.release_page(f.offset + offset, phys),
+            Self::File(f) => f.file.release_page(f.offset + offset, phys, dirty),
         }
     }
 
@@ -137,16 +145,27 @@ impl<TA: TableAllocator> Inner<TA> {
         for i in 0..page_count {
             let offset = (i * L3_PAGE_SIZE) as u64;
             let virt = address + i * L3_PAGE_SIZE;
-            let phys = match backing.get_page(offset) {
+            let page = match backing.get_page(offset) {
                 Ok(page) => page,
                 Err(err) => {
                     return Err((i, err));
                 }
             };
+            // let phys = match backing.get_page(offset) {
+            //     Ok(page) => page,
+            //     Err(err) => {
+            //         return Err((i, err));
+            //     }
+            // };
 
-            if let Err(err) = unsafe { self.table.map_page(virt, phys, attributes) } {
-                backing.release_page(offset, phys).unwrap();
-                return Err((i, err));
+            match page {
+                VirtualPage::OnDemand => (),
+                VirtualPage::Immediate(phys) => {
+                    if let Err(err) = unsafe { self.table.map_page(virt, phys, attributes) } {
+                        backing.release_page(offset, phys, false).unwrap();
+                        return Err((i, err));
+                    }
+                }
             }
         }
 
@@ -167,9 +186,9 @@ impl<TA: TableAllocator> Inner<TA> {
                         let offset = (pfn - origin_pfn) * L3_PAGE_SIZE;
                         let virt = pfn * L3_PAGE_SIZE;
 
-                        let phys = self.table.unmap_page(virt)?;
-
-                        backing.release_page(offset as u64, phys)?;
+                        if let Ok((phys, dirty)) = self.table.unmap_page(virt) {
+                            backing.release_page(offset as u64, phys, dirty)?;
+                        }
                     }
                 }
 
@@ -202,16 +221,16 @@ impl<TA: TableAllocator> Inner<TA> {
         Ok(())
     }
 
-    fn map_single(
+    fn map_single_anon(
         &mut self,
         address: usize,
-        backing: VirtualRangeBacking,
         attributes: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
         let start_pfn = address / L3_PAGE_SIZE;
-        self.allocator.insert(start_pfn, 1, backing.clone())?;
+        self.allocator
+            .insert(start_pfn, 1, VirtualRangeBacking::anonymous())?;
 
-        let phys = match backing.get_page(0) {
+        let phys = match phys::alloc_page() {
             Ok(page) => page,
             Err(err) => {
                 // Do nothing, as the page has not been allocated to this range yet
@@ -266,9 +285,13 @@ impl<TA: TableAllocator> Inner<TA> {
                     let offset = ((pfn - origin_pfn) * L3_PAGE_SIZE) as u64;
 
                     let virt = pfn * L3_PAGE_SIZE;
-                    let phys = self.table.unmap_page(virt)?;
+                    let (phys, dirty) = match self.table.unmap_page(virt) {
+                        Ok(res) => res,
+                        Err(Error::DoesNotExist) => continue,
+                        Err(error) => return Err(error),
+                    };
 
-                    backing.release_page(offset, phys)?;
+                    backing.release_page(offset, phys, dirty)?;
                 }
 
                 Ok(())
@@ -284,9 +307,13 @@ impl<TA: TableAllocator> Inner<TA> {
                 let offset = ((pfn - origin_pfn) * L3_PAGE_SIZE) as u64;
 
                 let virt = pfn * L3_PAGE_SIZE;
-                let phys = unsafe { self.table.unmap_page(virt)? };
+                let (phys, dirty) = match unsafe { self.table.unmap_page(virt) } {
+                    Ok(res) => res,
+                    Err(Error::DoesNotExist) => continue,
+                    Err(error) => return Err(error),
+                };
 
-                backing.release_page(offset, phys)?;
+                backing.release_page(offset, phys, dirty)?;
             }
 
             Ok(())
@@ -325,6 +352,78 @@ impl<TA: TableAllocator> Inner<TA> {
 
         Ok(())
     }
+
+    fn handle_fault(&mut self, address: usize, kind: PageFaultKind) -> Result<(), Error> {
+        let (ondemand, write) = match kind {
+            PageFaultKind::WriteFault => (false, true),
+            PageFaultKind::TranslationFault { write: true } => (true, true),
+            PageFaultKind::TranslationFault { write: false } => (true, false),
+            _ => return Err(Error::DoesNotExist),
+        };
+
+        if ondemand {
+            // TODO hardcoded "prefetch" hint
+            const PREFETCH_HINT: usize = 16;
+
+            let pfn = address / L3_PAGE_SIZE;
+            let (pfn_range, backing) = self.allocator.get(pfn).ok_or(Error::DoesNotExist)?;
+            let remaining = (pfn_range.end - pfn).min(PREFETCH_HINT);
+
+            // TODO rollback
+            for i in 0..remaining {
+                let address = (pfn + i) * L3_PAGE_SIZE;
+                let offset = (pfn - pfn_range.start + i) * L3_PAGE_SIZE;
+
+                if self.table.translate(address).is_ok() {
+                    continue;
+                }
+
+                // TODO query cpu flags to determine whether it supports hardware dirty bit
+                //      tracking. If so, just map data read-write here
+                let page = backing.ondemand_fetch(offset as u64)?;
+                let mut attrs = MapAttributes::USER_READ;
+                if page.writeable {
+                    // Used as an indicator the page can be made writeable, used on platforms
+                    // without hardware dirty state tracking
+                    attrs |= MapAttributes::DIRTY;
+                }
+
+                match unsafe { self.table.map_page(address, page.physical, attrs) } {
+                    Ok(()) => (),
+                    Err(error) => {
+                        log::warn!("Failed to map on-demand resolved page: {error:?}");
+                        backing
+                            .release_page(offset as u64, page.physical, false)
+                            .ok();
+                        return Err(error);
+                    }
+                }
+            }
+        }
+
+        if write && !cfg!(target_arch = "x86_64") {
+            // TODO this will break permission architecture, need to check if the range is actually
+            //      writeable first
+            let (_, attrs) = self.table.translate(address)?;
+
+            if attrs.contains(MapAttributes::DIRTY) {
+                unsafe {
+                    self.table.update_page_attributes(
+                        address & !(L3_PAGE_SIZE - 1),
+                        &PageAttributeUpdate {
+                            user_write: Some(true),
+                            dirty: Some(true),
+                        },
+                    )?;
+                }
+            } else {
+                log::warn!("Write to non-demand/non-dirty page @ {address:#x}");
+                return Err(Error::DoesNotExist);
+            }
+        }
+
+        Ok(())
+    }
 }
 
 impl<TA: TableAllocator> ProcessAddressSpace<TA> {
@@ -406,15 +505,14 @@ impl<TA: TableAllocator> ProcessAddressSpace<TA> {
     }
 
     /// Adds a single-page mapping to the address space
-    pub fn map_single(
+    pub fn map_single_anon(
         &self,
         address: usize,
-        backing: VirtualRangeBacking,
         attributes: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
         assert_eq!(address & (L3_PAGE_SIZE - 1), 0);
 
-        self.inner.lock().map_single(address, backing, attributes)
+        self.inner.lock().map_single_anon(address, attributes)
     }
 
     /// Returns the [PhysicalAddress] associated with given virtual `address`,
@@ -462,6 +560,11 @@ impl<TA: TableAllocator> ProcessAddressSpace<TA> {
         let mut inner = self.inner.lock();
         unsafe { inner.clear() }
     }
+
+    /// Handles a translation fault, possibly fetching on-demand pages
+    pub fn handle_fault(&self, address: usize, kind: PageFaultKind) -> Result<(), Error> {
+        self.inner.lock().handle_fault(address, kind)
+    }
 }
 
 impl<TA: TableAllocator> Drop for ProcessAddressSpace<TA> {

kernel/libk/libk-util/src/event.rs

@@ -35,6 +35,11 @@ pub struct OneTimeEvent<T> {
     notify: QueueWaker,
 }
 
+pub struct CounterEvent<N: EventNotify> {
+    counter: AtomicU64,
+    notify: N,
+}
+
 pub struct BitmapEvent<N: EventNotify> {
     value: AtomicU64,
     notify: N,
@@ -218,3 +223,39 @@ impl<N: EventNotify> BitmapEvent<N> {
         .await
     }
 }
+
+impl<N: EventNotify> CounterEvent<N> {
+    pub const fn new(notify: N) -> Self {
+        Self {
+            counter: AtomicU64::new(0),
+            notify,
+        }
+    }
+
+    pub fn signal(&self) {
+        self.counter.fetch_add(1, Ordering::Release);
+        self.notify.notify_all();
+    }
+
+    pub fn try_take(&self) -> Option<u64> {
+        let value = self.counter.swap(0, Ordering::Acquire);
+        if value > 0 {
+            Some(value)
+        } else {
+            None
+        }
+    }
+
+    pub async fn take(&self) -> u64 {
+        poll_fn(|cx| {
+            if let Some(value) = self.try_take() {
+                self.notify.unsubscribe(cx.waker());
+                Poll::Ready(value)
+            } else {
+                self.notify.subscribe(cx.waker());
+                Poll::Pending
+            }
+        })
+        .await
+    }
+}

kernel/libk/libk-util/src/hash_table.rs

@@ -49,6 +49,24 @@ impl<K: Hash + Eq, V, const N: usize> HashTable<K, V, DefaultHashBuilder, N> {
         }
     }
 
+    pub fn remove<Q>(&mut self, key: &Q) -> Option<V>
+    where
+        Q: Hash + Eq + ?Sized,
+        K: Borrow<Q>,
+    {
+        let h = self.hasher.hash_one(key);
+        let bucket = &mut self.buckets[h as usize % self.buckets.len()];
+
+        for i in 0..bucket.len() {
+            if bucket[i].0.borrow() == key {
+                let (_, value) = bucket.remove(i);
+                return Some(value);
+            }
+        }
+
+        None
+    }
+
     pub fn get<Q>(&self, key: &Q) -> Option<&V>
     where
         Q: Hash + Eq + ?Sized,

kernel/libk/libk-util/src/lib.rs

@@ -144,13 +144,13 @@ impl<T, const N: usize> Deref for StaticVector<T, N> {
     type Target = [T];
 
     fn deref(&self) -> &Self::Target {
-        unsafe { MaybeUninit::slice_assume_init_ref(&self.data[..self.len]) }
+        unsafe { self.data[..self.len].assume_init_ref() }
     }
 }
 
 impl<T, const N: usize> DerefMut for StaticVector<T, N> {
     fn deref_mut(&mut self) -> &mut Self::Target {
-        unsafe { MaybeUninit::slice_assume_init_mut(&mut self.data[..self.len]) }
+        unsafe { self.data[..self.len].assume_init_mut() }
     }
 }
 

kernel/libk/libk-util/src/ring.rs

@@ -29,6 +29,12 @@ pub struct LossyRingQueue<T: Copy> {
     read_notify: QueueWaker,
 }
 
+pub struct BlockingRingQueue<T: Copy> {
+    ring: IrqSafeSpinlock<RingBuffer<T>>,
+    write_notify: QueueWaker,
+    read_notify: QueueWaker,
+}
+
 impl<T: Copy> RingBuffer<T> {
     /// Constructs an empty [RingBuffer].
     ///
@@ -231,3 +237,86 @@ impl<T: Copy> LossyRingQueue<T> {
         self.read_notify.wake_all();
     }
 }
+
+impl<T: Copy> BlockingRingQueue<T> {
+    pub const fn with_capacity(capacity: usize) -> Self {
+        Self {
+            ring: IrqSafeSpinlock::new(RingBuffer::with_capacity(capacity)),
+            write_notify: QueueWaker::new(),
+            read_notify: QueueWaker::new(),
+        }
+    }
+
+    pub fn try_with_capacity(capacity: usize) -> Result<Self, Error> {
+        Ok(Self {
+            ring: IrqSafeSpinlock::new(RingBuffer::try_with_capacity(capacity)?),
+            write_notify: QueueWaker::new(),
+            read_notify: QueueWaker::new(),
+        })
+    }
+
+    pub async fn write(&self, data: T) {
+        poll_fn(|cx| {
+            if self.try_write(data) {
+                self.notify_readers();
+                self.write_notify.remove(cx.waker());
+                Poll::Ready(())
+            } else {
+                self.write_notify.register(cx.waker());
+                Poll::Pending
+            }
+        })
+        .await
+    }
+
+    pub async fn write_all(&self, data: &[T]) {
+        for &ch in data {
+            self.write(ch).await;
+        }
+    }
+
+    pub fn try_write(&self, data: T) -> bool {
+        let mut lock = self.ring.lock();
+        if !lock.is_writable() {
+            return false;
+        }
+        lock.write(data);
+        true
+    }
+
+    pub async fn read_lock(&self) -> IrqSafeSpinlockGuard<RingBuffer<T>> {
+        poll_fn(|cx| self.poll_read_lock(cx)).await
+    }
+
+    #[inline]
+    pub fn poll_read_lock(
+        &self,
+        cx: &mut Context<'_>,
+    ) -> Poll<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+        if let Some(lock) = self.try_read_lock() {
+            self.read_notify.remove(cx.waker());
+            Poll::Ready(lock)
+        } else {
+            self.read_notify.register(cx.waker());
+            Poll::Pending
+        }
+    }
+
+    #[inline]
+    pub fn try_read_lock(&self) -> Option<IrqSafeSpinlockGuard<RingBuffer<T>>> {
+        let lock = self.ring.lock();
+        lock.is_readable().then_some(lock)
+    }
+
+    pub fn poll_readable(&self, cx: &mut Context<'_>) -> Poll<()> {
+        self.poll_read_lock(cx).map(|_| ())
+    }
+
+    pub fn notify_readers(&self) {
+        self.read_notify.wake_all();
+    }
+
+    pub fn notify_writers(&self) {
+        self.write_notify.wake_all();
+    }
+}

kernel/libk/src/debug/mod.rs

@@ -7,15 +7,16 @@ use alloc::{
 use core::{
     fmt,
     str::FromStr,
-    sync::atomic::{AtomicBool, Ordering},
+    sync::atomic::{AtomicBool, AtomicU32, Ordering},
 };
 use ring::RingLoggerSink;
 
-use libk_util::{sync::IrqSafeSpinlock, OneTimeInit};
+use libk_util::OneTimeInit;
 use sink::DEBUG_SINKS;
 use yggdrasil_abi::error::Error;
 
 use crate::{
+    arch::Cpu,
     fs::sysfs::{
         attribute::{StringAttribute, StringAttributeOps},
         object::KObject,
@@ -31,7 +32,7 @@ pub use panic::{panic_log, PanicLoggerSink};
 pub use ring::add_kernel_log_file;
 pub use sink::{add_early_sink, add_serial_sink, add_sink, disable_early_sinks, DebugSink};
 
-static DEBUG_LOCK: IrqSafeSpinlock<()> = IrqSafeSpinlock::new(());
+static DEBUG_LOCK: AtomicU32 = AtomicU32::new(u32::MAX);
 static MUTE_DEBUG: AtomicBool = AtomicBool::new(false);
 
 pub struct MuteGuard(bool);
@@ -104,6 +105,17 @@ impl From<log::Level> for LogLevel {
     }
 }
 
+fn lock_debug() -> bool {
+    let cpu = Cpu::try_local().map_or(0, |cpu| cpu.id());
+    loop {
+        match DEBUG_LOCK.compare_exchange(u32::MAX, cpu, Ordering::Acquire, Ordering::Relaxed) {
+            Ok(_) => return true,
+            Err(x) if x == cpu => return false,
+            _ => core::hint::spin_loop(),
+        }
+    }
+}
+
 impl log::Log for KernelLoggerSink {
     fn enabled(&self, metadata: &log::Metadata) -> bool {
         metadata.target() != "io"
@@ -120,12 +132,15 @@ impl log::Log for KernelLoggerSink {
 
         RingLoggerSink.log(record);
 
-        let _guard = DEBUG_LOCK.lock();
+        if !lock_debug() {
+            return;
+        }
         for sink in DEBUG_SINKS.read().iter() {
             if sink.enabled(record.metadata()) {
                 sink.log(record);
             }
         }
+        DEBUG_LOCK.store(u32::MAX, Ordering::Release);
     }
 
     fn flush(&self) {}
@@ -165,9 +180,10 @@ fn make_sysfs_sink_object(index: usize) -> Arc<KObject<usize>> {
 }
 
 /// Print a trace message coming from a process
-pub fn program_trace(process: &Process, _thread: &Thread, message: &str) {
-    log::debug!(
+pub fn program_trace(process: &Process, _thread: &Thread, level: log::Level, message: &str) {
+    log::log!(
         target: ":program",
+        level,
         "{} ({}) {message}\n",
         process.name,
         process.id,

kernel/libk/src/debug/ring.rs

@@ -219,7 +219,7 @@ pub fn add_kernel_log_file(parent: &NodeRef) {
     let node = Node::regular(
         LogFile,
         NodeFlags::IN_MEMORY_SIZE | NodeFlags::IN_MEMORY_PROPS,
-        Some(Metadata::now_root(FileMode::new(0o400))),
+        Some(Metadata::now_root(FileMode::new(0o400), 0)),
         None,
     );
 

kernel/libk/src/device/block/cache.rs

@@ -1,8 +1,5 @@
 #![allow(clippy::missing_transmute_annotations)]
-use core::{
-    mem::MaybeUninit,
-    ops::{Deref, DerefMut},
-};
+use core::ops::{Deref, DerefMut};
 
 use alloc::sync::Arc;
 use libk_util::{lru_hash_table::LruCache, sync::spin_rwlock::IrqSafeRwLock};
@@ -132,7 +129,7 @@ impl UncachedCache {
         self.device
             .read_aligned(pos, buffer.slice_mut(0..self.block_size))
             .await?;
-        let result = mapper(unsafe { MaybeUninit::slice_assume_init_ref(&buffer[..]) })?;
+        let result = mapper(unsafe { buffer[..].assume_init_ref() })?;
         Ok(result)
     }
 

kernel/libk/src/device/block/mod.rs

@@ -121,11 +121,7 @@ pub trait BlockDevice: Device + PageProvider {
             let slice = read_buffer.slice_mut(0..block_count * bs);
             self.read_aligned(lba * bs as u64, slice).await?;
 
-            let src = unsafe {
-                MaybeUninit::slice_assume_init_ref(
-                    &read_buffer[block_offset..block_offset + amount],
-                )
-            };
+            let src = unsafe { read_buffer[block_offset..block_offset + amount].assume_init_ref() };
             let dst = &mut buffer[offset..offset + amount];
 
             dst.copy_from_slice(src);

kernel/libk/src/device/block/partition/mod.rs

@@ -3,7 +3,9 @@ use core::mem::MaybeUninit;
 use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use device_api::device::Device;
-use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider};
+use libk_mm::{
+    address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage,
+};
 use yggdrasil_abi::error::Error;
 
 use crate::dma::{DmaBuffer, DmaSlice, DmaSliceMut};
@@ -32,8 +34,21 @@ impl Partition {
 }
 
 impl PageProvider for Partition {
-    fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
-        todo!()
+    fn ondemand_fetch(&self, _opaque: u64) -> Result<OnDemandPage, Error> {
+        unimplemented!()
+    }
+
+    fn get_page(&self, _offset: u64) -> Result<VirtualPage, Error> {
+        unimplemented!()
+    }
+
+    fn release_page(
+        &self,
+        _offset: u64,
+        _phys: PhysicalAddress,
+        _dirty: bool,
+    ) -> Result<(), Error> {
+        unimplemented!()
     }
 
     fn clone_page(
@@ -42,11 +57,7 @@ impl PageProvider for Partition {
         _src_phys: PhysicalAddress,
         _src_attrs: MapAttributes,
     ) -> Result<PhysicalAddress, Error> {
-        todo!()
-    }
-
-    fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
-        todo!()
+        unimplemented!()
     }
 }
 

kernel/libk/src/device/char.rs

@@ -9,6 +9,13 @@ use crate::vfs::{CommonImpl, FileReadiness, NodeRef};
 
 #[async_trait]
 pub trait CharDevice: Device + FileReadiness {
+    fn open(&self) -> Result<(), Error> {
+        Ok(())
+    }
+    fn close(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
     async fn read(&self, buffer: &mut [u8]) -> Result<usize, Error> {
         self.read_nonblocking(buffer)
     }

kernel/libk/src/device/display/mod.rs

@@ -3,7 +3,10 @@ use core::mem::MaybeUninit;
 use alloc::{boxed::Box, sync::Arc};
 use async_trait::async_trait;
 use device_api::device::Device;
-use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider, L3_PAGE_SIZE};
+use libk_mm::{
+    address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage,
+    L3_PAGE_SIZE,
+};
 use yggdrasil_abi::{
     bitflags,
     error::Error,
@@ -214,6 +217,34 @@ impl BlockDevice for DisplayWrapper {
 }
 
 impl PageProvider for DisplayWrapper {
+    // fn clone_page(
+    //     &self,
+    //     offset: u64,
+    //     src_phys: PhysicalAddress,
+    //     src_attrs: MapAttributes,
+    // ) -> Result<PhysicalAddress, Error> {
+    //     self.device.clone_page(offset, src_phys, src_attrs)
+    // }
+
+    // fn get_page(&self, offset: u64) -> Result<VirtualPage, Error> {
+    //     self.device.get_page(offset)
+    // }
+
+    // fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> {
+    //     self.device.release_page(offset, phys)
+    // }
+    fn ondemand_fetch(&self, opaque: u64) -> Result<OnDemandPage, Error> {
+        self.device.ondemand_fetch(opaque)
+    }
+
+    fn get_page(&self, offset: u64) -> Result<VirtualPage, Error> {
+        self.device.get_page(offset)
+    }
+
+    fn release_page(&self, offset: u64, phys: PhysicalAddress, dirty: bool) -> Result<(), Error> {
+        self.device.release_page(offset, phys, dirty)
+    }
+
     fn clone_page(
         &self,
         offset: u64,
@@ -222,14 +253,6 @@ impl PageProvider for DisplayWrapper {
     ) -> Result<PhysicalAddress, Error> {
         self.device.clone_page(offset, src_phys, src_attrs)
     }
-
-    fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
-        self.device.get_page(offset)
-    }
-
-    fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> {
-        self.device.release_page(offset, phys)
-    }
 }
 
 impl Device for DisplayWrapper {

kernel/libk/src/dma.rs

@@ -85,7 +85,7 @@ impl<T> DmaBuffer<T> {
         T: Copy,
     {
         let mut uninit = DmaBuffer::new_uninit_slice(allocator, source.len())?;
-        MaybeUninit::copy_from_slice(&mut uninit[..], source);
+        uninit[..].write_copy_of_slice(source);
         Ok(unsafe { DmaBuffer::assume_init_slice(uninit) })
     }
 
@@ -247,6 +247,7 @@ impl<T: ?Sized> Drop for DmaBuffer<T> {
         log::trace!("Drop DmaBuffer @ {:#x}", self.host_physical);
         unsafe {
             ptr::drop_in_place(self.host_pointer.as_ptr());
+            #[cfg(any(rust_analyzer, target_os = "none"))]
             for i in 0..self.page_count {
                 phys::free_page(self.host_physical.add(i * L3_PAGE_SIZE));
             }

kernel/libk/src/fs/devfs.rs

@@ -60,7 +60,7 @@ pub fn add_named_char_device<S: AsRef<str>>(
     let node = Node::char(
         dev,
         NodeFlags::IN_MEMORY_PROPS,
-        Metadata::now_root(mode),
+        Metadata::now_root(mode, 0),
         None,
     );
 
@@ -80,7 +80,7 @@ pub fn add_named_block_device<S: AsRef<str>>(
     let node = Node::block(
         dev,
         NodeFlags::IN_MEMORY_PROPS,
-        Metadata::now_root(mode),
+        Metadata::now_root(mode, 0),
         None,
     );
 

kernel/libk/src/fs/sysfs/attribute/bytes.rs

@@ -118,7 +118,7 @@ impl<V: BytesAttributeOps> Attribute<V::Data> for BytesAttribute<V> {
                 _pd: PhantomData::<V>,
             },
             NodeFlags::IN_MEMORY_PROPS,
-            Some(Metadata::now_root(mode)),
+            Some(Metadata::now_root(mode, 0)),
             None,
         ))
     }

kernel/libk/src/fs/sysfs/attribute/string.rs

@@ -186,7 +186,7 @@ impl<V: StringAttributeOps> Attribute<V::Data> for StringAttribute<V> {
                 _pd: PhantomData::<V>,
             },
             NodeFlags::IN_MEMORY_PROPS,
-            Some(Metadata::now_root(mode)),
+            Some(Metadata::now_root(mode, 0)),
             None,
         ))
     }

kernel/libk/src/fs/sysfs/mod.rs

@@ -29,6 +29,10 @@ pub fn bus() -> Option<&'static Arc<KObject<()>>> {
     object::BUS_OBJECT.try_get()
 }
 
+pub fn proc() -> Option<&'static Arc<KObject<()>>> {
+    object::PROC_OBJECT.try_get()
+}
+
 pub fn init() {
     ROOT.init(object::setup_fixed_objects());
 }

kernel/libk/src/fs/sysfs/object.rs

@@ -18,7 +18,7 @@ impl<D> KObject<D> {
         let node = Node::directory(
             MemoryDirectory,
             NodeFlags::IN_MEMORY_SIZE | NodeFlags::IN_MEMORY_PROPS,
-            Some(Metadata::now_root(FileMode::new(0o555))),
+            Some(Metadata::now_root(FileMode::new(0o555), 0)),
             None,
         );
         Arc::new(Self { data, node })
@@ -46,6 +46,11 @@ impl<D> KObject<D> {
     pub(super) fn data(&self) -> &D {
         &self.data
     }
+
+    pub fn unlink(&self) {
+        let parent = self.node.parent();
+        parent.unlink_child(&self.node).ok();
+    }
 }
 
 impl<D> Deref for KObject<D> {
@@ -76,6 +81,8 @@ pub static DEVICE_OBJECT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
 pub static DEBUG_OBJECT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
 // `/bus`
 pub static BUS_OBJECT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
+// `/proc`
+pub static PROC_OBJECT: OneTimeInit<Arc<KObject<()>>> = OneTimeInit::new();
 
 fn setup_fixed_object(root: &Arc<KObject<()>>, obj: &OneTimeInit<Arc<KObject<()>>>, name: &str) {
     let obj = obj.init(KObject::new(()));
@@ -89,6 +96,7 @@ pub fn setup_fixed_objects() -> Arc<Node> {
     setup_fixed_object(root, &DEVICE_OBJECT, "device");
     setup_fixed_object(root, &DEBUG_OBJECT, "debug");
     setup_fixed_object(root, &BUS_OBJECT, "bus");
+    setup_fixed_object(root, &PROC_OBJECT, "proc");
 
     root.node.clone()
 }

kernel/libk/src/lib.rs

@@ -13,10 +13,8 @@
     never_type,
     let_chains,
     allocator_api,
-    maybe_uninit_uninit_array,
     trait_alias,
     if_let_guard,
-    trait_upcasting,
     arbitrary_self_types,
     slice_split_once,
     arbitrary_self_types_pointers,

kernel/libk/src/task/binary/elf.rs

@@ -189,7 +189,7 @@ pub fn load_elf_from_file<F: Read + Seek>(
     let (image_load_base, ip_offset) =
         elf_load_address(aslr, elf.ehdr.e_type, vaddr_min, image_load_size);
 
-    log::debug!(
+    log::info!(
         "Loading ELF virtual {:#x?} -> real {:#x?}",
         vaddr_min..vaddr_max,
         image_load_base..image_load_base + image_load_size

kernel/libk/src/task/binary/mod.rs

@@ -30,6 +30,7 @@ pub mod elf;
 
 pub type LoadedProcess = (Arc<Process>, Arc<Thread>);
 
+#[derive(Debug)]
 pub struct LoadOptions<'e, P: AsRef<Path>> {
     pub parent: Option<Arc<Process>>,
     pub group_id: ProcessGroupId,
@@ -112,14 +113,14 @@ impl<'a> ArgPlacer<'a> {
 fn setup_program_env(
     space: &ProcessAddressSpace,
     virt: usize,
+    real_program: Option<&str>,
     args: &Vec<String>,
     envs: &Vec<String>,
     aux: &[AuxValue],
 ) -> Result<usize, Error> {
     // TODO growing buffer
-    let phys_page = space.map_single(
+    let phys_page = space.map_single_anon(
         virt,
-        VirtualRangeBacking::anonymous(),
         MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL,
     )?;
     let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) };
@@ -139,6 +140,12 @@ fn setup_program_env(
     let argv = placer.put_ptr_array(&arg_ptrs)? + virt;
     let envp = placer.put_ptr_array(&env_ptrs)? + virt;
     let auxv = placer.put_aux_array(aux)? + virt;
+    let real_program = match real_program {
+        Some(path) => placer.put_str(path)? + virt,
+        _ => 0,
+    };
+
+    placer.align(size_of::<usize>())?;
 
     // Put ProgramArgumentInner struct
     let arg_address = placer.position + virt;
@@ -146,6 +153,7 @@ fn setup_program_env(
     placer.put(&argv)?;
     placer.put(&envp)?;
     placer.put(&auxv)?;
+    placer.put(&real_program)?;
 
     Ok(arg_address)
 }
@@ -154,6 +162,7 @@ fn setup_context<P>(
     options: &LoadOptions<P>,
     space: &ProcessAddressSpace,
     image: &ProcessImage,
+    image_path: &str,
     args: &Vec<String>,
     envs: &Vec<String>,
 ) -> Result<TaskContextImpl, Error>
@@ -174,6 +183,12 @@ where
         MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL,
     )?;
 
+    log::info!(
+        "stack: {:#x}..{:#x}",
+        virt_stack_base,
+        virt_stack_base + USER_STACK_PAGES * 0x1000
+    );
+
     let mut auxv = vec![];
     if let Some(tls_image) = image.tls_image.as_ref() {
         if tls_image.master_copy_base != 0 {
@@ -198,7 +213,7 @@ where
         });
     }
 
-    let argument = setup_program_env(space, virt_args_base, args, envs, &auxv)?;
+    let argument = setup_program_env(space, virt_args_base, Some(image_path), args, envs, &auxv)?;
 
     let user_sp =
         virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN;
@@ -241,6 +256,7 @@ fn setup_binary<S, P>(
     name: S,
     space: ProcessAddressSpace,
     image: ProcessImage,
+    image_path: &str,
     args: &Vec<String>,
     envs: &Vec<String>,
 ) -> Result<LoadedProcess, Error>
@@ -248,7 +264,7 @@ where
     S: Into<String>,
     P: AsRef<Path>,
 {
-    let context = setup_context(options, &space, &image, args, envs)?;
+    let context = setup_context(options, &space, &image, image_path, args, envs)?;
     let info = ProcessCreateInfo {
         name,
         context,
@@ -332,12 +348,12 @@ pub fn load<P: AsRef<Path>>(
     let space = ProcessAddressSpace::new()?;
     let (image, args, envs) =
         xxx_load_program(&space, ioctx, path, args, envs, !options.disable_aslr)?;
-    let name = path.display();
-    let name = match name.rsplit_once('/') {
+    let real_path = path.display();
+    let name = match real_path.rsplit_once('/') {
         Some((_, name)) => name,
-        None => name,
+        None => real_path,
     };
-    setup_binary(options, name, space, image, &args, &envs)
+    setup_binary(options, name, space, image, real_path, &args, &envs)
 }
 
 pub fn load_into<P: AsRef<Path>>(
@@ -359,7 +375,7 @@ pub fn load_into<P: AsRef<Path>>(
         envs,
         !options.disable_aslr,
     )?;
-    let context = setup_context(options, &space, &image, &args, &envs)?;
+    let context = setup_context(options, &space, &image, path.display(), &args, &envs)?;
 
     Ok((context, image))
 }

kernel/libk/src/task/debug.rs

@@ -1,5 +1,3 @@
-use yggdrasil_abi::{debug::DebugFrame, error::Error};
-
 pub struct ThreadDebugger {}
 
 impl ThreadDebugger {
@@ -10,16 +8,16 @@ impl ThreadDebugger {
     //     Self { channel }
     // }
 
-    pub fn send(&self, frame: &DebugFrame) -> Result<(), Error> {
-        let _ = frame;
-        todo!()
-        // let bytes = serde_json::to_vec(frame).unwrap();
-        // self.channel
-        //     .send_message(
-        //         MessagePayload::Data(bytes.into_boxed_slice()),
-        //         MessageDestination::AllExceptSelf,
-        //     )
-        //     .unwrap();
-        // Ok(())
-    }
+    // pub fn send(&self, frame: &DebugFrame) -> Result<(), Error> {
+    //     let _ = frame;
+    //     todo!()
+    //     // let bytes = serde_json::to_vec(frame).unwrap();
+    //     // self.channel
+    //     //     .send_message(
+    //     //         MessagePayload::Data(bytes.into_boxed_slice()),
+    //     //         MessageDestination::AllExceptSelf,
+    //     //     )
+    //     //     .unwrap();
+    //     // Ok(())
+    // }
 }

kernel/libk/src/task/process.rs

@@ -1,4 +1,5 @@
 use core::{
+    fmt,
     sync::atomic::{AtomicU32, AtomicUsize, Ordering},
     task::{Context, Poll},
 };
@@ -6,6 +7,7 @@ use core::{
 use abi_lib::SyscallRegister;
 use alloc::{
     collections::{btree_map, BTreeMap},
+    format,
     string::String,
     sync::{Arc, Weak},
     vec::Vec,
@@ -18,7 +20,7 @@ use libk_mm::{phys::GlobalPhysicalAllocator, process::ProcessAddressSpace};
 use libk_util::{
     event::{BoolEvent, OneTimeEvent},
     sync::{
-        spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockWriteGuard},
+        spin_rwlock::{IrqSafeRwLock, IrqSafeRwLockReadGuard, IrqSafeRwLockWriteGuard},
         IrqSafeSpinlock,
     },
 };
@@ -26,12 +28,17 @@ use yggdrasil_abi::{
     error::Error,
     option::OptionValue,
     process::{
-        options::ProcessOptionVariant, ExitCode, ProcessGroupId, ProcessId, Signal,
+        options::ProcessOptionVariant, ExitCode, ProcessGroupId, ProcessId, Signal, ThreadEvent,
         ThreadSpawnOptions, WaitFlags,
     },
 };
 
 use crate::{
+    fs::sysfs::{
+        self,
+        attribute::{StringAttribute, StringAttributeOps},
+        object::KObject,
+    },
     task::{
         futex::UserspaceMutex, thread::Thread, types::AllocateProcessId, TaskContextImpl, ThreadId,
     },
@@ -73,8 +80,6 @@ pub struct ProcessInner {
     mutexes: BTreeMap<usize, Arc<UserspaceMutex>>,
     space: Option<Arc<ProcessAddressSpace>>,
     image: Option<ProcessImage>,
-
-    thread_exits: BTreeMap<ThreadId, Arc<BoolEvent>>,
 }
 
 /// Describes a process within the system
@@ -91,6 +96,8 @@ pub struct Process {
     pub(crate) exit: OneTimeEvent<ExitCode>,
     pub(crate) child_exit_notify: BoolEvent,
 
+    sysfs_node: IrqSafeRwLock<Option<Arc<KObject<Weak<Process>>>>>,
+
     /// Process I/O information
     pub io: IrqSafeSpinlock<ProcessIo>,
 }
@@ -125,9 +132,13 @@ impl Process {
             signal_entry: AtomicUsize::new(0),
             child_exit_notify: BoolEvent::new(),
             exit: OneTimeEvent::new(),
+            sysfs_node: IrqSafeRwLock::new(None),
             io: IrqSafeSpinlock::new(ProcessIo::new()),
         });
 
+        let sysfs_node = add_sysfs_node(&process);
+        *process.sysfs_node.write() = Some(sysfs_node.clone());
+
         // Add a child if parent specified
         if let Some(parent) = parent {
             parent.inner.write().register_child(&process);
@@ -135,6 +146,7 @@ impl Process {
 
         // Create "main" thread
         let thread = Thread::new_uthread(process.id, Some(name), info.space, info.context);
+        thread.add_sysfs_node(&sysfs_node);
         process.inner.write().register_thread(thread.clone());
 
         MANAGER.register_process(process.clone());
@@ -209,6 +221,9 @@ impl Process {
         let thread = Thread::new_uthread(self.id, None, space.clone(), context);
         let id = thread.id;
 
+        if let Some(sysfs_node) = self.sysfs_node.read().as_ref() {
+            thread.add_sysfs_node(&sysfs_node);
+        }
         inner.register_thread(thread.clone());
 
         thread.enqueue();
@@ -304,6 +319,12 @@ impl Process {
             ProcessOptionVariant::SignalEntry => {
                 options::SignalEntry::store(&self.signal_entry(), buffer)
             }
+            ProcessOptionVariant::MainThread => {
+                let id = self.inner.read().threads[0].id;
+                let abi =
+                    unsafe { yggdrasil_abi::process::ThreadId::from_raw(id.as_user() as u32) };
+                options::MainThread::store(&abi, buffer)
+            }
         }
     }
 
@@ -322,6 +343,7 @@ impl Process {
                 self.set_signal_entry(value);
                 Ok(())
             }
+            ProcessOptionVariant::MainThread => Err(Error::ReadOnly),
         }
     }
 
@@ -421,7 +443,7 @@ impl Process {
 
         if let Some(status) = child.get_exit_status() {
             log::debug!("Child {id} exited with status {status:?} (didn't block)");
-            // TODO remove child
+            self.inner.write().remove_child(id);
             return Ok(status);
         }
 
@@ -433,7 +455,7 @@ impl Process {
             loop {
                 if let Some(status) = child.get_exit_status() {
                     log::debug!("Child {id} exited with status {status:?}", );
-                    // TODO remove child
+                    self.inner.write().remove_child(id);
                     break status;
                 }
 
@@ -442,13 +464,23 @@ impl Process {
         }
     }
 
+    fn poll_any_child_exit(&self) -> Option<(Arc<Process>, IrqSafeRwLockWriteGuard<ProcessInner>)> {
+        let read = self.inner.read();
+        if let Some(child) = read.any_exited_child() {
+            let write = IrqSafeRwLockReadGuard::upgrade(read);
+            Some((child, write))
+        } else {
+            None
+        }
+    }
+
     pub fn wait_for_any_child(&self, flags: WaitFlags) -> Result<(ProcessId, ExitCode), Error> {
-        if let Some(child) = self.inner.read().any_exited_child() {
+        if let Some((child, mut guard)) = self.poll_any_child_exit() {
             let id = child.id;
             // unwrap ok: ProcessInner tells the child already exited
             let status = child.get_exit_status().unwrap();
-            log::debug!("Child {id} exited with status {status:?} (didn't block)");
-            // TODO remove child
+            log::info!("Child {id} exited with status {status:?} (didn't block)");
+            guard.remove_child(id);
             return Ok((id, status));
         }
 
@@ -458,12 +490,12 @@ impl Process {
 
         block! {
             loop {
-                if let Some(child) = self.inner.read().any_exited_child() {
+                if let Some((child, mut guard)) = self.poll_any_child_exit() {
                     let id = child.id;
                     // unwrap ok: ProcessInner tells the child already exited
                     let status = child.get_exit_status().unwrap();
-                    log::debug!("Child {id} exited with status {status:?}", );
-                    // TODO remove child
+                    log::info!("Child {id} exited with status {status:?}", );
+                    guard.remove_child(id);
                     break (id, status);
                 }
 
@@ -478,6 +510,7 @@ impl Process {
 
     /// Cleans up process resources
     fn cleanup(&self, mut inner: IrqSafeRwLockWriteGuard<ProcessInner>) {
+        self.remove_sysfs_node();
         self.io.lock().handle_exit();
         inner.threads.clear();
         if let Some(space) = inner.space.take() {
@@ -487,6 +520,12 @@ impl Process {
         }
     }
 
+    fn remove_sysfs_node(&self) {
+        if let Some(node) = self.sysfs_node.write().take() {
+            node.unlink();
+        }
+    }
+
     /// Updates the signal entry point
     pub fn set_signal_entry(&self, entry: usize) -> usize {
         self.signal_entry.swap(entry, Ordering::Release)
@@ -583,18 +622,18 @@ impl Process {
         }
     }
 
-    pub async fn wait_for_thread(&self, thread: ThreadId) -> Result<(), Error> {
-        let exit = {
-            let inner = self.inner.read();
-            inner
-                .thread_exits
-                .get(&thread)
-                .cloned()
-                .ok_or(Error::DoesNotExist)?
-        };
+    pub async fn wait_for_thread(&self, thread: ThreadId) -> Result<ThreadEvent, Error> {
+        let thread = Thread::get(thread).ok_or(Error::ProcessNotFound)?;
 
-        exit.wait().await;
-        Ok(())
+        // Check that the process is the thread's tracer or parent
+        if !thread.is_child_of(self.id) && !thread.is_tracee_of(self.id) {
+            // Cannot wait for events from unrelated threads
+            log::warn!("{:?} is not tracer nor parent of {:?}", self.id, thread.id);
+            return Err(Error::PermissionDenied);
+        }
+        let with_trace = thread.is_tracee_of(self.id);
+
+        Ok(thread.events.wait(with_trace).await)
     }
 
     /// Terminates all threads with the exclusion of `except`. This may be useful when a thread
@@ -610,7 +649,7 @@ impl Process {
 
             log::debug!("Terminate thread {}", thread.id);
             thread.terminate();
-            thread.exit.wait().await;
+            thread.events.exit.wait().await;
             log::debug!("{} died", thread.id);
         }
 
@@ -618,6 +657,12 @@ impl Process {
     }
 }
 
+impl fmt::Debug for Process {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&self.id, f)
+    }
+}
+
 impl Drop for Process {
     fn drop(&mut self) {
         log::debug!("Drop Process {}", self.id);
@@ -647,13 +692,10 @@ impl ProcessInner {
             mutexes: BTreeMap::new(),
             image,
             space: space.clone(),
-
-            thread_exits: BTreeMap::new(),
         }
     }
 
     pub fn register_thread(&mut self, thread: Arc<Thread>) {
-        self.thread_exits.insert(thread.id, thread.exit.clone());
         self.threads.push(thread);
     }
 
@@ -673,6 +715,10 @@ impl ProcessInner {
             .find(|child| child.has_exited())
     }
 
+    pub fn remove_child(&mut self, id: ProcessId) -> bool {
+        self.children.remove(&id).is_some()
+    }
+
     pub fn remove_thread(&mut self, id: ThreadId) -> bool {
         let n = self.threads.len();
         self.threads.retain(|t| t.id != id);
@@ -743,3 +789,44 @@ impl ProcessIo {
         self.ioctx = Some(ioctx);
     }
 }
+
+fn add_sysfs_node(process: &Arc<Process>) -> Arc<KObject<Weak<Process>>> {
+    struct Name;
+    struct Parent;
+
+    impl StringAttributeOps for Name {
+        type Data = Weak<Process>;
+        const NAME: &'static str = "name";
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            let process = state.upgrade().ok_or(Error::ProcessNotFound)?;
+            Ok(process.name.clone())
+        }
+    }
+
+    impl StringAttributeOps for Parent {
+        type Data = Weak<Process>;
+        const NAME: &'static str = "parent";
+
+        fn read(state: &Self::Data) -> Result<String, Error> {
+            let process = state.upgrade().ok_or(Error::ProcessNotFound)?;
+            if let Some(parent) = process.parent.as_ref().and_then(Weak::upgrade) {
+                Ok(format!("{}", parent.id))
+            } else {
+                Ok("0".into())
+            }
+        }
+    }
+
+    let name = format!("{}", process.id);
+    let process = Arc::downgrade(process);
+    let object = KObject::new(process);
+
+    object.add_attribute(StringAttribute::from(Name)).ok();
+    object.add_attribute(StringAttribute::from(Parent)).ok();
+
+    if let Some(proc) = sysfs::proc() {
+        proc.add_object(name, object.clone()).ok();
+    }
+    object
+}

kernel/libk/src/task/thread.rs

@@ -1,7 +1,15 @@
-use core::{cell::Cell, mem::size_of, ops::Deref};
+use core::{
+    cell::Cell,
+    future::poll_fn,
+    mem::size_of,
+    ops::Deref,
+    sync::atomic::{AtomicU32, AtomicU64, Ordering},
+    task::Poll,
+};
 
 use alloc::{
-    collections::{btree_map, BTreeMap},
+    collections::BTreeMap,
+    format,
     string::String,
     sync::{Arc, Weak},
 };
@@ -9,35 +17,43 @@ use crossbeam_queue::SegQueue;
 use futures_util::task::ArcWake;
 use kernel_arch::{
     task::{Scheduler, TaskContext, TaskFrame},
-    Architecture, ArchitectureImpl, CpuImpl,
+    CpuImpl,
 };
-use libk_mm::process::ProcessAddressSpace;
+use libk_mm::{process::ProcessAddressSpace, PageFaultKind};
 use libk_util::{
     event::BoolEvent,
-    sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock},
+    ring::LossyRingQueue,
+    sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard},
 };
 use yggdrasil_abi::{
     arch::SavedFrame,
-    debug::DebugFrame,
+    debug::{self, DebugControl, TraceEvent, TraceEventPayload, TraceFlags},
     error::Error,
-    option::OptionValue,
+    option::{OptionValue, RequestValue},
     process::{
         thread::{ThreadOptionVariant, ThreadSignalStack},
-        ExitCode, ProcessId, Signal, SignalEntryData,
+        ExitCode, ProcessId, Signal, SignalEntryData, ThreadEvent,
     },
 };
 
-use crate::task::{
-    mem::ForeignPointer,
-    sched::CpuQueue,
-    types::{ThreadAffinity, ThreadId, ThreadState},
-    TaskContextImpl,
+use crate::{
+    fs::sysfs::{
+        attribute::{StringAttribute, StringAttributeOps},
+        object::KObject,
+    },
+    task::{
+        mem::ForeignPointer,
+        sched::CpuQueue,
+        types::{ThreadAffinity, ThreadId, ThreadState},
+        TaskContextImpl,
+    },
+    time::monotonic_time,
 };
 
-use super::{debug::ThreadDebugger, process::Process};
+use super::process::Process;
 
-type BreakpointType = <ArchitectureImpl as Architecture>::BreakpointType;
-const BREAKPOINT_VALUE: BreakpointType = ArchitectureImpl::BREAKPOINT_VALUE;
+// type BreakpointType = <ArchitectureImpl as Architecture>::BreakpointType;
+// const BREAKPOINT_VALUE: BreakpointType = ArchitectureImpl::BREAKPOINT_VALUE;
 
 /// Provides details about how the thread is scheduled onto CPUs
 pub struct ThreadSchedulingInfo {
@@ -50,18 +66,21 @@ pub struct ThreadSchedulingInfo {
     pub queue: Option<&'static CpuQueue>,
 }
 
-pub struct ThreadDebuggingInfo {
+pub struct ThreadTracingInfo {
+    pub tracer: Option<ProcessId>,
     pub single_step: bool,
-    pub restore_breakpoint: Option<usize>,
-    pub debugger: Option<ThreadDebugger>,
-    pub saved_frame: Option<SavedFrame>,
-    pub breakpoints: BTreeMap<usize, BreakpointType>,
+    pub state: SavedFrame,
 }
 
 pub struct ThreadInfo {
     pub signal_stack: ThreadSignalStack,
 }
 
+pub struct ThreadEvents {
+    pub exit: BoolEvent,
+    pub trace: LossyRingQueue<TraceEvent>,
+}
+
 /// Describes a single thread within the system
 pub struct Thread {
     /// Unique thread ID
@@ -74,19 +93,26 @@ pub struct Thread {
     pub context: Cell<TaskContextImpl>,
     process: Option<ProcessId>,
     space: Option<Arc<ProcessAddressSpace>>,
-    debug: IrqSafeSpinlock<ThreadDebuggingInfo>,
+    debug: IrqSafeSpinlock<ThreadTracingInfo>,
+    trace_flags: AtomicU32,
+    syscall_trace_seq: AtomicU64,
 
     // inner: IrqSafeSpinlock<ThreadInner>,
     info: IrqSafeRwLock<ThreadInfo>,
     signal_queue: SegQueue<Signal>,
+    sysfs_node: IrqSafeRwLock<Option<Arc<KObject<Weak<Thread>>>>>,
 
-    pub exit: Arc<BoolEvent>,
+    pub events: ThreadEvents,
     pub kill: BoolEvent,
 
     /// CPU scheduling affinity mask
     pub affinity: ThreadAffinity,
 }
 
+// Prevents thread from getting scheduled during some critical section
+#[allow(unused)]
+pub struct SuspendGuard<'a>(IrqSafeSpinlockGuard<'a, ThreadSchedulingInfo>);
+
 /// Wrapper which guarantees the thread referred to is the current one on the current CPU
 #[repr(C)]
 pub struct CurrentThread(Arc<Thread>, IrqGuard);
@@ -121,24 +147,20 @@ impl Thread {
                 in_queue: false,
                 queue: None,
             }),
-            // TODO lazy initialization for debugging info
-            debug: IrqSafeSpinlock::new(ThreadDebuggingInfo {
-                single_step: false,
-                restore_breakpoint: None,
-                debugger: None,
-                saved_frame: None,
-                breakpoints: BTreeMap::new(),
-            }),
+            debug: IrqSafeSpinlock::new(ThreadTracingInfo::default()),
             context: Cell::new(context),
             process,
             space,
+            trace_flags: AtomicU32::new(0),
+            syscall_trace_seq: AtomicU64::new(1),
 
             info: IrqSafeRwLock::new(ThreadInfo {
                 signal_stack: ThreadSignalStack::default(),
             }),
             signal_queue: SegQueue::new(),
-            exit: Arc::new(BoolEvent::new()),
+            events: ThreadEvents::new(),
             kill: BoolEvent::new(),
+            sysfs_node: IrqSafeRwLock::new(None),
 
             affinity: ThreadAffinity::any_cpu(),
         });
@@ -294,6 +316,11 @@ impl Thread {
         self.process.unwrap()
     }
 
+    /// Returns `true` if the thread is a child of given process
+    pub fn is_child_of(&self, pid: ProcessId) -> bool {
+        self.process.map_or(false, |p| p == pid)
+    }
+
     /// Returns the thread's parent process reference
     pub fn try_get_process(&self) -> Option<Arc<Process>> {
         self.process.and_then(Process::get)
@@ -315,78 +342,39 @@ impl Thread {
     /// Updates the thread's terminated status and wakes up any other threads waiting for it to
     /// exit
     pub fn set_terminated(&self) {
-        self.exit.signal_saturating();
+        self.events.exit.signal_saturating();
+    }
+
+    pub fn trace_event<F: TaskFrame>(&self, suspend: bool, frame: &F, payload: TraceEventPayload) {
+        let mut debug = self.debug.lock();
+        if debug.tracer.is_some() {
+            let timestamp = monotonic_time();
+            debug.store_state(frame);
+            self.events.trace.write(TraceEvent {
+                suspend,
+                timestamp,
+                payload,
+            });
+        }
     }
 
     // Signals
 
     /// Pushes a signal to the thread's signal queue
-    pub fn raise_signal(&self, signal: Signal) {
-        log::debug!("{}: raise signal {signal:?}", self.id);
-        self.signal_queue.push(signal);
-        self.enqueue();
-    }
+    pub fn raise_signal(&self, signal: Signal) -> bool {
+        let is_traced = self.debug.lock().tracer.is_some();
 
-    // Debugging
-    pub fn attach_debugger(&self, debugger: ThreadDebugger) {
-        // TODO kick out the previous debugger
-        let mut debug = self.debug.lock();
-
-        debug.saved_frame = None;
-        debug.single_step = true;
-        debug.debugger = Some(debugger);
-
-        self.signal_queue.push(Signal::Debug);
-
-        let frame = self
-            .process()
-            .map_image(|img| DebugFrame::Startup {
-                image_base: img.load_base,
-                ip_offset: img.ip_offset,
-                ip: img.entry,
-            })
-            .unwrap();
-
-        debug.debugger.as_mut().unwrap().send(&frame).ok();
-    }
-
-    pub fn set_breakpoint(&self, address: usize) -> Result<(), Error> {
-        log::debug!(
-            "Set breakpoint in {} ({:?}) @ {:#x}",
-            self.id,
-            *self.name.read(),
-            address
-        );
-        let mut debug = self.debug.lock();
-        debug.set_breakpoint_inner(self.address_space(), address)
-    }
-
-    pub fn read_memory(&self, address: usize, buffer: &mut [u8]) -> Result<(), Error> {
-        log::debug!(
-            "Read memory in {} ({:?}) @ {:#x}",
-            self.id,
-            *self.name.read(),
-            address
-        );
-
-        let space = self.address_space();
-
-        // TODO optimize this later
-        for (i, item) in buffer.iter_mut().enumerate() {
-            *item = unsafe { ((address + i) as *const u8).try_read_foreign_volatile(space) }?;
+        if is_traced && signal != Signal::Debug && signal != Signal::Killed {
+            log::info!("{}: caught traced signal {signal:?}", self.id);
+            // TODO
+            // self.trace_event(true, );
+            true
+        } else {
+            log::info!("{}: raise signal {signal:?}", self.id);
+            self.signal_queue.push(signal);
+            self.enqueue();
+            false
         }
-
-        Ok(())
-    }
-
-    pub fn resume(&self, single_step: bool) {
-        {
-            let mut debug = self.debug.lock();
-
-            debug.single_step = single_step;
-        }
-
-        self.enqueue();
     }
 
     // Scheduling
@@ -472,6 +460,18 @@ impl Thread {
         self.enqueue();
     }
 
+    fn cleanup(&self) {
+        // Can detach debugger now
+        {
+            let mut debug = self.debug.lock();
+            debug.tracer = None;
+            debug.single_step = false;
+            self.trace_flags.store(0, Ordering::Release);
+        }
+
+        self.drop_sysfs_node();
+    }
+
     /// Returns the current thread on the CPU.
     ///
     /// # Panics
@@ -508,6 +508,142 @@ impl Thread {
         let strong = weak.upgrade()?;
         Some(CurrentThread(strong, guard))
     }
+
+    // Tracing/debugging
+
+    /// Returns `true` if the thread is a tracee of given process
+    pub fn is_tracee_of(&self, pid: ProcessId) -> bool {
+        self.debug.lock().tracer.map_or(false, |p| p == pid)
+    }
+
+    pub fn attach_trace(&self, tracer: ProcessId) -> Result<(), Error> {
+        let mut debug = self.debug.lock();
+        if debug.tracer.is_some() {
+            log::warn!("{:?}: tracer already attached", self.id);
+            return Err(Error::AlreadyExists);
+        }
+        debug.tracer = Some(tracer);
+        debug.single_step = true;
+        log::info!("{:?}: tracer ({:?}) attached", self.id, tracer);
+        Ok(())
+    }
+
+    pub fn set_trace_flags(&self, flags: TraceFlags) {
+        let debug = self.debug.lock();
+        let value = if debug.tracer.is_some() {
+            flags.bits()
+        } else {
+            0
+        };
+        self.trace_flags.store(value, Ordering::Release);
+    }
+
+    pub fn trace_flags(&self) -> TraceFlags {
+        TraceFlags::new(self.trace_flags.load(Ordering::Acquire))
+    }
+
+    pub fn resume(&self, single_step: bool) {
+        {
+            let mut debug = self.debug.lock();
+
+            debug.single_step = single_step;
+        }
+
+        self.enqueue();
+    }
+
+    pub fn hold_suspended(&self) -> Result<SuspendGuard, Error> {
+        let sched = self.sched.lock();
+        if sched.state == ThreadState::Running {
+            return Err(Error::InvalidOperation);
+        }
+        Ok(SuspendGuard(sched))
+    }
+
+    pub fn debug_control(
+        &self,
+        option: u32,
+        buffer: &mut [u8],
+        len: usize,
+    ) -> Result<usize, Error> {
+        // Deserialize request
+        let input = &buffer[..len];
+        let option = DebugControl::try_from(option)?;
+
+        match option {
+            // TODO runtime attach
+            DebugControl::Attach => todo!(),
+            // TODO
+            DebugControl::Detach => todo!(),
+            DebugControl::Resume => {
+                let single_step = debug::Resume::load_request(input)?;
+                self.resume(single_step);
+                debug::Resume::store_response(&(), buffer)
+            }
+            DebugControl::SetTraceFlags => {
+                let flags = debug::SetTraceFlags::load_request(buffer)?;
+                self.set_trace_flags(flags);
+                debug::SetTraceFlags::store_response(&(), buffer)
+            }
+            DebugControl::GetTraceFlags => {
+                debug::GetTraceFlags::store_response(&self.trace_flags(), buffer)
+            }
+            DebugControl::GetRegisters => {
+                let _guard = self.hold_suspended()?;
+                let debug = self.debug.lock();
+                debug::GetRegisters::store_response(&debug.state, buffer)
+            }
+            DebugControl::SetRegisters => {
+                let _guard = self.hold_suspended()?;
+                todo!()
+            }
+            DebugControl::ReadMemory => {
+                let address = debug::ReadMemory::load_request(buffer)?;
+                let pointer = core::ptr::with_exposed_provenance::<usize>(address);
+                let _guard = self.hold_suspended()?;
+                let space = self.address_space();
+                let word = unsafe { pointer.try_read_foreign_volatile(space)? };
+                debug::ReadMemory::store_response(&word, buffer)
+            }
+            DebugControl::WriteMemory => {
+                todo!();
+            }
+        }
+    }
+
+    // sysfs operations
+
+    pub(super) fn add_sysfs_node(self: &Arc<Self>, parent: &Arc<KObject<Weak<Process>>>) {
+        let ThreadId::User(id) = self.id else {
+            return;
+        };
+
+        struct Name;
+
+        impl StringAttributeOps for Name {
+            type Data = Weak<Thread>;
+            const NAME: &'static str = "name";
+
+            fn read(state: &Self::Data) -> Result<String, Error> {
+                let thread = state.upgrade().ok_or(Error::ProcessNotFound)?;
+                let name = thread.name.read();
+                Ok(name.clone())
+            }
+        }
+
+        let object = KObject::new(Arc::downgrade(self));
+
+        object.add_attribute(StringAttribute::from(Name)).ok();
+
+        parent.add_object(format!("{id}"), object.clone()).ok();
+        *self.sysfs_node.write() = Some(object);
+    }
+
+    fn drop_sysfs_node(&self) {
+        if let Some(node) = self.sysfs_node.write().take() {
+            node.unlink();
+        }
+    }
 }
 
 impl GlobalThreadList {
@@ -552,13 +688,44 @@ impl CurrentThread {
         Arc::downgrade(&self.0)
     }
 
+    pub fn raise_signal(&self, signal: Signal) {
+        if self.0.raise_signal(signal) {
+            // Signal was intercepted and the thread will need to be suspended
+            log::info!("{}: suspending: signal intercepted by tracer", self.id);
+            self.suspend().ok();
+        }
+    }
+
+    // TODO point data
+    pub fn trace_point<F: TaskFrame, P: FnOnce() -> TraceEventPayload>(
+        &self,
+        filter: TraceFlags,
+        frame: &F,
+        producer: P,
+    ) {
+        if self.trace_flags().contains(filter) {
+            self.trace_event(true, frame, producer());
+            self.suspend().ok();
+        }
+    }
+
+    pub fn trace_syscall_entry<F: TaskFrame>(&self, frame: &F) -> u64 {
+        let seq = self.syscall_trace_seq.fetch_add(1, Ordering::Relaxed);
+        self.trace_point(TraceFlags::SYSCALL_ENTRY, frame, || {
+            TraceEventPayload::SyscallEntry(seq)
+        });
+        seq
+    }
+
+    pub fn trace_syscall_exit<F: TaskFrame>(&self, frame: &F, seq: u64) {
+        self.trace_point(TraceFlags::SYSCALL_EXIT, frame, || {
+            TraceEventPayload::SyscallExit(seq)
+        });
+    }
+
     /// Terminate the current thread
     pub fn exit(&self, code: ExitCode) -> ! {
-        // Can detach debugger now
-        let debug = self.debug.lock();
-        if let Some(debugger) = debug.debugger.as_ref() {
-            debugger.send(&DebugFrame::Exited).ok();
-        }
+        self.cleanup();
 
         if let Some(process) = self.try_get_process() {
             process.handle_thread_exit(self.id, code);
@@ -597,81 +764,63 @@ impl CurrentThread {
 
     pub fn handle_single_step<F: TaskFrame>(&self, frame: &mut F) -> bool {
         {
-            let mut debug = self.debug.lock();
-            if debug.debugger.is_none() {
-                return false;
+            let debug = self.debug.lock();
+            if !debug.single_step {
+                frame.set_single_step(false);
+                return true;
             }
-            let space = self.address_space();
-
-            if let Some(original) = debug.restore_breakpoint.take() {
-                let brk_range = original..original + size_of::<BreakpointType>();
-                assert!(!brk_range.contains(&frame.user_ip()));
-
-                log::debug!(
-                    "Restore breakpoint, current_ip={:#x}, breakpoint_ip={:#x}",
-                    frame.user_ip(),
-                    original
-                );
-                debug.set_breakpoint_inner(space, original).unwrap();
-            } else {
-                // Single step cleared
-                if !debug.single_step {
-                    log::debug!("Clear single step ({} {:?})", self.id, *self.name.read());
-                    frame.set_single_step(false);
-                    return true;
-                }
-            }
-
-            let frame = frame.store();
-            debug.saved_frame = Some(frame.clone());
-            // TODO handle cases of detached debugger
-            let debugger = debug.debugger.as_ref().unwrap();
-            debugger.send(&DebugFrame::Step { frame }).ok();
         }
 
+        self.trace_event(true, frame, TraceEventPayload::SingleStep);
+
         match self.suspend() {
             Ok(_) | Err(Error::Interrupted) => true,
-            Err(err) => panic!("TODO: handle error in debug suspend: {:?}", err),
+            Err(err) => panic!("TODO: handle error in single-step suspend: {err:?}"),
         }
     }
 
-    pub fn handle_breakpoint<F: TaskFrame>(&self, frame: &mut F) -> bool {
-        let mut debug = self.debug.lock();
-        let ip = frame.user_ip();
+    pub fn handle_breakpoint<F: TaskFrame>(&self, _frame: &mut F) -> bool {
+        false
+        // let mut debug = self.debug.lock();
+        // let ip = frame.user_ip();
 
-        if let Some(value) = debug.breakpoints.remove(&ip) {
-            let space = self.address_space();
+        // if let Some(value) = debug.breakpoints.remove(&ip) {
+        //     let space = self.address_space();
 
-            // Restore original code
-            let pointer = ip as *mut BreakpointType;
-            unsafe { pointer.write_foreign_volatile(space, value) };
+        //     // Restore original code
+        //     let pointer = ip as *mut BreakpointType;
+        //     unsafe { pointer.write_foreign_volatile(space, value) };
 
-            log::debug!(
-                "Thread {} ({:?}) hit a breakpoint @ {:#x}, step={}",
-                self.id,
-                *self.name.read(),
-                ip,
-                debug.single_step
-            );
+        //     log::debug!(
+        //         "Thread {} ({:?}) hit a breakpoint @ {:#x}, step={}",
+        //         self.id,
+        //         *self.name.read(),
+        //         ip,
+        //         debug.single_step
+        //     );
 
-            // TODO handle cases when no debugger is attached (clear breakpoint and resume)
-            frame.set_single_step(true);
+        //     // TODO handle cases when no debugger is attached (clear breakpoint and resume)
+        //     frame.set_single_step(true);
 
-            let frame = frame.store();
-            debug.restore_breakpoint = Some(ip);
-            debug.saved_frame = Some(frame.clone());
+        //     let frame = frame.store();
+        //     debug.restore_breakpoint = Some(ip);
+        //     debug.saved_frame = Some(frame.clone());
 
-            let debugger = debug.debugger.as_ref().unwrap();
-            debugger.send(&DebugFrame::HitBreakpoint { frame }).ok();
+        //     let debugger = debug.debugger.as_ref().unwrap();
+        //     debugger.send(&DebugFrame::HitBreakpoint { frame }).ok();
 
-            drop(debug);
+        //     drop(debug);
 
-            self.suspend().unwrap();
+        //     self.suspend().unwrap();
 
-            true
-        } else {
-            false
-        }
+        //     true
+        // } else {
+        //     false
+        // }
+    }
+
+    pub fn handle_page_fault(&self, address: usize, kind: PageFaultKind) -> Result<(), Error> {
+        self.address_space().handle_fault(address, kind)
     }
 
     /// Sets up a return frame to handle a pending signal, if any is present in the task's queue.
@@ -693,19 +842,8 @@ impl CurrentThread {
             );
 
             if signal == Signal::Debug {
-                frame.set_single_step(true);
-
-                let frame = frame.store();
-                let mut debug = self.debug.lock();
-
-                debug.single_step = true;
-                debug.saved_frame = Some(frame.clone());
-
-                let debugger = debug.debugger.as_ref().unwrap();
-
-                debugger.send(&DebugFrame::Step { frame }).ok();
-
-                return;
+                // Debugger attached in runtime
+                todo!()
             }
 
             let ip = self.process().signal_entry();
@@ -762,27 +900,70 @@ impl Deref for CurrentThread {
     }
 }
 
-impl ThreadDebuggingInfo {
-    fn set_breakpoint_inner(
-        &mut self,
-        space: &ProcessAddressSpace,
-        address: usize,
-    ) -> Result<(), Error> {
-        match self.breakpoints.entry(address) {
-            btree_map::Entry::Vacant(vacant) => {
-                let pointer = address as *mut BreakpointType;
+// impl ThreadTracingInfo {
+//     fn set_breakpoint_inner(
+//         &mut self,
+//         space: &ProcessAddressSpace,
+//         address: usize,
+//     ) -> Result<(), Error> {
+//         match self.breakpoints.entry(address) {
+//             btree_map::Entry::Vacant(vacant) => {
+//                 let pointer = address as *mut BreakpointType;
+//
+//                 // Read old code from the address space at that location
+//                 let original =
+//                     unsafe { (pointer as *const BreakpointType).try_read_foreign_volatile(space) }?;
+//
+//                 unsafe { pointer.write_foreign_volatile(space, BREAKPOINT_VALUE) };
+//
+//                 vacant.insert(original);
+//                 Ok(())
+//             }
+//             // No need, breakpoint already present
+//             btree_map::Entry::Occupied(_) => Err(Error::AlreadyExists),
+//         }
+//     }
+// }
 
-                // Read old code from the address space at that location
-                let original =
-                    unsafe { (pointer as *const BreakpointType).try_read_foreign_volatile(space) }?;
+impl ThreadTracingInfo {
+    pub fn store_state<F: TaskFrame>(&mut self, frame: &F) {
+        self.state = frame.store();
+    }
+}
 
-                unsafe { pointer.write_foreign_volatile(space, BREAKPOINT_VALUE) };
-
-                vacant.insert(original);
-                Ok(())
-            }
-            // No need, breakpoint already present
-            btree_map::Entry::Occupied(_) => Err(Error::AlreadyExists),
+impl Default for ThreadTracingInfo {
+    fn default() -> Self {
+        Self {
+            single_step: false,
+            tracer: None,
+            state: Default::default(),
         }
     }
 }
+
+impl ThreadEvents {
+    pub fn new() -> Self {
+        Self {
+            exit: BoolEvent::new(),
+            trace: LossyRingQueue::with_capacity(8),
+        }
+    }
+
+    pub async fn wait(&self, with_trace: bool) -> ThreadEvent {
+        poll_fn(|cx| {
+            if with_trace {
+                if let Poll::Ready(mut lock) = self.trace.poll_lock(cx) {
+                    let event = unsafe { lock.read_single_unchecked() };
+                    return Poll::Ready(ThreadEvent::Trace(event));
+                }
+            }
+
+            if self.exit.poll(cx).is_ready() {
+                Poll::Ready(ThreadEvent::Exited)
+            } else {
+                Poll::Pending
+            }
+        })
+        .await
+    }
+}